Hi,
I have installed openstack on RHEL7 machine using packstack and I have enabled and configured the heat service manually by following the steps provided in the openstack documentation.
I'm using rally to benchmark scenarios and the execution is failing with error message:
"public endpoint for orchestration service in RegionOne region not found".
I deleted the endpoint and re-created it with --region option and now the scenario is failing with:
HTTPServiceUnavailable: ERROR: 503 Service Unavailable
The server is currently unavailable. Please try again at a later time.
I googled for the error and couldn't find much help on the error. As suggested in one of the forums, i have manually executed some of the heat commands in debug mode and the console output is included below:
[root@localhost ~(keystone_admin)]# openstack orchestration service list --debug
START with options: ['orchestration', 'service', 'list', '--debug']
options: Namespace(access_token_endpoint='', auth_type='', auth_url='http://10.0.0.24:5000/v2.0', cacert='', client_id='', client_secret='***', cloud='', debug=True, default_domain='default', deferred_help=False, domain_id='', domain_name='', endpoint='', identity_provider='', identity_provider_url='', insecure=None, interface='', log_file=None, os_alarming_api_version='2', os_compute_api_version='', os_data_processing_api_version='1.1', os_data_processing_url='', os_dns_api_version='2', os_identity_api_version='', os_image_api_version='', os_key_manager_api_version='1', os_network_api_version='', os_object_api_version='', os_orchestration_api_version='1', os_project_id=None, os_project_name=None, os_queues_api_version='1.1', os_volume_api_version='', password='***', profile=None, project_domain_id='', project_domain_name='', project_id='', project_name='admin', protocol='', region_name='RegionOne', scope='', service_provider_endpoint='', timing=False, token='***', trust_id='', url='', user_domain_id='', user_domain_name='', user_id='', username='admin', verbose_level=3, verify=None)
defaults: {u'auth_type': 'password', u'compute_api_version': u'2', 'key': None, u'database_api_version': u'1.0', 'api_timeout': None, u'baremetal_api_version': u'1', u'image_api_version': u'2', 'cacert': None, u'image_api_use_tasks': False, u'floating_ip_source': u'neutron', u'orchestration_api_version': u'1', u'interface': None, u'network_api_version': u'2', u'image_format': u'qcow2', u'key_manager_api_version': u'v1', u'metering_api_version': u'2', 'verify': True, u'identity_api_version': u'2.0', u'volume_api_version': u'2', 'cert': None, u'secgroup_source': u'neutron', u'container_api_version': u'1', u'dns_api_version': u'2', u'object_store_api_version': u'1', u'disable_vendor_agent': {}}
cloud cfg: {'auth_type': 'password', u'compute_api_version': u'2', 'key': None, u'database_api_version': u'1.0', 'data_processing_api_version': '1.1', u'network_api_version': u'2', u'image_format': u'qcow2', u'image_api_version': u'2', 'verify': True, u'dns_api_version': '2', u'object_store_api_version': u'1', 'verbose_level': 3, 'region_name': 'RegionOne', 'api_timeout': None, u'baremetal_api_version': u'1', 'queues_api_version': '1.1', 'auth': {'username': 'admin', 'project_name': 'admin', 'password': '***', 'auth_url': 'http://10.0.0.24:5000/v2.0'}, 'default_domain': 'default', u'container_api_version': u'1', u'image_api_use_tasks': False, u'floating_ip_source': u'neutron', u'orchestration_api_version': '1', 'timing': False, 'cacert': None, u'key_manager_api_version': '1', u'metering_api_version': u'2', 'deferred_help': False, u'identity_api_version': u'2.0', u'volume_api_version': u'2', 'cert': None, u'secgroup_source': u'neutron', 'alarming_api_version': '2', 'debug': True, u'interface': None, u'disable_vendor_agent': {}}
compute API version 2, cmd group openstack.compute.v2
network API version 2, cmd group openstack.network.v2
image API version 2, cmd group openstack.image.v2
volume API version 2, cmd group openstack.volume.v2
identity API version 2.0, cmd group openstack.identity.v2
object_store API version 1, cmd group openstack.object_store.v1
dns API version 2, cmd group openstack.dns.v2
orchestration API version 1, cmd group openstack.orchestration.v1
data_processing API version 1.1, cmd group openstack.data_processing.v1
alarming API version 2, cmd group openstack.alarming.v2
messaging API version 1.1, cmd group openstack.messaging.v1
key_manager API version 1, cmd group openstack.key_manager.v1
command: orchestration service list -> heatclient.osc.v1.service.ListService
Auth plugin password selected
auth_type: password
Using auth plugin: password
Using parameters {'username': 'admin', 'password': '***', 'project_name': 'admin', 'auth_url': 'http://10.0.0.24:5000/v2.0'}
Get auth_ref
REQ: curl -g -i -X GET http://10.0.0.24:5000/v2.0 -H "Accept: application/json" -H "User-Agent: python-openstackclient keystoneauth1/2.4.1 python-requests/2.10.0 CPython/2.7.5"
Starting new HTTP connection (1): 10.0.0.24
"GET /v2.0 HTTP/1.1" 200 223
RESP: [200] Date: Mon, 05 Dec 2016 19:45:08 GMT Server: Apache/2.4.6 (Red Hat Enterprise Linux) Vary: X-Auth-Token,Accept-Encoding x-openstack-request-id: req-1f8f3da3-6fa3-407f-87e2-72df6f7c859a Content-Encoding: gzip Content-Length: 223 Connection: close Content-Type: application/json
RESP BODY: {"version": {"status": "stable", "updated": "2014-04-17T00:00:00Z", "media-types": [{"base": "application/json", "type": "application/vnd.openstack.identity-v2.0+json"}], "id": "v2.0", "links": [{"href": "http://10.0.0.24:5000/v2.0/", "rel": "self"}, {"href": "http://docs.openstack.org/", "type": "text/html", "rel": "describedby"}]}}
Making authentication request to http://10.0.0.24:5000/v2.0/tokens
Resetting dropped connection: 10.0.0.24
"POST /v2.0/tokens HTTP/1.1" 200 1130
take_action(Namespace(columns=[], formatter='table', max_width=0, noindent=False, quote_mode='nonnumeric'))
Instantiating orchestration client: <class 'heatclient.v1.client.Client'>
Making authentication request to http://10.0.0.24:5000/v2.0/tokens
Resetting dropped connection: 10.0.0.24
"POST /v2.0/tokens HTTP/1.1" 200 1128
REQ: curl -g -i -X GET http://10.0.0.24:8004/v1/9403f2f36b124ef581562737c269caba/services -H "User-Agent: python-heatclient" -H "Accept: application/json" -H "X-Auth-Token: {SHA1}fb2188ad1edd5a3d319c7bb918cd5285142bf739"
Starting new HTTP connection (1): 10.0.0.24
"GET /v1/9403f2f36b124ef581562737c269caba/services HTTP/1.1" 503 100
RESP: [503] Content-Length: 100 Content-Type: text/plain; charset=UTF-8 X-Openstack-Request-Id: req-fe1af4e4-5617-4203-911e-9742dcffb4cc Date: Mon, 05 Dec 2016 19:45:08 GMT Connection: keep-alive
RESP BODY: 503 Service Unavailable
The server is currently unavailable. Please try again at a later time.
ERROR: 503 Service Unavailable
The server is currently unavailable. Please try again at a later time.
Traceback (most recent call last):
File "/usr/lib/python2.7/site-packages/cliff/app.py", line 346, in run_subcommand
result = cmd.run(parsed_args)
File "/usr/lib/python2.7/site-packages/cliff/display.py", line 79, in run
column_names, data = self.take_action(parsed_args)
File "/usr/lib/python2.7/site-packages/heatclient/osc/v1/service.py", line 37, in take_action
services = heat_client.services.list()
File "/usr/lib/python2.7/site-packages/heatclient/v1/services.py", line 33, in list
return self._list(url, "services")
File "/usr/lib/python2.7/site-packages/heatclient/openstack/common/apiclient/base.py", line 135, in _list
body = self.client.get(url).json()
File "/usr/lib/python2.7/site-packages/keystoneclient/adapter.py", line 176, in get
return self.request(url, 'GET', **kwargs)
File "/usr/lib/python2.7/site-packages/heatclient/common/http.py", line 324, in request
raise exc.from_response(resp)
HTTPServiceUnavailable: ERROR: 503 Service Unavailable
The server is currently unavailable. Please try again at a later time.
clean_up ListService: ERROR: 503 Service Unavailable
The server is currently unavailable. Please try again at a later time.
Traceback (most recent call last):
File "/usr/lib/python2.7/site-packages/openstackclient/shell.py", line 118, in run
ret_val = super(OpenStackShell, self).run(argv)
File "/usr/lib/python2.7/site-packages/cliff/app.py", line 226, in run
result = self.run_subcommand(remainder)
File "/usr/lib/python2.7/site-packages/openstackclient/shell.py", line 153, in run_subcommand
ret_value = super(OpenStackShell, self).run_subcommand(argv)
File "/usr/lib/python2.7/site-packages/cliff/app.py", line 346, in run_subcommand
result = cmd.run(parsed_args)
File "/usr/lib/python2.7/site-packages/cliff/display.py", line 79, in run
column_names, data = self.take_action(parsed_args)
File "/usr/lib/python2.7/site-packages/heatclient/osc/v1/service.py", line 37, in take_action
services = heat_client.services.list()
File "/usr/lib/python2.7/site-packages/heatclient/v1/services.py", line 33, in list
return self._list(url, "services")
File "/usr/lib/python2.7/site-packages/heatclient/openstack/common/apiclient/base.py", line 135, in _list
body = self.client.get(url).json()
File "/usr/lib/python2.7/site-packages/keystoneclient/adapter.py", line 176, in get
return self.request(url, 'GET', **kwargs)
File "/usr/lib/python2.7/site-packages/heatclient/common/http.py", line 324, in request
raise exc.from_response(resp)
HTTPServiceUnavailable: ERROR: 503 Service Unavailable
The server is currently unavailable. Please try again at a later time.
i had restarted all the heat services after each config changes and it is not helping me.
Requesting someone to help me in solving the issue.
[root@localhost ~(keystone_admin)]# openstack service list
+----------------------------------+---------------+----------------+
| ID | Name | Type |
+----------------------------------+---------------+----------------+
| 06c8f11eaf5841679eba3b7c3c531c81 | neutron | network |
| 0cc0802cf3024edb89359da9ccd2e1ae | swift | object-store |
| 22be3b38f3a04e23a01772c0841d87b6 | aodh | alarming |
| 2641478361ca4159a934f5737002dbc5 | heat | orchestration |
| 58419da42033406dac021edfa2b21e7a | Image Service | image |
| 8b622654cf4840e5a0018b2c2d419e55 | cinder | volume |
| 9793e82246ce436786288d8e3934abca | gnocchi | metric |
| a09c9d35b401413b9e1e22409eb4398e | cinderv3 | volumev3 |
| a58b471d03b541f6b8a8cd1aa56574ca | nova | compute |
| acca13c16c8f489d9d018de727f289fc | cinderv2 | volumev2 |
| b509bb7a1a2f447291e80830f93a2bb9 | novav3 | computev3 |
| cb56dee818604ea6bb8ae962494d1f7e | heat-cfn | cloudformation |
| f954669a2a9e412fbd168cb48b103eb2 | ceilometer | metering |
| faefaa5375134a5f99dcb501285ba14f | keystone | identity |
+----------------------------------+---------------+----------------+
[root@localhost ~(keystone_admin)]#
[root@localhost ~(keystone_admin)]#
[root@localhost ~(keystone_admin)]# openstack endpoint list
+----------------------------------+-----------+---------------+----------------+
| ID | Region | Service Name | Service Type |
+----------------------------------+-----------+---------------+----------------+
| d35aa398996a44c49fc111294a4e9cc7 | RegionOne | heat | orchestration |
| 3706ef1d3f614b578e633f11aa377eae | RegionOne | nova | compute |
| edb6963bf75f4a6cba80cd932d027921 | RegionOne | Image Service | image |
| edbd88a1f86c4306a1547269d5c7e461 | RegionOne | aodh | alarming |
| 297e3d3970ba485f88c14c8be98ee509 | RegionOne | cinderv2 | volumev2 |
| 38b58717140d4b9db56604801a81afcf | RegionOne | cinder | volume |
| 968dfda45c094b40a3fa7da50d7d6194 | RegionOne | cinderv3 | volumev3 |
| daf77f5766ba44e18948e463d2c0f964 | RegionOne | ceilometer | metering |
| 4d69dff89d21486faa96d56c0388377c | RegionOne | novav3 | computev3 |
| 02a5e1ce4e9b4fd5ba9f268a469161eb | RegionOne | gnocchi | metric |
| b3573cffb6b445a8ad2b42b5d40e6c47 | RegionOne | neutron | network |
| ed07a909c1554ca3884448feb878dca5 | RegionOne | heat-cfn | cloudformation |
| 4beda1969ea749759e811ca4ce1ffb1d | RegionOne | swift | object-store |
| c93ac012aeaa452bbd96d58758597106 | RegionOne | keystone | identity |
+----------------------------------+-----------+---------------+----------------+
please let me know if i need to provide additional information.
Thanks,
Rajeev