I have installed sucessufully all openstack services, including heat on the same server and I was able to create stacks and start vms.
Then I tried to install heat on a dedicated server using the same packages and versions and it is able to communicate correctly with keystone and the amqp service.
I am able to do "heat-cfn list" without any problem, but when I try to create a new stack (using the same template that was working on the other installation) he says me "Timeout: Timeout while waiting on RPC response."
I'm using heat from rpm version 2013.1.2-3.el6.
I tried to use both qpid and rabbitmq, but the problem is always the same.
This is the log from heat-cfn client:
DEBUG:Debug level logging enabled
Traceback (most recent call last):
File "/usr/bin/heat-cfn", line 682, in <module>
main()
File "/usr/bin/heat-cfn", line 669, in main
result = cmd(opts, args)
File "/usr/lib/python2.6/site-packages/heat/cfn_client/utils.py", line 32, in wrapper
ret = func(*arguments, **kwargs)
File "/usr/bin/heat-cfn", line 216, in stack_create
result = c.create_stack(**parameters)
File "/usr/lib/python2.6/site-packages/heat/cfn_client/client.py", line 66, in create_stack
return self.stack_request("CreateStack", "POST", **kwargs)
File "/usr/lib/python2.6/site-packages/heat/cfn_client/client.py", line 55, in stack_request
res = self.do_request(method, "/", params=params, headers=headers)
File "/usr/lib/python2.6/site-packages/heat/common/client.py", line 49, in wrapped
return func(self, *args, **kwargs)
File "/usr/lib/python2.6/site-packages/heat/common/client.py", line 372, in do_request
headers=headers)
File "/usr/lib/python2.6/site-packages/heat/common/client.py", line 66, in wrapped
return func(self, method, url, body, headers)
File "/usr/lib/python2.6/site-packages/heat/common/client.py", line 496, in _do_request
raise Exception("Internal Server error: %s" % res.read())
Exception: Internal Server error: Traceback (most recent call last):
File "/usr/lib/python2.6/site-packages/eventlet/wsgi.py", line 382, in handle_one_response
result = self.application(self.environ, start_response)
File "/usr/lib/python2.6/site-packages/WebOb-1.0.8-py2.6.egg/webob/dec.py", line 147, in __call__
resp = self.call_func(req, *args, **self.kwargs)
File "/usr/lib/python2.6/site-packages/WebOb-1.0.8-py2.6.egg/webob/dec.py", line 208, in call_func
return self.func(req, *args, **kwargs)
File "/usr/lib/python2.6/site-packages/heat/common/wsgi.py", line 305, in __call__
response = req.get_response(self.application)
File "/usr/lib/python2.6/site-packages/WebOb-1.0.8-py2.6.egg/webob/request.py", line 1053, in get_response
application, catch_exc_info=False)
File "/usr/lib/python2.6/site-packages/WebOb-1.0.8-py2.6.egg/webob/request.py", line 1022, in call_application
app_iter = application(self.environ, start_response)
File "/usr/lib/python2.6/site-packages/WebOb-1.0.8-py2.6.egg/webob/dec.py", line 159, in __call__
return resp(environ, start_response)
File "/usr/lib/python2.6/site-packages/heat/common/auth_token.py", line 176, in __call__
return self.app(env, start_response)
File "/usr/lib/python2.6/site-packages/WebOb-1.0.8-py2.6.egg/webob/dec.py", line 147, in __call__
resp = self.call_func(req, *args, **self.kwargs)
File "/usr/lib/python2.6/site-packages/WebOb-1.0.8-py2.6.egg/webob/dec.py", line 208, in call_func
return self.func(req, *args, **kwargs)
File "/usr/lib/python2.6/site-packages/heat/common/wsgi.py", line 305, in __call__
response = req.get_response(self.application)
File "/usr/lib/python2.6/site-packages/WebOb-1.0.8-py2.6.egg/webob/request.py", line 1053, in get_response
application, catch_exc_info=False)
File "/usr/lib/python2.6/site-packages/WebOb-1.0.8-py2.6.egg/webob/request.py", line 1022, in call_application
app_iter = application(self.environ, start_response)
File "/usr/lib/python2.6/site-packages/WebOb-1.0.8-py2.6.egg/webob/dec.py", line 159, in __call__
return resp(environ, start_response)
File "/usr/lib/python2.6/site-packages/Routes-1.12.3-py2.6.egg/routes/middleware.py", line 131, in __call__
response = self.app(environ, start_response)
File "/usr/lib/python2.6/site-packages/WebOb-1.0.8-py2.6.egg/webob/dec.py", line 159, in __call__
return resp(environ, start_response)
File "/usr/lib/python2.6/site-packages/WebOb-1.0.8-py2.6.egg/webob/dec.py", line 147, in __call__
resp = self.call_func(req, *args, **self.kwargs)
File "/usr/lib/python2.6/site-packages/WebOb-1.0.8-py2.6.egg/webob/dec.py", line 208, in call_func
return self.func(req, *args, **kwargs)
File "/usr/lib/python2.6/site-packages/heat/common/wsgi.py", line 563, in __call__
request, **action_args)
File "/usr/lib/python2.6/site-packages/heat/common/wsgi.py", line 605, in dispatch
return method(*args, **kwargs)
File "/usr/lib/python2.6/site-packages/heat/api/cfn/v1/stacks.py", line 266, in create
return self.create_or_update(req, self.CREATE_STACK)
File "/usr/lib/python2.6/site-packages/heat/api/cfn/v1/stacks.py", line 335, in create_or_update
result = engine_action[action](con, **args)
File "/usr/lib/python2.6/site-packages/heat/rpc/client.py", line 109, in create_stack
topic=_engine_topic(self.topic, ctxt, None))
File "/usr/lib/python2.6/site-packages/heat/openstack/common/rpc/proxy.py", line 80, in call
return rpc.call(context, self._get_topic(topic), msg, timeout)
File "/usr/lib/python2.6/site-packages/heat/openstack/common/rpc/__init__.py", line 140, in call
return _get_impl().call(CONF, context, topic, msg, timeout)
File "/usr/lib/python2.6/site-packages/heat/openstack/common/rpc/impl_kombu.py", line 798, in call
rpc_amqp.get_connection_pool(conf, Connection))
File "/usr/lib/python2.6/site-packages/heat/openstack/common/rpc/amqp.py", line 614, in call
rv = list(rv)
File "/usr/lib/python2.6/site-packages/heat/openstack/common/rpc/amqp.py", line 500, in __iter__
raise rpc_common.Timeout()
Timeout: Timeout while waiting on RPC response.
This is the log from engine.log:
2013-07-26 12:53:49.446 28049 DEBUG heat.openstack.common.rpc.amqp [-] received {u'_context_roles': [u'admin'], u'_msg_id': u'5d4f555901df469ba8a02eb47fb46f86', u'args': {u'stack_name': [[[TEMPALTE CUTTED]]]]
2013-07-26 12:53:49.476 28049 INFO heat.engine.resource [-] Validating SecurityGroup "TestSecurityGroup"
2013-07-26 12:53:49.477 28049 INFO heat.engine.resource [-] Validating Instance "Test"
2013-07-26 12:54:52.508 28049 ERROR heat.openstack.common.rpc.amqp [-] Exception during message handling
2013-07-26 12:54:52.508 28049 TRACE heat.openstack.common.rpc.amqp Traceback (most recent call last):
2013-07-26 12:54:52.508 28049 TRACE heat.openstack.common.rpc.amqp File "/usr/lib/python2.6/site-packages/heat/openstack/common/rpc/amqp.py", line 430, in _process_data
2013-07-26 12:54:52.508 28049 TRACE heat.openstack.common.rpc.amqp rval = self.proxy.dispatch(ctxt, version, method, **args)
2013-07-26 12:54:52.508 28049 TRACE heat.openstack.common.rpc.amqp File "/usr/lib/python2.6/site-packages/heat/openstack/common/rpc/dispatcher.py", line 133, in dispatch
2013-07-26 12:54:52.508 28049 TRACE heat.openstack.common.rpc.amqp return getattr(proxyobj, method)(ctxt, **kwargs)
2013-07-26 12:54:52.508 28049 TRACE heat.openstack.common.rpc.amqp File "/usr/lib/python2.6/site-packages/heat/engine/service.py", line 49, in wrapped
2013-07-26 12:54:52.508 28049 TRACE heat.openstack.common.rpc.amqp return func(self, ctx, *args, **kwargs)
2013-07-26 12:54:52.508 28049 TRACE heat.openstack.common.rpc.amqp File "/usr/lib/python2.6/site-packages/heat/engine/service.py", line 219, in create_stack
2013-07-26 12:54:52.508 28049 TRACE heat.openstack.common.rpc.amqp stack.validate()
2013-07-26 12:54:52.508 28049 TRACE heat.openstack.common.rpc.amqp File "/usr/lib/python2.6/site-packages/heat/engine/parser.py", line 244, in validate
2013-07-26 12:54:52.508 28049 TRACE heat.openstack.common.rpc.amqp raise StackValidationFailed(message=str(ex))
2013-07-26 12:54:52.508 28049 TRACE heat.openstack.common.rpc.amqp StackValidationFailed: [Errno 110] ETIMEDOUT
2013-07-26 12:54:52.508 28049 TRACE heat.openstack.common.rpc.amqp
2013-07-26 12:54:52.510 28049 ERROR heat.openstack.common.rpc.common [-] Returning exception [Errno 110] ETIMEDOUT to caller
2013-07-26 12:54:52.510 28049 ERROR heat.openstack.common.rpc.common [-] ['Traceback (most recent call last):\n', ' File "/usr/lib/python2.6/site-packages/heat/openstack/common/rpc/amqp.py", line 430, in _process_data\n rval = self.proxy.dispatch(ctxt, version, method, **args)\n', ' File "/usr/lib/python2.6/site-packages/heat/openstack/common/rpc/dispatcher.py", line 133, in dispatch\n return getattr(proxyobj, method)(ctxt, **kwargs)\n', ' File "/usr/lib/python2.6/site-packages/heat/engine/service.py", line 49, in wrapped\n return func(self, ctx, *args, **kwargs)\n', ' File "/usr/lib/python2.6/site-packages/heat/engine/service.py", line 219, in create_stack\n stack.validate()\n', ' File "/usr/lib/python2.6/site-packages/heat/engine/parser.py", line 244, in validate\n raise StackValidationFailed(message=str(ex))\n', 'StackValidationFailed: [Errno 110] ETIMEDOUT\n']
2013-07-26 12:54:52.511 28049 DEBUG heat.openstack.common.rpc.amqp [-] UNIQUE_ID is adfa424eabae4c3dbfe8f2c63f3ac88a. _add_unique_id /usr/lib/python2.6/site-packages/heat/openstack/common/rpc/amqp.py:337
This is the log from api-cfn.log:
2013-07-26 12:53:49.255 28011 DEBUG heat.api.middleware.version_negotiation [-] Processing request: POST /v1/ Accept: process_request /usr/lib/python2.6/site-packages/heat/api/middleware/version_negotiation.py:51
2013-07-26 12:53:49.256 28011 DEBUG heat.api.middleware.version_negotiation [-] Matched versioned URI. Version: 1.0 process_request /usr/lib/python2.6/site-packages/heat/api/middleware/version_negotiation.py:63
2013-07-26 12:53:49.256 28011 INFO heat.api.aws.ec2token [-] Checking AWS credentials..
2013-07-26 12:53:49.257 28011 INFO heat.api.aws.ec2token [-] No AWS Signature found.
2013-07-26 12:53:49.258 28011 DEBUG heat.common.auth_token [-] Authenticating user token __call__ /usr/lib/python2.6/site-packages/heat/common/auth_token.py:169
2013-07-26 12:53:49.258 28011 DEBUG heat.common.auth_token [-] Removing headers from request environment: X-Identity-Status,X-Tenant-Id,X-Tenant-Name,X-User-Id,X-User-Name,X-Roles,X-User,X-Tenant,X-Role _remove_auth_headers /usr/lib/python2.6/site-packages/heat/common/auth_token.py:211
2013-07-26 12:53:49.419 28011 DEBUG heat.api.cfn.v1.stacks [-] TemplateBody ... _get_template /usr/lib/python2.6/site-packages/heat/api/cfn/v1/stacks.py:246
2013-07-26 12:53:49.426 28011 DEBUG heat.openstack.common.rpc.amqp [-] Making synchronous call on engine.heat-test001 ... multicall /usr/lib/python2.6/site-packages/heat/openstack/common/rpc/amqp.py:584
2013-07-26 12:53:49.427 28011 DEBUG heat.openstack.common.rpc.amqp [-] MSG_ID is 5d4f555901df469ba8a02eb47fb46f86 multicall /usr/lib/python2.6/site-packages/heat/openstack/common/rpc/amqp.py:587
2013-07-26 12:53:49.428 28011 DEBUG heat.openstack.common.rpc.amqp [-] UNIQUE_ID is 5b54271f1f304c378e38bb6109e55ffa. _add_unique_id /usr/lib/python2.6/site-packages/heat/openstack/common/rpc/amqp.py:337
2013-07-26 12:54:49.483 28011 ERROR heat.openstack.common.rpc.amqp [-] Timed out waiting for RPC response.
2013-07-26 12:54:49.483 28011 TRACE heat.openstack.common.rpc.amqp Traceback (most recent call last):
2013-07-26 12:54:49.483 28011 TRACE heat.openstack.common.rpc.amqp File "/usr/lib/python2.6/site-packages/heat/openstack/common/rpc/amqp.py", line 495, in __iter__
2013-07-26 12:54:49.483 28011 TRACE heat.openstack.common.rpc.amqp data = self._dataqueue.get(timeout=self._timeout)
2013-07-26 12:54:49.483 28011 TRACE heat.openstack.common.rpc.amqp File "/usr/lib/python2.6/site-packages/eventlet/queue.py", line 298, in get
2013-07-26 12:54:49.483 28011 TRACE heat.openstack.common.rpc.amqp return waiter.wait()
2013-07-26 12:54:49.483 28011 TRACE heat.openstack.common.rpc.amqp File "/usr/lib/python2.6/site-packages/eventlet/queue.py", line 129, in wait
2013-07-26 12:54:49.483 28011 TRACE heat.openstack.common.rpc.amqp return get_hub().switch()
2013-07-26 12:54:49.483 28011 TRACE heat.openstack.common.rpc.amqp File "/usr/lib/python2.6/site-packages/eventlet/hubs/hub.py", line 177, in switch
2013-07-26 12:54:49.483 28011 TRACE heat.openstack.common.rpc.amqp return self.greenlet.switch()
2013-07-26 12:54:49.483 28011 TRACE heat.openstack.common.rpc.amqp Empty
2013-07-26 12:54:49.483 28011 TRACE heat.openstack.common.rpc.amqp
2013-07-26 12:54:52.556 28011 WARNING heat.openstack.common.rpc.amqp [-] no calling threads waiting for msg_id : 5d4f555901df469ba8a02eb47fb46f86, message : {u'_unique_id': u'adfa424eabae4c3dbfe8f2c63f3ac88a', u'failure': u'{"args": [], "module": "heat.common.exception", "kwargs": {}, "message": "[Errno 110] ETIMEDOUT", "tb": ["Traceback (most recent call last):\\n", " File \\"/usr/lib/python2.6/site-packages/heat/openstack/common/rpc/amqp.py\\", line 430, in _process_data\\n rval = self.proxy.dispatch(ctxt, version, method, **args)\\n", " File \\"/usr/lib/python2.6/site-packages/heat/openstack/common/rpc/dispatcher.py\\", line 133, in dispatch\\n return getattr(proxyobj, method)(ctxt, **kwargs)\\n", " File \\"/usr/lib/python2.6/site-packages/heat/engine/service.py\\", line 49, in wrapped\\n return func(self, ctx, *args, **kwargs)\\n", " File \\"/usr/lib/python2.6/site-packages/heat/engine/service.py\\", line 219, in create_stack\\n stack.validate()\\n", " File \\"/usr/lib/python2.6/site-packages/heat/engine/parser.py\\", line 244, in validate\\n raise StackValidationFailed(message=str(ex))\\n", "StackValidationFailed: [Errno 110] ETIMEDOUT\\n"], "class": "StackValidationFailed"}', u'result': None}
How I can fix this probem? Is it normal the warning in api-cfn.log "heat.openstack.common.rpc.amqp [-] no calling threads waiting for msg_id : 5d4f555901df469ba8a02eb47fb46f8" ?