From 98c30ac4315db112b1b58c8dc235f2e4d57dfe97 Mon Sep 17 00:00:00 2001 From: Bart Wensley Date: Mon, 25 Feb 2019 07:42:24 -0600 Subject: [PATCH] Improve VIM rabbitmq initialization robustness While attempting to reproduce a rabbitmq related failure on an AIO-DX system, I somehow caused a situation where the rabbitmq pods were running, but were missing some of the nova queues - specifically, the notifications.info queue was missing. When the VIM initialized, it attempted to attach a consumer to this non existent queue, resulting in an exception, which prevents the VIM from coming up, which causes SM to decide the host was unhealthy, which resulted in a swact to the other controller. The same problem happened there, which caused an endless series of swacts. The VIM already has code to check whether the nova exchange has been created in the rabbitmq pod, but it did not catch the case where the exchange was there, but some of the queues were missing. This fix updates the VIM to detect this situation and avoid attempting to create its consumer until the queue has been created. Change-Id: Ib5446bd15823cb0e7204ad8d0ff4f37270044c4b Closes-Bug: 1816766 Signed-off-by: Bart Wensley --- .../nfv_plugins/nfvi_plugins/nfvi_compute_api.py | 2 +- .../nfvi_plugins/openstack/rpc_listener.py | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_compute_api.py b/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_compute_api.py index 6bd842ec..27e84658 100755 --- a/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_compute_api.py +++ b/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_compute_api.py @@ -3449,7 +3449,7 @@ class NFVIComputeAPI(nfvi.api.v1.NFVIComputeAPI): return rpc_listener.test_connection( config.CONF['amqp']['host'], config.CONF['amqp']['port'], config.CONF['amqp']['user_id'], config.CONF['amqp']['password'], - config.CONF['amqp']['virt_host'], "nova") + config.CONF['amqp']['virt_host'], "nova", "notifications.info") def initialize(self, config_file): """ diff --git a/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/rpc_listener.py b/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/rpc_listener.py index 80200a1e..4120a9fc 100755 --- a/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/rpc_listener.py +++ b/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/rpc_listener.py @@ -149,7 +149,8 @@ class RPCListener(threading.Thread): self._exit.set() -def test_connection(host, port, user_id, password, virt_host, exchange_name): +def test_connection(host, port, user_id, password, virt_host, exchange_name, + queue_name): """ Test a connection to an exchange on a virtual host """ @@ -167,6 +168,15 @@ def test_connection(host, port, user_id, password, virt_host, exchange_name): exchange = Exchange(exchange_name, channel=connection, type='topic', durable=False, passive=True) exchange.declare() + + # Check whether the queue exists - will raise exception if it + # fails. + rpc_receive_queue = Queue(queue_name, + durable=True, + exchange=exchange, + channel=connection) + rpc_receive_queue.queue_declare(passive=True) + success = True except Exception as e: DLOG.info("Unable to connect to virt_host %s, exchange %s, error: %s" %