From 6d4fa855462cc6faf2e962f9d825b832f2885aa3 Mon Sep 17 00:00:00 2001 From: Tee Ngo Date: Mon, 4 May 2020 23:53:20 -0400 Subject: [PATCH] Extend subcloud audit to include loads audit In this commit, the existing patching audit logic is extended to include subcloud loads audit every other cycle since it already has to retrieve subcloud loads info in order to determine patching sync status. Tests: - Add and manage a few subclouds. - Unmanage and remove a couple of subclouds. - Verify that loads audit only occurs every other cycle. - Simulate update in progress in one of the subclouds and verify its sync status. - Simulate old software version in one of the subclouds and verify its sync status. Story: 2007403 Task: 39646 Change-Id: I496086bf90070a732b10bb7597c1af1f1fcd7952 Signed-off-by: Tee Ngo --- .../dccommon/drivers/openstack/sysinv_v1.py | 4 + distributedcloud/dcmanager/common/consts.py | 3 + .../dcmanager/manager/patch_audit_manager.py | 67 ++++++- .../unit/manager/test_patch_audit_manager.py | 174 +++++++++++++++++- distributedcloud/dcorch/common/consts.py | 10 +- distributedcloud/dcorch/engine/subcloud.py | 4 +- 6 files changed, 244 insertions(+), 18 deletions(-) diff --git a/distributedcloud/dccommon/drivers/openstack/sysinv_v1.py b/distributedcloud/dccommon/drivers/openstack/sysinv_v1.py index f72769bbc..45d7d9440 100644 --- a/distributedcloud/dccommon/drivers/openstack/sysinv_v1.py +++ b/distributedcloud/dccommon/drivers/openstack/sysinv_v1.py @@ -160,6 +160,10 @@ class SysinvClient(base.DriverBase): """Get a list of loads.""" return self.sysinv_client.load.list() + def get_upgrades(self): + """Get a list of upgrades.""" + return self.sysinv_client.upgrade.list() + def get_applications(self): """Get a list of containerized applications""" diff --git a/distributedcloud/dcmanager/common/consts.py b/distributedcloud/dcmanager/common/consts.py index 7bf6dd192..cd01aaeea 100644 --- a/distributedcloud/dcmanager/common/consts.py +++ b/distributedcloud/dcmanager/common/consts.py @@ -124,3 +124,6 @@ DEPLOY_COMMON_FILE_OPTIONS = [ DEPLOY_OVERRIDES, DEPLOY_CHART ] + +# Active load state +LOAD_STATE_ACTIVE = 'active' diff --git a/distributedcloud/dcmanager/manager/patch_audit_manager.py b/distributedcloud/dcmanager/manager/patch_audit_manager.py index f7f6020b0..115f958da 100644 --- a/distributedcloud/dcmanager/manager/patch_audit_manager.py +++ b/distributedcloud/dcmanager/manager/patch_audit_manager.py @@ -56,6 +56,7 @@ class PatchAuditManager(manager.Manager): self.subcloud_manager = kwargs['subcloud_manager'] # Wait 20 seconds before doing the first audit self.wait_time_passed = DEFAULT_PATCH_AUDIT_DELAY_SECONDS - 25 + self.audit_count = 0 # Used to force an audit on the next interval _force_audit = False @@ -98,11 +99,19 @@ class PatchAuditManager(manager.Manager): except Exception as e: LOG.exception(e) + def _update_subcloud_sync_status(self, sc_name, sc_endpoint_type, sc_status): + self.subcloud_manager.update_subcloud_endpoint_status( + self.context, + subcloud_name=sc_name, + endpoint_type=sc_endpoint_type, + sync_status=sc_status) + def _periodic_patch_audit_loop(self): """Audit patch status of subclouds loop.""" # We are running in our own green thread here. LOG.info('Triggered patch audit.') + self.audit_count += 1 try: m_os_ks_client = OpenStackDriver( @@ -119,6 +128,15 @@ class PatchAuditManager(manager.Manager): regionone_patches = patching_client.query() LOG.debug("regionone_patches: %s" % regionone_patches) + # Get the active software version in RegionOne as it may be needed + # later for subcloud load audit. + sysinv_client = SysinvClient( + consts.DEFAULT_REGION_NAME, m_os_ks_client.session) + regionone_loads = sysinv_client.get_loads() + for load in regionone_loads: + if load.state == consts.LOAD_STATE_ACTIVE: + regionone_software_version = load.software_version + # Build lists of patches that should be applied or committed in all # subclouds, based on their state in RegionOne. Check repostate # (not patchstate) as we only care if the patch has been applied to @@ -191,8 +209,12 @@ class PatchAuditManager(manager.Manager): LOG.warn('Cannot retrieve loads for subcloud: %s' % subcloud.name) continue + + subcloud_software_version = None for load in loads: installed_loads.append(load.software_version) + if load.state == consts.LOAD_STATE_ACTIVE: + subcloud_software_version = load.software_version out_of_sync = False @@ -241,16 +263,41 @@ class PatchAuditManager(manager.Manager): if out_of_sync: LOG.debug("Subcloud %s is out-of-sync for patching" % subcloud.name) - self.subcloud_manager.update_subcloud_endpoint_status( - self.context, - subcloud_name=subcloud.name, - endpoint_type=dcorch_consts.ENDPOINT_TYPE_PATCHING, - sync_status=consts.SYNC_STATUS_OUT_OF_SYNC) + self._update_subcloud_sync_status( + subcloud.name, dcorch_consts.ENDPOINT_TYPE_PATCHING, + consts.SYNC_STATUS_OUT_OF_SYNC) else: LOG.debug("Subcloud %s is in-sync for patching" % subcloud.name) - self.subcloud_manager.update_subcloud_endpoint_status( - self.context, - subcloud_name=subcloud.name, - endpoint_type=dcorch_consts.ENDPOINT_TYPE_PATCHING, - sync_status=consts.SYNC_STATUS_IN_SYNC) + self._update_subcloud_sync_status( + subcloud.name, dcorch_consts.ENDPOINT_TYPE_PATCHING, + consts.SYNC_STATUS_IN_SYNC) + + # Check subcloud software version every other audit cycle + if self.audit_count % 2 != 0: + LOG.debug('Auditing load of subcloud %s' % subcloud.name) + try: + upgrades = sysinv_client.get_upgrades() + except Exception: + LOG.warn('Cannot retrieve upgrade info for subcloud: %s' % + subcloud.name) + continue + + if not upgrades: + # No upgrade in progress + if subcloud_software_version == regionone_software_version: + self._update_subcloud_sync_status( + subcloud.name, dcorch_consts.ENDPOINT_TYPE_LOAD, + consts.SYNC_STATUS_IN_SYNC) + else: + self._update_subcloud_sync_status( + subcloud.name, dcorch_consts.ENDPOINT_TYPE_LOAD, + consts.SYNC_STATUS_OUT_OF_SYNC) + else: + # As upgrade is still in progress, set the subcloud load + # status as out-of-sync. + self._update_subcloud_sync_status( + subcloud.name, dcorch_consts.ENDPOINT_TYPE_LOAD, + consts.SYNC_STATUS_OUT_OF_SYNC) + + LOG.info('Patch audit completed.') diff --git a/distributedcloud/dcmanager/tests/unit/manager/test_patch_audit_manager.py b/distributedcloud/dcmanager/tests/unit/manager/test_patch_audit_manager.py index 7c1a3a6e1..cdcd20f3c 100644 --- a/distributedcloud/dcmanager/tests/unit/manager/test_patch_audit_manager.py +++ b/distributedcloud/dcmanager/tests/unit/manager/test_patch_audit_manager.py @@ -53,8 +53,14 @@ class Subcloud(object): class Load(object): - def __init__(self, software_version): + def __init__(self, software_version, state): self.software_version = software_version + self.state = state + + +class Upgrade(object): + def __init__(self, state): + self.state = state class FakePatchingClientInSync(object): @@ -175,11 +181,47 @@ class FakePatchingClientExtraPatches(object): class FakeSysinvClientOneLoad(object): def __init__(self, region, session): - self.loads = [Load('17.07')] + self.loads = [Load('17.07', 'active')] + self.upgrades = [] def get_loads(self): return self.loads + def get_upgrades(self): + return self.upgrades + + +class FakeSysinvClientOneLoadUnmatchedSoftwareVersion(object): + def __init__(self, region, session): + self.region = region + self.loads = [Load('17.07', 'active')] + self.upgrades = [] + + def get_loads(self): + if self.region == 'subcloud2': + return [Load('17.06', 'active')] + else: + return self.loads + + def get_upgrades(self): + return self.upgrades + + +class FakeSysinvClientOneLoadUpgradeInProgress(object): + def __init__(self, region, session): + self.region = region + self.loads = [Load('17.07', 'active')] + self.upgrades = [] + + def get_loads(self): + return self.loads + + def get_upgrades(self): + if self.region == 'subcloud2': + return [Upgrade('started')] + else: + return self.upgrades + class TestAuditManager(base.DCManagerTestCase): def setUp(self): @@ -232,10 +274,18 @@ class TestAuditManager(base.DCManagerTestCase): subcloud_name='subcloud1', endpoint_type=dcorch_consts.ENDPOINT_TYPE_PATCHING, sync_status=consts.SYNC_STATUS_IN_SYNC), + mock.call(mock.ANY, + subcloud_name='subcloud1', + endpoint_type=dcorch_consts.ENDPOINT_TYPE_LOAD, + sync_status=consts.SYNC_STATUS_IN_SYNC), mock.call(mock.ANY, subcloud_name='subcloud2', endpoint_type=dcorch_consts.ENDPOINT_TYPE_PATCHING, sync_status=consts.SYNC_STATUS_IN_SYNC), + mock.call(mock.ANY, + subcloud_name='subcloud2', + endpoint_type=dcorch_consts.ENDPOINT_TYPE_LOAD, + sync_status=consts.SYNC_STATUS_IN_SYNC), ] mock_sm.update_subcloud_endpoint_status.assert_has_calls( expected_calls) @@ -277,22 +327,39 @@ class TestAuditManager(base.DCManagerTestCase): subcloud_name='subcloud1', endpoint_type=dcorch_consts.ENDPOINT_TYPE_PATCHING, sync_status=consts.SYNC_STATUS_OUT_OF_SYNC), + mock.call(mock.ANY, + subcloud_name='subcloud1', + endpoint_type=dcorch_consts.ENDPOINT_TYPE_LOAD, + sync_status=consts.SYNC_STATUS_IN_SYNC), mock.call(mock.ANY, subcloud_name='subcloud2', endpoint_type=dcorch_consts.ENDPOINT_TYPE_PATCHING, sync_status=consts.SYNC_STATUS_OUT_OF_SYNC), + mock.call(mock.ANY, + subcloud_name='subcloud2', + endpoint_type=dcorch_consts.ENDPOINT_TYPE_LOAD, + sync_status=consts.SYNC_STATUS_IN_SYNC), mock.call(mock.ANY, subcloud_name='subcloud3', endpoint_type=dcorch_consts.ENDPOINT_TYPE_PATCHING, sync_status=consts.SYNC_STATUS_IN_SYNC), + mock.call(mock.ANY, + subcloud_name='subcloud3', + endpoint_type=dcorch_consts.ENDPOINT_TYPE_LOAD, + sync_status=consts.SYNC_STATUS_IN_SYNC), mock.call(mock.ANY, subcloud_name='subcloud4', endpoint_type=dcorch_consts.ENDPOINT_TYPE_PATCHING, sync_status=consts.SYNC_STATUS_OUT_OF_SYNC), + mock.call(mock.ANY, + subcloud_name='subcloud4', + endpoint_type=dcorch_consts.ENDPOINT_TYPE_LOAD, + sync_status=consts.SYNC_STATUS_IN_SYNC), ] mock_sm.update_subcloud_endpoint_status.assert_has_calls( expected_calls) + @mock.patch.object(patch_audit_manager, 'SysinvClient') @mock.patch.object(patch_audit_manager, 'db_api') @mock.patch.object(patch_audit_manager, 'PatchingClient') @mock.patch.object(patch_audit_manager, 'OpenStackDriver') @@ -301,7 +368,8 @@ class TestAuditManager(base.DCManagerTestCase): self, mock_context, mock_openstack_driver, mock_patching_client, - mock_db_api): + mock_db_api, + mock_sysinv_client): mock_context.get_admin_context.return_value = self.ctxt mock_sm = mock.Mock() am = patch_audit_manager.PatchAuditManager( @@ -349,10 +417,110 @@ class TestAuditManager(base.DCManagerTestCase): subcloud_name='subcloud1', endpoint_type=dcorch_consts.ENDPOINT_TYPE_PATCHING, sync_status=consts.SYNC_STATUS_OUT_OF_SYNC), + mock.call(mock.ANY, + subcloud_name='subcloud1', + endpoint_type=dcorch_consts.ENDPOINT_TYPE_LOAD, + sync_status=consts.SYNC_STATUS_IN_SYNC), mock.call(mock.ANY, subcloud_name='subcloud2', endpoint_type=dcorch_consts.ENDPOINT_TYPE_PATCHING, sync_status=consts.SYNC_STATUS_OUT_OF_SYNC), + mock.call(mock.ANY, + subcloud_name='subcloud2', + endpoint_type=dcorch_consts.ENDPOINT_TYPE_LOAD, + sync_status=consts.SYNC_STATUS_IN_SYNC), + ] + mock_sm.update_subcloud_endpoint_status.assert_has_calls( + expected_calls) + + @mock.patch.object(patch_audit_manager, 'SysinvClient') + @mock.patch.object(patch_audit_manager, 'db_api') + @mock.patch.object(patch_audit_manager, 'PatchingClient') + @mock.patch.object(patch_audit_manager, 'OpenStackDriver') + @mock.patch.object(patch_audit_manager, 'context') + def test_periodic_patch_audit_unmatched_software_version( + self, mock_context, + mock_openstack_driver, + mock_patching_client, + mock_db_api, + mock_sysinv_client): + mock_context.get_admin_context.return_value = self.ctxt + mock_sm = mock.Mock() + am = patch_audit_manager.PatchAuditManager(subcloud_manager=mock_sm) + + mock_patching_client.side_effect = FakePatchingClientInSync + mock_sysinv_client.side_effect = FakeSysinvClientOneLoadUnmatchedSoftwareVersion + fake_subcloud1 = Subcloud(1, 'subcloud1', + is_managed=True, is_online=True) + fake_subcloud2 = Subcloud(2, 'subcloud2', + is_managed=True, is_online=True) + mock_db_api.subcloud_get_all.return_value = [fake_subcloud1, + fake_subcloud2] + + am._periodic_patch_audit_loop() + expected_calls = [ + mock.call(mock.ANY, + subcloud_name='subcloud1', + endpoint_type=dcorch_consts.ENDPOINT_TYPE_PATCHING, + sync_status=consts.SYNC_STATUS_IN_SYNC), + mock.call(mock.ANY, + subcloud_name='subcloud1', + endpoint_type=dcorch_consts.ENDPOINT_TYPE_LOAD, + sync_status=consts.SYNC_STATUS_IN_SYNC), + mock.call(mock.ANY, + subcloud_name='subcloud2', + endpoint_type=dcorch_consts.ENDPOINT_TYPE_PATCHING, + sync_status=consts.SYNC_STATUS_IN_SYNC), + mock.call(mock.ANY, + subcloud_name='subcloud2', + endpoint_type=dcorch_consts.ENDPOINT_TYPE_LOAD, + sync_status=consts.SYNC_STATUS_OUT_OF_SYNC), + ] + mock_sm.update_subcloud_endpoint_status.assert_has_calls( + expected_calls) + + @mock.patch.object(patch_audit_manager, 'SysinvClient') + @mock.patch.object(patch_audit_manager, 'db_api') + @mock.patch.object(patch_audit_manager, 'PatchingClient') + @mock.patch.object(patch_audit_manager, 'OpenStackDriver') + @mock.patch.object(patch_audit_manager, 'context') + def test_periodic_patch_audit_upgrade_in_progress( + self, mock_context, + mock_openstack_driver, + mock_patching_client, + mock_db_api, + mock_sysinv_client): + mock_context.get_admin_context.return_value = self.ctxt + mock_sm = mock.Mock() + am = patch_audit_manager.PatchAuditManager(subcloud_manager=mock_sm) + + mock_patching_client.side_effect = FakePatchingClientInSync + mock_sysinv_client.side_effect = FakeSysinvClientOneLoadUpgradeInProgress + fake_subcloud1 = Subcloud(1, 'subcloud1', + is_managed=True, is_online=True) + fake_subcloud2 = Subcloud(2, 'subcloud2', + is_managed=True, is_online=True) + mock_db_api.subcloud_get_all.return_value = [fake_subcloud1, + fake_subcloud2] + + am._periodic_patch_audit_loop() + expected_calls = [ + mock.call(mock.ANY, + subcloud_name='subcloud1', + endpoint_type=dcorch_consts.ENDPOINT_TYPE_PATCHING, + sync_status=consts.SYNC_STATUS_IN_SYNC), + mock.call(mock.ANY, + subcloud_name='subcloud1', + endpoint_type=dcorch_consts.ENDPOINT_TYPE_LOAD, + sync_status=consts.SYNC_STATUS_IN_SYNC), + mock.call(mock.ANY, + subcloud_name='subcloud2', + endpoint_type=dcorch_consts.ENDPOINT_TYPE_PATCHING, + sync_status=consts.SYNC_STATUS_IN_SYNC), + mock.call(mock.ANY, + subcloud_name='subcloud2', + endpoint_type=dcorch_consts.ENDPOINT_TYPE_LOAD, + sync_status=consts.SYNC_STATUS_OUT_OF_SYNC), ] mock_sm.update_subcloud_endpoint_status.assert_has_calls( expected_calls) diff --git a/distributedcloud/dcorch/common/consts.py b/distributedcloud/dcorch/common/consts.py index b22d6115a..8b8096439 100644 --- a/distributedcloud/dcorch/common/consts.py +++ b/distributedcloud/dcorch/common/consts.py @@ -126,11 +126,17 @@ ENDPOINT_TYPE_PATCHING = "patching" ENDPOINT_TYPE_IDENTITY = "identity" ENDPOINT_TYPE_FM = "faultmanagement" ENDPOINT_TYPE_NFV = "nfv" +ENDPOINT_TYPE_LOAD = "load" -# platform endpoint types +# All endpoint types ENDPOINT_TYPES_LIST = [ENDPOINT_TYPE_PLATFORM, ENDPOINT_TYPE_PATCHING, - ENDPOINT_TYPE_IDENTITY] + ENDPOINT_TYPE_IDENTITY, + ENDPOINT_TYPE_LOAD] + +# Dcorch sync endpoint types +SYNC_ENDPOINT_TYPES_LIST = [ENDPOINT_TYPE_PLATFORM, + ENDPOINT_TYPE_IDENTITY] ENDPOINT_QUOTA_MAPPING = { ENDPOINT_TYPE_COMPUTE: NOVA_QUOTA_FIELDS, diff --git a/distributedcloud/dcorch/engine/subcloud.py b/distributedcloud/dcorch/engine/subcloud.py index e9671c073..d979e0edc 100644 --- a/distributedcloud/dcorch/engine/subcloud.py +++ b/distributedcloud/dcorch/engine/subcloud.py @@ -48,9 +48,7 @@ class SubCloudEngine(object): self.subcloud = subcloud else: capabilities = {} - endpoint_type_list = dco_consts.ENDPOINT_TYPES_LIST[:] - # patching is handled by dcmanager - endpoint_type_list.remove(dco_consts.ENDPOINT_TYPE_PATCHING) + endpoint_type_list = dco_consts.SYNC_ENDPOINT_TYPES_LIST[:] capabilities.update({'endpoint_types': endpoint_type_list}) self.subcloud = Subcloud( context, region_name=name, software_version=version,