starlingx
/
distcloud
Code Issues Proposed changes

Merge "Extend subcloud audit to include loads audit"

This commit is contained in:
Zuul 2020-05-08 16:26:41 +00:00 committed by Gerrit Code Review
parent ff0c902314 6d4fa85546
commit 0a58888099
6 changed files with 244 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
distributedcloud/dccommon/drivers/openstack/sysinv_v1.py
View File

@ -160,6 +160,10 @@ class SysinvClient(base.DriverBase):
"""Get a list of loads."""
return self.sysinv_client.load.list()
def get_upgrades(self):
"""Get a list of upgrades."""
return self.sysinv_client.upgrade.list()
def get_applications(self):
"""Get a list of containerized applications"""

3
distributedcloud/dcmanager/common/consts.py
View File

@ -134,3 +134,6 @@ DEPLOY_COMMON_FILE_OPTIONS = [
DEPLOY_OVERRIDES,
DEPLOY_CHART
]
# Active load state
LOAD_STATE_ACTIVE = 'active'

67
distributedcloud/dcmanager/manager/patch_audit_manager.py
View File

@ -56,6 +56,7 @@ class PatchAuditManager(manager.Manager):
self.subcloud_manager = kwargs['subcloud_manager']
# Wait 20 seconds before doing the first audit
self.wait_time_passed = DEFAULT_PATCH_AUDIT_DELAY_SECONDS - 25
self.audit_count = 0
# Used to force an audit on the next interval
_force_audit = False
@ -98,11 +99,19 @@ class PatchAuditManager(manager.Manager):
except Exception as e:
LOG.exception(e)
def _update_subcloud_sync_status(self, sc_name, sc_endpoint_type, sc_status):
self.subcloud_manager.update_subcloud_endpoint_status(
self.context,
subcloud_name=sc_name,
endpoint_type=sc_endpoint_type,
sync_status=sc_status)
def _periodic_patch_audit_loop(self):
"""Audit patch status of subclouds loop."""
# We are running in our own green thread here.
LOG.info('Triggered patch audit.')
self.audit_count += 1
try:
m_os_ks_client = OpenStackDriver(
@ -119,6 +128,15 @@ class PatchAuditManager(manager.Manager):
regionone_patches = patching_client.query()
LOG.debug("regionone_patches: %s" % regionone_patches)
# Get the active software version in RegionOne as it may be needed
# later for subcloud load audit.
sysinv_client = SysinvClient(
consts.DEFAULT_REGION_NAME, m_os_ks_client.session)
regionone_loads = sysinv_client.get_loads()
for load in regionone_loads:
if load.state == consts.LOAD_STATE_ACTIVE:
regionone_software_version = load.software_version
# Build lists of patches that should be applied or committed in all
# subclouds, based on their state in RegionOne. Check repostate
# (not patchstate) as we only care if the patch has been applied to
@ -191,8 +209,12 @@ class PatchAuditManager(manager.Manager):
LOG.warn('Cannot retrieve loads for subcloud: %s' %
subcloud.name)
continue
subcloud_software_version = None
for load in loads:
installed_loads.append(load.software_version)
if load.state == consts.LOAD_STATE_ACTIVE:
subcloud_software_version = load.software_version
out_of_sync = False
@ -241,16 +263,41 @@ class PatchAuditManager(manager.Manager):
if out_of_sync:
LOG.debug("Subcloud %s is out-of-sync for patching" %
subcloud.name)
self.subcloud_manager.update_subcloud_endpoint_status(
self.context,
subcloud_name=subcloud.name,
endpoint_type=dcorch_consts.ENDPOINT_TYPE_PATCHING,
sync_status=consts.SYNC_STATUS_OUT_OF_SYNC)
self._update_subcloud_sync_status(
subcloud.name, dcorch_consts.ENDPOINT_TYPE_PATCHING,
consts.SYNC_STATUS_OUT_OF_SYNC)
else:
LOG.debug("Subcloud %s is in-sync for patching" %
subcloud.name)
self.subcloud_manager.update_subcloud_endpoint_status(
self.context,
subcloud_name=subcloud.name,
endpoint_type=dcorch_consts.ENDPOINT_TYPE_PATCHING,
sync_status=consts.SYNC_STATUS_IN_SYNC)
self._update_subcloud_sync_status(
subcloud.name, dcorch_consts.ENDPOINT_TYPE_PATCHING,
consts.SYNC_STATUS_IN_SYNC)
# Check subcloud software version every other audit cycle
if self.audit_count % 2 != 0:
LOG.debug('Auditing load of subcloud %s' % subcloud.name)
try:
upgrades = sysinv_client.get_upgrades()
except Exception:
LOG.warn('Cannot retrieve upgrade info for subcloud: %s' %
subcloud.name)
continue
if not upgrades:
# No upgrade in progress
if subcloud_software_version == regionone_software_version:
self._update_subcloud_sync_status(
subcloud.name, dcorch_consts.ENDPOINT_TYPE_LOAD,
consts.SYNC_STATUS_IN_SYNC)
else:
self._update_subcloud_sync_status(
subcloud.name, dcorch_consts.ENDPOINT_TYPE_LOAD,
consts.SYNC_STATUS_OUT_OF_SYNC)
else:
# As upgrade is still in progress, set the subcloud load
# status as out-of-sync.
self._update_subcloud_sync_status(
subcloud.name, dcorch_consts.ENDPOINT_TYPE_LOAD,
consts.SYNC_STATUS_OUT_OF_SYNC)
LOG.info('Patch audit completed.')

174
distributedcloud/dcmanager/tests/unit/manager/test_patch_audit_manager.py
View File

@ -53,8 +53,14 @@ class Subcloud(object):
class Load(object):
def __init__(self, software_version):
def __init__(self, software_version, state):
self.software_version = software_version
self.state = state
class Upgrade(object):
def __init__(self, state):
self.state = state
class FakePatchingClientInSync(object):
@ -175,11 +181,47 @@ class FakePatchingClientExtraPatches(object):
class FakeSysinvClientOneLoad(object):
def __init__(self, region, session):
self.loads = [Load('17.07')]
self.loads = [Load('17.07', 'active')]
self.upgrades = []
def get_loads(self):
return self.loads
def get_upgrades(self):
return self.upgrades
class FakeSysinvClientOneLoadUnmatchedSoftwareVersion(object):
def __init__(self, region, session):
self.region = region
self.loads = [Load('17.07', 'active')]
self.upgrades = []
def get_loads(self):
if self.region == 'subcloud2':
return [Load('17.06', 'active')]
else:
return self.loads
def get_upgrades(self):
return self.upgrades
class FakeSysinvClientOneLoadUpgradeInProgress(object):
def __init__(self, region, session):
self.region = region
self.loads = [Load('17.07', 'active')]
self.upgrades = []
def get_loads(self):
return self.loads
def get_upgrades(self):
if self.region == 'subcloud2':
return [Upgrade('started')]
else:
return self.upgrades
class TestAuditManager(base.DCManagerTestCase):
def setUp(self):
@ -232,10 +274,18 @@ class TestAuditManager(base.DCManagerTestCase):
subcloud_name='subcloud1',
endpoint_type=dcorch_consts.ENDPOINT_TYPE_PATCHING,
sync_status=consts.SYNC_STATUS_IN_SYNC),
mock.call(mock.ANY,
subcloud_name='subcloud1',
endpoint_type=dcorch_consts.ENDPOINT_TYPE_LOAD,
sync_status=consts.SYNC_STATUS_IN_SYNC),
mock.call(mock.ANY,
subcloud_name='subcloud2',
endpoint_type=dcorch_consts.ENDPOINT_TYPE_PATCHING,
sync_status=consts.SYNC_STATUS_IN_SYNC),
mock.call(mock.ANY,
subcloud_name='subcloud2',
endpoint_type=dcorch_consts.ENDPOINT_TYPE_LOAD,
sync_status=consts.SYNC_STATUS_IN_SYNC),
]
mock_sm.update_subcloud_endpoint_status.assert_has_calls(
expected_calls)
@ -277,22 +327,39 @@ class TestAuditManager(base.DCManagerTestCase):
subcloud_name='subcloud1',
endpoint_type=dcorch_consts.ENDPOINT_TYPE_PATCHING,
sync_status=consts.SYNC_STATUS_OUT_OF_SYNC),
mock.call(mock.ANY,
subcloud_name='subcloud1',
endpoint_type=dcorch_consts.ENDPOINT_TYPE_LOAD,
sync_status=consts.SYNC_STATUS_IN_SYNC),
mock.call(mock.ANY,
subcloud_name='subcloud2',
endpoint_type=dcorch_consts.ENDPOINT_TYPE_PATCHING,
sync_status=consts.SYNC_STATUS_OUT_OF_SYNC),
mock.call(mock.ANY,
subcloud_name='subcloud2',
endpoint_type=dcorch_consts.ENDPOINT_TYPE_LOAD,
sync_status=consts.SYNC_STATUS_IN_SYNC),
mock.call(mock.ANY,
subcloud_name='subcloud3',
endpoint_type=dcorch_consts.ENDPOINT_TYPE_PATCHING,
sync_status=consts.SYNC_STATUS_IN_SYNC),
mock.call(mock.ANY,
subcloud_name='subcloud3',
endpoint_type=dcorch_consts.ENDPOINT_TYPE_LOAD,
sync_status=consts.SYNC_STATUS_IN_SYNC),
mock.call(mock.ANY,
subcloud_name='subcloud4',
endpoint_type=dcorch_consts.ENDPOINT_TYPE_PATCHING,
sync_status=consts.SYNC_STATUS_OUT_OF_SYNC),
mock.call(mock.ANY,
subcloud_name='subcloud4',
endpoint_type=dcorch_consts.ENDPOINT_TYPE_LOAD,
sync_status=consts.SYNC_STATUS_IN_SYNC),
]
mock_sm.update_subcloud_endpoint_status.assert_has_calls(
expected_calls)
@mock.patch.object(patch_audit_manager, 'SysinvClient')
@mock.patch.object(patch_audit_manager, 'db_api')
@mock.patch.object(patch_audit_manager, 'PatchingClient')
@mock.patch.object(patch_audit_manager, 'OpenStackDriver')
@ -301,7 +368,8 @@ class TestAuditManager(base.DCManagerTestCase):
self, mock_context,
mock_openstack_driver,
mock_patching_client,
mock_db_api):
mock_db_api,
mock_sysinv_client):
mock_context.get_admin_context.return_value = self.ctxt
mock_sm = mock.Mock()
am = patch_audit_manager.PatchAuditManager(
@ -349,10 +417,110 @@ class TestAuditManager(base.DCManagerTestCase):
subcloud_name='subcloud1',
endpoint_type=dcorch_consts.ENDPOINT_TYPE_PATCHING,
sync_status=consts.SYNC_STATUS_OUT_OF_SYNC),
mock.call(mock.ANY,
subcloud_name='subcloud1',
endpoint_type=dcorch_consts.ENDPOINT_TYPE_LOAD,
sync_status=consts.SYNC_STATUS_IN_SYNC),
mock.call(mock.ANY,
subcloud_name='subcloud2',
endpoint_type=dcorch_consts.ENDPOINT_TYPE_PATCHING,
sync_status=consts.SYNC_STATUS_OUT_OF_SYNC),
mock.call(mock.ANY,
subcloud_name='subcloud2',
endpoint_type=dcorch_consts.ENDPOINT_TYPE_LOAD,
sync_status=consts.SYNC_STATUS_IN_SYNC),
]
mock_sm.update_subcloud_endpoint_status.assert_has_calls(
expected_calls)
@mock.patch.object(patch_audit_manager, 'SysinvClient')
@mock.patch.object(patch_audit_manager, 'db_api')
@mock.patch.object(patch_audit_manager, 'PatchingClient')
@mock.patch.object(patch_audit_manager, 'OpenStackDriver')
@mock.patch.object(patch_audit_manager, 'context')
def test_periodic_patch_audit_unmatched_software_version(
self, mock_context,
mock_openstack_driver,
mock_patching_client,
mock_db_api,
mock_sysinv_client):
mock_context.get_admin_context.return_value = self.ctxt
mock_sm = mock.Mock()
am = patch_audit_manager.PatchAuditManager(subcloud_manager=mock_sm)
mock_patching_client.side_effect = FakePatchingClientInSync
mock_sysinv_client.side_effect = FakeSysinvClientOneLoadUnmatchedSoftwareVersion
fake_subcloud1 = Subcloud(1, 'subcloud1',
is_managed=True, is_online=True)
fake_subcloud2 = Subcloud(2, 'subcloud2',
is_managed=True, is_online=True)
mock_db_api.subcloud_get_all.return_value = [fake_subcloud1,
fake_subcloud2]
am._periodic_patch_audit_loop()
expected_calls = [
mock.call(mock.ANY,
subcloud_name='subcloud1',
endpoint_type=dcorch_consts.ENDPOINT_TYPE_PATCHING,
sync_status=consts.SYNC_STATUS_IN_SYNC),
mock.call(mock.ANY,
subcloud_name='subcloud1',
endpoint_type=dcorch_consts.ENDPOINT_TYPE_LOAD,
sync_status=consts.SYNC_STATUS_IN_SYNC),
mock.call(mock.ANY,
subcloud_name='subcloud2',
endpoint_type=dcorch_consts.ENDPOINT_TYPE_PATCHING,
sync_status=consts.SYNC_STATUS_IN_SYNC),
mock.call(mock.ANY,
subcloud_name='subcloud2',
endpoint_type=dcorch_consts.ENDPOINT_TYPE_LOAD,
sync_status=consts.SYNC_STATUS_OUT_OF_SYNC),
]
mock_sm.update_subcloud_endpoint_status.assert_has_calls(
expected_calls)
@mock.patch.object(patch_audit_manager, 'SysinvClient')
@mock.patch.object(patch_audit_manager, 'db_api')
@mock.patch.object(patch_audit_manager, 'PatchingClient')
@mock.patch.object(patch_audit_manager, 'OpenStackDriver')
@mock.patch.object(patch_audit_manager, 'context')
def test_periodic_patch_audit_upgrade_in_progress(
self, mock_context,
mock_openstack_driver,
mock_patching_client,
mock_db_api,
mock_sysinv_client):
mock_context.get_admin_context.return_value = self.ctxt
mock_sm = mock.Mock()
am = patch_audit_manager.PatchAuditManager(subcloud_manager=mock_sm)
mock_patching_client.side_effect = FakePatchingClientInSync
mock_sysinv_client.side_effect = FakeSysinvClientOneLoadUpgradeInProgress
fake_subcloud1 = Subcloud(1, 'subcloud1',
is_managed=True, is_online=True)
fake_subcloud2 = Subcloud(2, 'subcloud2',
is_managed=True, is_online=True)
mock_db_api.subcloud_get_all.return_value = [fake_subcloud1,
fake_subcloud2]
am._periodic_patch_audit_loop()
expected_calls = [
mock.call(mock.ANY,
subcloud_name='subcloud1',
endpoint_type=dcorch_consts.ENDPOINT_TYPE_PATCHING,
sync_status=consts.SYNC_STATUS_IN_SYNC),
mock.call(mock.ANY,
subcloud_name='subcloud1',
endpoint_type=dcorch_consts.ENDPOINT_TYPE_LOAD,
sync_status=consts.SYNC_STATUS_IN_SYNC),
mock.call(mock.ANY,
subcloud_name='subcloud2',
endpoint_type=dcorch_consts.ENDPOINT_TYPE_PATCHING,
sync_status=consts.SYNC_STATUS_IN_SYNC),
mock.call(mock.ANY,
subcloud_name='subcloud2',
endpoint_type=dcorch_consts.ENDPOINT_TYPE_LOAD,
sync_status=consts.SYNC_STATUS_OUT_OF_SYNC),
]
mock_sm.update_subcloud_endpoint_status.assert_has_calls(
expected_calls)

10
distributedcloud/dcorch/common/consts.py
View File

@ -126,11 +126,17 @@ ENDPOINT_TYPE_PATCHING = "patching"
ENDPOINT_TYPE_IDENTITY = "identity"
ENDPOINT_TYPE_FM = "faultmanagement"
ENDPOINT_TYPE_NFV = "nfv"
ENDPOINT_TYPE_LOAD = "load"
# platform endpoint types
# All endpoint types
ENDPOINT_TYPES_LIST = [ENDPOINT_TYPE_PLATFORM,
ENDPOINT_TYPE_PATCHING,
ENDPOINT_TYPE_IDENTITY]
ENDPOINT_TYPE_IDENTITY,
ENDPOINT_TYPE_LOAD]
# Dcorch sync endpoint types
SYNC_ENDPOINT_TYPES_LIST = [ENDPOINT_TYPE_PLATFORM,
ENDPOINT_TYPE_IDENTITY]
ENDPOINT_QUOTA_MAPPING = {
ENDPOINT_TYPE_COMPUTE: NOVA_QUOTA_FIELDS,

4
distributedcloud/dcorch/engine/subcloud.py
View File

@ -48,9 +48,7 @@ class SubCloudEngine(object):
self.subcloud = subcloud
else:
capabilities = {}
endpoint_type_list = dco_consts.ENDPOINT_TYPES_LIST[:]
# patching is handled by dcmanager
endpoint_type_list.remove(dco_consts.ENDPOINT_TYPE_PATCHING)
endpoint_type_list = dco_consts.SYNC_ENDPOINT_TYPES_LIST[:]
capabilities.update({'endpoint_types': endpoint_type_list})
self.subcloud = Subcloud(
context, region_name=name, software_version=version,