From 01d689f0f9362816b88df4d09cf5b447cc06c82d Mon Sep 17 00:00:00 2001 From: Hugo Brito Date: Wed, 21 Feb 2024 09:16:07 -0300 Subject: [PATCH] Create a software audit Today, the software audit is being conducted alongside the patch audit (load/patching). However, because the software audit uses a different RestAPI, any failures in it also result in patch audit failures. This commit creates a new software audit to separate it from the patch audit. Note that the patch audit will be removed later on. TEST PLAN: PASS: Manage a subcloud and get all audits in-sync. - Software, load, and patching audits working properly. PASS: Test with Software RestAPI disabled/broken. - Load and patching audits working properly - Software audit unknown status. Story: 2010676 Task: 49598 Change-Id: I38c00bfdf4d86d56e1e656892f7de32206755865 Signed-off-by: Hugo Brito --- distributedcloud/dccommon/consts.py | 4 + .../dcmanager/audit/patch_audit.py | 160 --------- distributedcloud/dcmanager/audit/rpcapi.py | 3 + distributedcloud/dcmanager/audit/service.py | 78 +++-- .../dcmanager/audit/software_audit.py | 209 ++++++++++++ .../dcmanager/audit/subcloud_audit_manager.py | 312 ++++++++++++------ .../audit/subcloud_audit_worker_manager.py | 43 ++- distributedcloud/dcmanager/audit/utils.py | 5 +- .../dcmanager/db/sqlalchemy/api.py | 49 +-- .../dcmanager/db/sqlalchemy/models.py | 21 +- .../orchestrator/patch_orch_thread.py | 3 +- .../orchestrator/software_orch_thread.py | 20 +- distributedcloud/dcmanager/state/service.py | 98 +++--- .../unit/audit/test_firmware_audit_manager.py | 13 +- .../unit/audit/test_kube_audit_manager.py | 13 +- .../test_kube_rootca_update_audit_manager.py | 11 +- .../unit/audit/test_patch_audit_manager.py | 16 +- .../test_subcloud_audit_worker_manager.py | 252 ++++++++------ distributedcloud/test-requirements.txt | 14 +- 19 files changed, 802 insertions(+), 522 deletions(-) create mode 100644 distributedcloud/dcmanager/audit/software_audit.py diff --git a/distributedcloud/dccommon/consts.py b/distributedcloud/dccommon/consts.py index f41dfde53..1daa50c11 100644 --- a/distributedcloud/dccommon/consts.py +++ b/distributedcloud/dccommon/consts.py @@ -107,12 +107,16 @@ ENDPOINT_TYPES_LIST = [ENDPOINT_TYPE_PLATFORM, ENDPOINT_TYPE_SOFTWARE] # All endpoint audit requests +# TODO(nicodemos): The ENDPOINT_TYPE_SOFTWARE will use the 'spare_audit_requested' +# temporarily until the USM feature is fully complete. Afterward, the software audit +# will replace the patch audit. ENDPOINT_AUDIT_REQUESTS = { ENDPOINT_TYPE_FIRMWARE: 'firmware_audit_requested', ENDPOINT_TYPE_KUBERNETES: 'kubernetes_audit_requested', ENDPOINT_TYPE_KUBE_ROOTCA: 'kube_rootca_update_audit_requested', ENDPOINT_TYPE_LOAD: 'load_audit_requested', ENDPOINT_TYPE_PATCHING: 'patch_audit_requested', + ENDPOINT_TYPE_SOFTWARE: 'spare_audit_requested', } # Well known region names diff --git a/distributedcloud/dcmanager/audit/patch_audit.py b/distributedcloud/dcmanager/audit/patch_audit.py index 40057abd7..05ad998fb 100644 --- a/distributedcloud/dcmanager/audit/patch_audit.py +++ b/distributedcloud/dcmanager/audit/patch_audit.py @@ -21,8 +21,6 @@ from dccommon import consts as dccommon_consts from dccommon.drivers.openstack import patching_v1 from dccommon.drivers.openstack.patching_v1 import PatchingClient from dccommon.drivers.openstack.sdk_platform import OpenStackDriver -from dccommon.drivers.openstack import software_v1 -from dccommon.drivers.openstack.software_v1 import SoftwareClient from dccommon.drivers.openstack.sysinv_v1 import SysinvClient from dcmanager.common import utils @@ -53,27 +51,6 @@ class PatchAuditData(object): return cls(**values) -class SoftwareAuditData(object): - def __init__(self, releases, deployed_release_ids, - committed_release_ids): - self.releases = releases - self.deployed_release_ids = deployed_release_ids - self.committed_release_ids = committed_release_ids - - def to_dict(self): - return { - 'releases': self.releases, - 'deployed_release_ids': self.deployed_release_ids, - 'committed_release_ids': self.committed_release_ids, - } - - @classmethod - def from_dict(cls, values): - if values is None: - return None - return cls(**values) - - class PatchAudit(object): """Manages tasks related to patch audits.""" @@ -102,45 +79,6 @@ class PatchAudit(object): sysinv_client.region_name) return upgrades - def get_software_regionone_audit_data(self): - """Query RegionOne to determine what releases should be deployed - - to the system as well as the current software version - - :return: A new SoftwareAuditData object - """ - try: - m_os_ks_client = OpenStackDriver( - region_name=dccommon_consts.DEFAULT_REGION_NAME, - region_clients=None).keystone_client - software_endpoint = m_os_ks_client.endpoint_cache.get_endpoint( - dccommon_consts.ENDPOINT_TYPE_SOFTWARE) - software_client = SoftwareClient(dccommon_consts.DEFAULT_REGION_NAME, - m_os_ks_client.session, - endpoint=software_endpoint) - except Exception: - LOG.exception('Failure initializing OS Client, skip software audit.') - return None - # First query RegionOne to determine what releases should be deployed - # to the system. - regionone_releases = software_client.query() - LOG.debug("regionone_releases: %s" % regionone_releases) - # Build lists of releases that should be deployed or committed in all - # subclouds, based on their state in RegionOne. - deployed_release_ids = list() - committed_release_ids = list() - for release_id in regionone_releases.keys(): - if regionone_releases[release_id]['state'] == \ - software_v1.DEPLOYED: - deployed_release_ids.append(release_id) - elif regionone_releases[release_id]['state'] == \ - software_v1.COMMITTED: - committed_release_ids.append(release_id) - LOG.debug("RegionOne deployed_release_ids: %s" % deployed_release_ids) - LOG.debug("RegionOne committed_release_ids: %s" % committed_release_ids) - return SoftwareAuditData(regionone_releases, deployed_release_ids, - committed_release_ids) - def get_regionone_audit_data(self): """Query RegionOne to determine what patches should be applied @@ -194,104 +132,6 @@ class PatchAudit(object): return PatchAuditData(regionone_patches, applied_patch_ids, committed_patch_ids, regionone_software_version) - def subcloud_audit( - self, subcloud_name, subcloud_region, audit_data, software_audit_data, - do_load_audit - ): - if software_audit_data: - self.subcloud_software_audit( - subcloud_name, subcloud_region, software_audit_data - ) - else: - self.subcloud_patch_audit(subcloud_name, subcloud_region, audit_data, - do_load_audit) - - def subcloud_software_audit(self, subcloud_name, subcloud_region, audit_data): - LOG.info('Triggered software audit for: %s.' % subcloud_name) - try: - sc_os_client = OpenStackDriver(region_name=subcloud_region, - region_clients=None).keystone_client - session = sc_os_client.session - software_endpoint = sc_os_client.endpoint_cache. \ - get_endpoint(dccommon_consts.ENDPOINT_TYPE_SOFTWARE) - software_client = SoftwareClient( - subcloud_region, session, - endpoint=software_endpoint) - except (keystone_exceptions.EndpointNotFound, - keystone_exceptions.ConnectFailure, - keystone_exceptions.ConnectTimeout, - IndexError): - LOG.exception("Endpoint for online subcloud %s not found, skip " - "software audit." % subcloud_name) - return - - # Retrieve all the releases that are present in this subcloud. - try: - subcloud_releases = software_client.query() - LOG.debug("Releases for subcloud %s: %s" % - (subcloud_name, subcloud_releases)) - except Exception: - LOG.warn('Cannot retrieve releases for subcloud: %s, skip software ' - 'audit' % subcloud_name) - return - - out_of_sync = False - - # audit_data will be a dict due to passing through RPC so objectify it - audit_data = SoftwareAuditData.from_dict(audit_data) - - # Check that all releases in this subcloud are in the correct - # state, based on the state of the release in RegionOne. For the - # subcloud. - for release_id in subcloud_releases.keys(): - if subcloud_releases[release_id]['state'] == \ - software_v1.DEPLOYED: - if release_id not in audit_data.deployed_release_ids: - if release_id not in audit_data.committed_release_ids: - LOG.debug("Release %s should not be deployed in %s" % - (release_id, subcloud_name)) - else: - LOG.debug("Release %s should be committed in %s" % - (release_id, subcloud_name)) - out_of_sync = True - elif subcloud_releases[release_id]['state'] == \ - software_v1.COMMITTED: - if (release_id not in audit_data.committed_release_ids and - release_id not in audit_data.deployed_release_ids): - LOG.warn("Release %s should not be committed in %s" % - (release_id, subcloud_name)) - out_of_sync = True - else: - # In steady state, all releases should either be deployed - # or committed in each subcloud. Release in other - # states mean a sync is required. - out_of_sync = True - - # Check that all deployed or committed releases in RegionOne are - # present in the subcloud. - for release_id in audit_data.deployed_release_ids: - if release_id not in subcloud_releases: - LOG.debug("Release %s missing from %s" % - (release_id, subcloud_name)) - out_of_sync = True - for release_id in audit_data.committed_release_ids: - if release_id not in subcloud_releases: - LOG.debug("Release %s missing from %s" % - (release_id, subcloud_name)) - out_of_sync = True - - if out_of_sync: - self._update_subcloud_sync_status( - subcloud_name, - subcloud_region, dccommon_consts.ENDPOINT_TYPE_SOFTWARE, - dccommon_consts.SYNC_STATUS_OUT_OF_SYNC) - else: - self._update_subcloud_sync_status( - subcloud_name, - subcloud_region, dccommon_consts.ENDPOINT_TYPE_SOFTWARE, - dccommon_consts.SYNC_STATUS_IN_SYNC) - LOG.info('Software audit completed for: %s.' % subcloud_name) - def subcloud_patch_audit(self, subcloud_name, subcloud_region, audit_data, do_load_audit): LOG.info('Triggered patch audit for: %s.' % subcloud_name) diff --git a/distributedcloud/dcmanager/audit/rpcapi.py b/distributedcloud/dcmanager/audit/rpcapi.py index 53549ef90..91222a37c 100644 --- a/distributedcloud/dcmanager/audit/rpcapi.py +++ b/distributedcloud/dcmanager/audit/rpcapi.py @@ -71,6 +71,9 @@ class ManagerAuditClient(object): def trigger_load_audit(self, ctxt): return self.cast(ctxt, self.make_msg('trigger_load_audit')) + def trigger_software_audit(self, ctxt): + return self.cast(ctxt, self.make_msg('trigger_software_audit')) + def trigger_subcloud_audits(self, ctxt, subcloud_id, exclude_endpoints=None): return self.cast(ctxt, self.make_msg('trigger_subcloud_audits', subcloud_id=subcloud_id, diff --git a/distributedcloud/dcmanager/audit/service.py b/distributedcloud/dcmanager/audit/service.py index b85a804d7..fcb9a405c 100644 --- a/distributedcloud/dcmanager/audit/service.py +++ b/distributedcloud/dcmanager/audit/service.py @@ -14,12 +14,12 @@ # import functools -import six from oslo_config import cfg from oslo_log import log as logging import oslo_messaging from oslo_service import service +import six from dcmanager.audit.subcloud_audit_manager import SubcloudAuditManager from dcmanager.audit.subcloud_audit_worker_manager import SubcloudAuditWorkerManager @@ -67,9 +67,9 @@ class DCManagerAuditService(service.Service): def start(self): utils.set_open_file_limit(cfg.CONF.worker_rlimit_nofile) - target = oslo_messaging.Target(version=self.rpc_api_version, - server=self.host, - topic=self.topic) + target = oslo_messaging.Target( + version=self.rpc_api_version, server=self.host, topic=self.topic + ) self.target = target self._rpc_server = rpc_messaging.get_rpc_server(self.target, self) self._rpc_server.start() @@ -93,10 +93,9 @@ class DCManagerAuditService(service.Service): try: self._rpc_server.stop() self._rpc_server.wait() - LOG.info('Engine service stopped successfully') + LOG.info("Engine service stopped successfully") except Exception as ex: - LOG.error('Failed to stop engine service: %s', - six.text_type(ex)) + LOG.error("Failed to stop engine service: %s", six.text_type(ex)) def stop(self): self._stop_rpc_server() @@ -120,8 +119,7 @@ class DCManagerAuditService(service.Service): """Used to force a kube rootca update audit on the next interval""" LOG.info("Trigger kube rootca update audit.") - return self.subcloud_audit_manager.trigger_kube_rootca_update_audit( - context) + return self.subcloud_audit_manager.trigger_kube_rootca_update_audit(context) @request_context def trigger_kubernetes_audit(self, context): @@ -144,27 +142,39 @@ class DCManagerAuditService(service.Service): LOG.info("Trigger load audit.") return self.subcloud_audit_manager.trigger_load_audit(context) + @request_context + def trigger_software_audit(self, context): + """Used to force a software audit on the next interval""" + + LOG.info("Trigger software audit.") + return self.subcloud_audit_manager.trigger_software_audit(context) + @request_context def trigger_subcloud_audits(self, context, subcloud_id, exclude_endpoints): """Trigger all subcloud audits for one subcloud.""" - LOG.info("Trigger all audits for subcloud %s except endpoints %s" % - (subcloud_id, exclude_endpoints)) + LOG.info( + "Trigger all audits for subcloud %s except endpoints %s" + % (subcloud_id, exclude_endpoints) + ) return self.subcloud_audit_manager.trigger_subcloud_audits( - context, subcloud_id, exclude_endpoints) + context, subcloud_id, exclude_endpoints + ) @request_context def trigger_subcloud_patch_load_audits(self, context, subcloud_id): """Trigger patch and load audits for one subcloud.""" LOG.info("Trigger patch and load audits for subcloud %s", subcloud_id) return self.subcloud_audit_manager.trigger_subcloud_patch_load_audits( - context, subcloud_id) + context, subcloud_id + ) @request_context def trigger_subcloud_endpoints_update(self, context, subcloud_name, endpoints): """Trigger update endpoints of services for a subcloud region.""" LOG.info("Trigger update endpoints for subcloud %s", subcloud_name) return self.subcloud_audit_manager.trigger_subcloud_endpoints_update( - context, subcloud_name, endpoints) + context, subcloud_name, endpoints + ) class DCManagerAuditWorkerService(service.Service): @@ -187,9 +197,9 @@ class DCManagerAuditWorkerService(service.Service): utils.set_open_file_limit(cfg.CONF.worker_rlimit_nofile) self.init_tgm() self.init_audit_managers() - target = oslo_messaging.Target(version=self.rpc_api_version, - server=self.host, - topic=self.topic) + target = oslo_messaging.Target( + version=self.rpc_api_version, server=self.host, topic=self.topic + ) self.target = target self._rpc_server = rpc_messaging.get_rpc_server(self.target, self) self._rpc_server.start() @@ -207,10 +217,11 @@ class DCManagerAuditWorkerService(service.Service): try: self._rpc_server.stop() self._rpc_server.wait() - LOG.info('Audit-worker RPC service stopped successfully') + LOG.info("Audit-worker RPC service stopped successfully") except Exception as ex: - LOG.error('Failed to stop audit-worker RPC service: %s', - six.text_type(ex)) + LOG.error( + "Failed to stop audit-worker RPC service: %s", six.text_type(ex) + ) def stop(self): self._stop_rpc_server() @@ -223,15 +234,17 @@ class DCManagerAuditWorkerService(service.Service): super(DCManagerAuditWorkerService, self).stop() @request_context - def audit_subclouds(self, - context, - subcloud_ids, - patch_audit_data, - firmware_audit_data, - kubernetes_audit_data, - do_openstack_audit, - kube_rootca_update_audit_data, - software_audit_data): + def audit_subclouds( + self, + context, + subcloud_ids, + patch_audit_data, + firmware_audit_data, + kubernetes_audit_data, + do_openstack_audit, + kube_rootca_update_audit_data, + software_audit_data, + ): """Used to trigger audits of the specified subcloud(s)""" self.subcloud_audit_worker_manager.audit_subclouds( context, @@ -241,13 +254,12 @@ class DCManagerAuditWorkerService(service.Service): kubernetes_audit_data, do_openstack_audit, kube_rootca_update_audit_data, - software_audit_data) + software_audit_data, + ) @request_context def update_subcloud_endpoints(self, context, subcloud_name, endpoints): """Update endpoints of services for a subcloud region""" self.subcloud_audit_worker_manager.update_subcloud_endpoints( - context, - subcloud_name, - endpoints + context, subcloud_name, endpoints ) diff --git a/distributedcloud/dcmanager/audit/software_audit.py b/distributedcloud/dcmanager/audit/software_audit.py new file mode 100644 index 000000000..6f9736977 --- /dev/null +++ b/distributedcloud/dcmanager/audit/software_audit.py @@ -0,0 +1,209 @@ +# +# Copyright (c) 2024 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +from keystoneauth1 import exceptions as keystone_exceptions +from oslo_log import log as logging + +from dccommon import consts as dccommon_consts +from dccommon.drivers.openstack import sdk_platform +from dccommon.drivers.openstack import software_v1 + +LOG = logging.getLogger(__name__) + + +class SoftwareAuditData(object): + def __init__(self, releases, deployed_release_ids, committed_release_ids): + self.releases = releases + self.deployed_release_ids = deployed_release_ids + self.committed_release_ids = committed_release_ids + + def to_dict(self): + return { + "releases": self.releases, + "deployed_release_ids": self.deployed_release_ids, + "committed_release_ids": self.committed_release_ids, + } + + @classmethod + def from_dict(cls, values): + if values is None: + return None + return cls(**values) + + +class SoftwareAudit(object): + """Manages tasks related to software audits.""" + + def __init__(self, context, dcmanager_state_rpc_client): + LOG.debug("SoftwareAudit initialization...") + self.context = context + self.state_rpc_client = dcmanager_state_rpc_client + self.audit_count = 0 + + def _update_subcloud_sync_status( + self, sc_name, sc_region, sc_endpoint_type, sc_status + ): + self.state_rpc_client.update_subcloud_endpoint_status( + self.context, + subcloud_name=sc_name, + subcloud_region=sc_region, + endpoint_type=sc_endpoint_type, + sync_status=sc_status, + ) + + @staticmethod + def _get_upgrades(sysinv_client): + upgrades = None + try: + upgrades = sysinv_client.get_upgrades() + except Exception: + LOG.exception( + "Cannot retrieve upgrade info for " + f"subcloud: {sysinv_client.region_name}" + ) + return upgrades + + def get_regionone_audit_data(self): + """Query RegionOne to determine what releases should be deployed + + to the system as well as the current software version + + :return: A new SoftwareAuditData object + """ + try: + m_os_ks_client = sdk_platform.OpenStackDriver( + region_name=dccommon_consts.DEFAULT_REGION_NAME, region_clients=None + ).keystone_client + software_endpoint = m_os_ks_client.endpoint_cache.get_endpoint( + dccommon_consts.ENDPOINT_TYPE_SOFTWARE + ) + software_client = software_v1.SoftwareClient( + dccommon_consts.DEFAULT_REGION_NAME, + m_os_ks_client.session, + endpoint=software_endpoint, + ) + except Exception: + LOG.exception("Failure initializing OS Client, skip software audit.") + return None + # First query RegionOne to determine what releases should be deployed + # to the system. + regionone_releases = software_client.query() + LOG.debug(f"regionone_releases: {regionone_releases}") + # Build lists of releases that should be deployed or committed in all + # subclouds, based on their state in RegionOne. + deployed_release_ids = list() + committed_release_ids = list() + for release_id in regionone_releases.keys(): + if regionone_releases[release_id]["state"] == software_v1.DEPLOYED: + deployed_release_ids.append(release_id) + elif regionone_releases[release_id]["state"] == software_v1.COMMITTED: + committed_release_ids.append(release_id) + LOG.debug(f"RegionOne deployed_release_ids: {deployed_release_ids}") + LOG.debug(f"RegionOne committed_release_ids: {committed_release_ids}") + return SoftwareAuditData( + regionone_releases, deployed_release_ids, committed_release_ids + ) + + def subcloud_software_audit(self, subcloud_name, subcloud_region, audit_data): + LOG.info(f"Triggered software audit for: {subcloud_name}.") + try: + sc_os_client = sdk_platform.OpenStackDriver( + region_name=subcloud_region, region_clients=None + ).keystone_client + session = sc_os_client.session + software_endpoint = sc_os_client.endpoint_cache.get_endpoint( + dccommon_consts.ENDPOINT_TYPE_SOFTWARE + ) + software_client = software_v1.SoftwareClient( + subcloud_region, session, endpoint=software_endpoint + ) + except ( + keystone_exceptions.EndpointNotFound, + keystone_exceptions.ConnectFailure, + keystone_exceptions.ConnectTimeout, + IndexError, + ): + LOG.exception( + f"Endpoint for online subcloud {subcloud_name} not found, skip " + "software audit." + ) + return + + # Retrieve all the releases that are present in this subcloud. + try: + subcloud_releases = software_client.query() + LOG.debug(f"Releases for subcloud {subcloud_name}: {subcloud_releases}") + except Exception: + LOG.warn( + f"Cannot retrieve releases for subcloud: {subcloud_name}, " + "skip software audit." + ) + return + + out_of_sync = False + + # audit_data will be a dict due to passing through RPC so objectify it + audit_data = SoftwareAuditData.from_dict(audit_data) + + # Check that all releases in this subcloud are in the correct + # state, based on the state of the release in RegionOne. For the + # subcloud. + for release_id in subcloud_releases.keys(): + if subcloud_releases[release_id]["state"] == software_v1.DEPLOYED: + if release_id not in audit_data.deployed_release_ids: + if release_id not in audit_data.committed_release_ids: + LOG.debug( + f"Release {release_id} should not be deployed " + f"in {subcloud_name}." + ) + else: + LOG.debug( + f"Release {release_id} should be committed " + f"in {subcloud_name}." + ) + out_of_sync = True + elif subcloud_releases[release_id]["state"] == software_v1.COMMITTED: + if ( + release_id not in audit_data.committed_release_ids + and release_id not in audit_data.deployed_release_ids + ): + LOG.warn( + f"Release {release_id} should not be committed " + f"in {subcloud_name}." + ) + out_of_sync = True + else: + # In steady state, all releases should either be deployed + # or committed in each subcloud. Release in other + # states mean a sync is required. + out_of_sync = True + + # Check that all deployed or committed releases in RegionOne are + # present in the subcloud. + for release_id in audit_data.deployed_release_ids: + if release_id not in subcloud_releases: + LOG.debug(f"Release {release_id} missing from {subcloud_name}.") + out_of_sync = True + for release_id in audit_data.committed_release_ids: + if release_id not in subcloud_releases: + LOG.debug(f"Release {release_id} missing from {subcloud_name}.") + out_of_sync = True + + if out_of_sync: + self._update_subcloud_sync_status( + subcloud_name, + subcloud_region, + dccommon_consts.ENDPOINT_TYPE_SOFTWARE, + dccommon_consts.SYNC_STATUS_OUT_OF_SYNC, + ) + else: + self._update_subcloud_sync_status( + subcloud_name, + subcloud_region, + dccommon_consts.ENDPOINT_TYPE_SOFTWARE, + dccommon_consts.SYNC_STATUS_IN_SYNC, + ) + LOG.info(f"Software audit completed for: {subcloud_name}.") diff --git a/distributedcloud/dcmanager/audit/subcloud_audit_manager.py b/distributedcloud/dcmanager/audit/subcloud_audit_manager.py index c57d337a7..d65a48277 100644 --- a/distributedcloud/dcmanager/audit/subcloud_audit_manager.py +++ b/distributedcloud/dcmanager/audit/subcloud_audit_manager.py @@ -30,6 +30,7 @@ from dcmanager.audit import kube_rootca_update_audit from dcmanager.audit import kubernetes_audit from dcmanager.audit import patch_audit from dcmanager.audit import rpcapi as dcmanager_audit_rpc_client +from dcmanager.audit import software_audit from dcmanager.audit import utils as audit_utils from dcmanager.common import context from dcmanager.common.i18n import _ @@ -41,15 +42,16 @@ LOG = logging.getLogger(__name__) # We will update the state of each subcloud in the dcorch about once per hour. # Calculate how many iterations that will be. -SUBCLOUD_STATE_UPDATE_ITERATIONS = \ +SUBCLOUD_STATE_UPDATE_ITERATIONS = ( dccommon_consts.SECONDS_IN_HOUR // CONF.scheduler.subcloud_audit_interval +) # Patch audit normally happens every CONF.scheduler.patch_audit_interval # seconds, but can be forced to happen on the next audit interval by calling # trigger_patch_audit. # Name of starlingx openstack helm application -HELM_APP_OPENSTACK = 'openstack' +HELM_APP_OPENSTACK = "openstack" # Every 4 audits triggers a kubernetes audit KUBERNETES_AUDIT_RATE = 4 @@ -73,60 +75,61 @@ class SubcloudAuditManager(manager.Manager): # Used to force kubernetes audit on the next interval force_kubernetes_audit = False + # Used to force patch audit on the next interval + force_software_audit = False + def __init__(self, *args, **kwargs): - LOG.debug(_('SubcloudAuditManager initialization...')) + LOG.debug(_("SubcloudAuditManager initialization...")) super(SubcloudAuditManager, self).__init__( - service_name="subcloud_audit_manager") + service_name="subcloud_audit_manager" + ) self.context = context.get_admin_context() self.audit_worker_rpc_client = ( - dcmanager_audit_rpc_client.ManagerAuditWorkerClient()) + dcmanager_audit_rpc_client.ManagerAuditWorkerClient() + ) # Number of audits since last subcloud state update self.audit_count = SUBCLOUD_STATE_UPDATE_ITERATIONS - 2 + self.patch_audit = patch_audit.PatchAudit(self.context, None) # Number of patch audits self.patch_audit_count = 0 - self.patch_audit = patch_audit.PatchAudit( - self.context, None) # trigger a patch audit on startup self.patch_audit_time = 0 - self.firmware_audit = firmware_audit.FirmwareAudit( - self.context, None) - self.kubernetes_audit = kubernetes_audit.KubernetesAudit( - self.context, None) - self.kube_rootca_update_audit = \ + self.firmware_audit = firmware_audit.FirmwareAudit(self.context, None) + self.kubernetes_audit = kubernetes_audit.KubernetesAudit(self.context, None) + self.kube_rootca_update_audit = ( kube_rootca_update_audit.KubeRootcaUpdateAudit(self.context, None) + ) + self.software_audit = software_audit.SoftwareAudit(self.context, None) def _add_missing_endpoints(self): # Update this flag file based on the most recent new endpoint - file_path_list = [ - os.path.join(CONFIG_PATH, '.kube_rootca_update_endpoint_added'), - os.path.join(CONFIG_PATH, '.usm_endpoint_added') - ] - for file_path in file_path_list: - # If file exists on the controller, all the endpoints have been - # added to DB since last time an endpoint was added - if not os.path.isfile(file_path): - # Ensures all endpoints exist for all subclouds - # If the endpoint doesn't exist, an entry will be made - # in endpoint_status table - for subcloud in db_api.subcloud_get_all(self.context): - subcloud_statuses = db_api.subcloud_status_get_all( - self.context, subcloud.id) - # Use set difference to find missing endpoints - endpoint_type_set = set(dccommon_consts.ENDPOINT_TYPES_LIST) - subcloud_set = set() - for subcloud_status in subcloud_statuses: - subcloud_set.add(subcloud_status.endpoint_type) + file_path = os.path.join(CONFIG_PATH, ".kube_rootca_update_endpoint_added") + # If file exists on the controller, all the endpoints have been + # added to DB since last time an endpoint was added + if not os.path.isfile(file_path): + # Ensures all endpoints exist for all subclouds + # If the endpoint doesn't exist, an entry will be made + # in endpoint_status table + for subcloud in db_api.subcloud_get_all(self.context): + subcloud_statuses = db_api.subcloud_status_get_all( + self.context, subcloud.id + ) + # Use set difference to find missing endpoints + endpoint_type_set = set(dccommon_consts.ENDPOINT_TYPES_LIST) + subcloud_set = set() + for subcloud_status in subcloud_statuses: + subcloud_set.add(subcloud_status.endpoint_type) - missing_endpoints = list(endpoint_type_set - subcloud_set) + missing_endpoints = list(endpoint_type_set - subcloud_set) - for endpoint in missing_endpoints: - db_api.subcloud_status_create(self.context, - subcloud.id, - endpoint) - # Add a flag on a replicated filesystem to avoid re-running - # the DB checks for missing subcloud endpoints - open(file_path, 'w').close() + for endpoint in missing_endpoints: + db_api.subcloud_status_create( + self.context, subcloud.id, endpoint + ) + # Add a flag on a replicated filesystem to avoid re-running + # the DB checks for missing subcloud endpoints + open(file_path, "w").close() @classmethod def trigger_firmware_audit(cls, context): @@ -181,21 +184,35 @@ class SubcloudAuditManager(manager.Manager): def reset_force_patch_audit(cls): cls.force_patch_audit = False + @classmethod + def trigger_software_audit(cls, context): + """Trigger software audit at next interval. + + This can be called from outside the dcmanager audit + """ + cls.force_software_audit = True + + @classmethod + def reset_software_audit(cls): + cls.force_software_audit = False + def trigger_subcloud_audits(self, context, subcloud_id, exclude_endpoints): """Trigger all subcloud audits for one subcloud.""" values = { - 'patch_audit_requested': True, - 'firmware_audit_requested': True, - 'load_audit_requested': True, - 'kubernetes_audit_requested': True, - 'kube_rootca_update_audit_requested': True, + "patch_audit_requested": True, + "firmware_audit_requested": True, + "load_audit_requested": True, + "kubernetes_audit_requested": True, + "kube_rootca_update_audit_requested": True, + "spare_audit_requested": True, } # For the endpoints excluded in the audit, set it to False in db # to disable the audit explicitly. if exclude_endpoints: for exclude_endpoint in exclude_endpoints: exclude_request = dccommon_consts.ENDPOINT_AUDIT_REQUESTS.get( - exclude_endpoint) + exclude_endpoint + ) if exclude_request: values.update({exclude_request: False}) db_api.subcloud_audits_update(context, subcloud_id, values) @@ -211,7 +228,8 @@ class SubcloudAuditManager(manager.Manager): def trigger_subcloud_endpoints_update(self, context, subcloud_name, endpoints): """Trigger update endpoints of services for a subcloud region.""" self.audit_worker_rpc_client.update_subcloud_endpoints( - context, subcloud_name, endpoints) + context, subcloud_name, endpoints + ) def periodic_subcloud_audit(self): """Audit availability of subclouds.""" @@ -223,13 +241,13 @@ class SubcloudAuditManager(manager.Manager): # audit them and request all sub-audits. # (This is for swact and process restart.) db_api.subcloud_audits_fix_expired_audits( - self.context, datetime.datetime.utcnow(), trigger_audits=True) + self.context, datetime.datetime.utcnow(), trigger_audits=True + ) # Blanket catch all exceptions in the audit so that the audit # does not die. while True: try: - eventlet.greenthread.sleep( - CONF.scheduler.subcloud_audit_interval) + eventlet.greenthread.sleep(CONF.scheduler.subcloud_audit_interval) self._periodic_subcloud_audit_loop() except eventlet.greenlet.GreenletExit: # We have been told to exit @@ -244,20 +262,30 @@ class SubcloudAuditManager(manager.Manager): audit_firmware = False audit_kubernetes = False audit_kube_rootca_updates = False + audit_software = False current_time = time.time() + # Determine whether to trigger a patch audit of each subcloud - if (SubcloudAuditManager.force_patch_audit or - (current_time - self.patch_audit_time >= - CONF.scheduler.patch_audit_interval)): + if SubcloudAuditManager.force_patch_audit or ( + current_time - self.patch_audit_time + >= CONF.scheduler.patch_audit_interval + ): LOG.info("Trigger patch audit") audit_patch = True self.patch_audit_time = current_time self.patch_audit_count += 1 # Check subcloud software version every other patch audit cycle - if (self.patch_audit_count % 2 != 0 or - SubcloudAuditManager.force_patch_audit): + if ( + self.patch_audit_count % 2 != 0 + or SubcloudAuditManager.force_patch_audit + ): LOG.info("Trigger load audit") audit_load = True + if self.patch_audit_count % 2 != 0: + LOG.info("Trigger software audit") + audit_software = True + # Reset force_software_audit only when software audit has been + SubcloudAuditManager.reset_software_audit() if self.patch_audit_count % 4 == 1: LOG.info("Trigger firmware audit") audit_firmware = True @@ -296,31 +324,47 @@ class SubcloudAuditManager(manager.Manager): audit_kube_rootca_updates = True SubcloudAuditManager.reset_force_kube_rootca_update_audit() - return (audit_patch, audit_load, audit_firmware, - audit_kubernetes, audit_kube_rootca_updates) + # Trigger a software audit as it is changed through proxy + if SubcloudAuditManager.force_software_audit: + LOG.info("Trigger software audit") + audit_software = True + SubcloudAuditManager.reset_software_audit() - def _get_audit_data(self, - audit_patch, - audit_firmware, - audit_kubernetes, - audit_kube_rootca_updates): + return ( + audit_patch, + audit_load, + audit_firmware, + audit_kubernetes, + audit_kube_rootca_updates, + audit_software, + ) + + def _get_audit_data( + self, + audit_patch, + audit_firmware, + audit_kubernetes, + audit_kube_rootca_updates, + audit_software, + ): """Return the patch / firmware / kubernetes audit data as needed.""" patch_audit_data = None software_audit_data = None firmware_audit_data = None kubernetes_audit_data = None kube_rootca_update_audit_data = None + software_audit_data = None # TODO(nicodemos): After the integration with VIM the patch audit and patch # orchestration will be removed from the dcmanager. The audit_patch will # be substituted by the software_audit. The software_audit will be # responsible for the patch and load audit. if audit_patch: - # Query RegionOne releases - software_audit_data = ( - self.patch_audit.get_software_regionone_audit_data()) # Query RegionOne patches and software version patch_audit_data = self.patch_audit.get_regionone_audit_data() + if audit_software: + # Query RegionOne releases + software_audit_data = self.patch_audit.get_regionone_audit_data() if audit_firmware: # Query RegionOne firmware firmware_audit_data = self.firmware_audit.get_regionone_audit_data() @@ -329,17 +373,22 @@ class SubcloudAuditManager(manager.Manager): kubernetes_audit_data = self.kubernetes_audit.get_regionone_audit_data() if audit_kube_rootca_updates: # Query RegionOne kube rootca update info - kube_rootca_update_audit_data = \ + kube_rootca_update_audit_data = ( self.kube_rootca_update_audit.get_regionone_audit_data() - return (patch_audit_data, firmware_audit_data, - kubernetes_audit_data, kube_rootca_update_audit_data, - software_audit_data) + ) + return ( + patch_audit_data, + firmware_audit_data, + kubernetes_audit_data, + kube_rootca_update_audit_data, + software_audit_data, + ) def _periodic_subcloud_audit_loop(self): """Audit availability of subclouds loop.""" # We will be running in our own green thread here. - LOG.debug('Triggered subcloud audit.') + LOG.debug("Triggered subcloud audit.") self.audit_count += 1 # Determine whether to trigger a state update to each subcloud. @@ -350,14 +399,26 @@ class SubcloudAuditManager(manager.Manager): update_subcloud_state = False # Determine whether we want to trigger specialty audits. - (audit_patch, audit_load, audit_firmware, - audit_kubernetes, - audit_kube_rootca_update) = self._get_audits_needed() + ( + audit_patch, + audit_load, + audit_firmware, + audit_kubernetes, + audit_kube_rootca_update, + audit_software, + ) = self._get_audits_needed() # Set desired audit flags for all subclouds. audit_utils.request_subcloud_audits( - self.context, update_subcloud_state, audit_patch, audit_load, - audit_firmware, audit_kubernetes, audit_kube_rootca_update) + self.context, + update_subcloud_state, + audit_patch, + audit_load, + audit_firmware, + audit_kubernetes, + audit_kube_rootca_update, + audit_software, + ) do_openstack_audit = False @@ -379,28 +440,35 @@ class SubcloudAuditManager(manager.Manager): current_time = datetime.datetime.utcnow() last_audit_threshold = current_time - datetime.timedelta( - seconds=CONF.scheduler.subcloud_audit_interval) + seconds=CONF.scheduler.subcloud_audit_interval + ) # The sysinv and patching subcloud REST API timeouts are 600 sec, # and we need to be greater than that, so lets go with that plus # an extra audit interval. last_audit_fixup_threshold = current_time - datetime.timedelta( - seconds=(sysinv_v1.SYSINV_CLIENT_REST_DEFAULT_TIMEOUT + - CONF.scheduler.subcloud_audit_interval)) + seconds=( + sysinv_v1.SYSINV_CLIENT_REST_DEFAULT_TIMEOUT + + CONF.scheduler.subcloud_audit_interval + ) + ) # Fix up any stale audit timestamps for subclouds that started an # audit but never finished it. start = datetime.datetime.utcnow() num_fixed = db_api.subcloud_audits_fix_expired_audits( - self.context, last_audit_fixup_threshold) + self.context, last_audit_fixup_threshold + ) end = datetime.datetime.utcnow() if num_fixed > 0: LOG.info( - 'Fixed up subcloud audit timestamp for %s subclouds.' % num_fixed) - LOG.info('Fixup took %s seconds' % (end - start)) + "Fixed up subcloud audit timestamp for %s subclouds." % num_fixed + ) + LOG.info("Fixup took %s seconds" % (end - start)) subcloud_ids = [] subcloud_audits = db_api.subcloud_audits_get_all_need_audit( - self.context, last_audit_threshold) + self.context, last_audit_threshold + ) # Now check whether any of these subclouds need patch audit or firmware # audit data and grab it if needed. @@ -430,29 +498,55 @@ class SubcloudAuditManager(manager.Manager): LOG.debug("DB says kube-rootca-update audit needed") audit_kube_rootca_update = True break - LOG.info("Triggered subcloud audit: patch=(%s) firmware=(%s) " - "kube=(%s) kube-rootca=(%s)" - % (audit_patch, audit_firmware, - audit_kubernetes, audit_kube_rootca_update)) - (patch_audit_data, firmware_audit_data, - kubernetes_audit_data, kube_rootca_update_audit_data, - software_audit_data) = \ - self._get_audit_data(audit_patch, - audit_firmware, - audit_kubernetes, - audit_kube_rootca_update) - LOG.debug("patch_audit_data: %s, " - "firmware_audit_data: %s, " - "kubernetes_audit_data: %s, " - "kube_rootca_update_audit_data: : %s, " - % (patch_audit_data, - firmware_audit_data, - kubernetes_audit_data, - kube_rootca_update_audit_data)) + if not audit_software: + for audit in subcloud_audits: + if audit.spare_audit_requested: + LOG.debug("DB says software audit needed") + audit_software = True + break + LOG.info( + "Triggered subcloud audit: patch=(%s) firmware=(%s) " + "kube=(%s) kube-rootca=(%s) software=(%s)" + % ( + audit_patch, + audit_firmware, + audit_kubernetes, + audit_kube_rootca_update, + audit_software, + ) + ) + ( + patch_audit_data, + firmware_audit_data, + kubernetes_audit_data, + kube_rootca_update_audit_data, + software_audit_data, + ) = self._get_audit_data( + audit_patch, + audit_firmware, + audit_kubernetes, + audit_kube_rootca_update, + audit_software, + ) + LOG.debug( + "patch_audit_data: %s, " + "firmware_audit_data: %s, " + "kubernetes_audit_data: %s, " + "kube_rootca_update_audit_data: : %s, " + "software_audit_data: %s" + % ( + patch_audit_data, + firmware_audit_data, + kubernetes_audit_data, + kube_rootca_update_audit_data, + software_audit_data, + ) + ) # We want a chunksize of at least 1 so add the number of workers. chunksize = (len(subcloud_audits) + CONF.audit_worker_workers) // ( - CONF.audit_worker_workers) + CONF.audit_worker_workers + ) for audit in subcloud_audits: subcloud_ids.append(audit.subcloud_id) if len(subcloud_ids) == chunksize: @@ -465,10 +559,11 @@ class SubcloudAuditManager(manager.Manager): kubernetes_audit_data, do_openstack_audit, kube_rootca_update_audit_data, - software_audit_data) + software_audit_data, + ) LOG.debug( - 'Sent subcloud audit request message for subclouds: %s' % - subcloud_ids + "Sent subcloud audit request message for subclouds: %s" + % subcloud_ids ) subcloud_ids = [] if len(subcloud_ids) > 0: @@ -481,10 +576,11 @@ class SubcloudAuditManager(manager.Manager): kubernetes_audit_data, do_openstack_audit, kube_rootca_update_audit_data, - software_audit_data) + software_audit_data, + ) LOG.debug( - 'Sent final subcloud audit request message for subclouds: %s' % - subcloud_ids + "Sent final subcloud audit request message for subclouds: %s" + % subcloud_ids ) else: - LOG.debug('Done sending audit request messages.') + LOG.debug("Done sending audit request messages.") diff --git a/distributedcloud/dcmanager/audit/subcloud_audit_worker_manager.py b/distributedcloud/dcmanager/audit/subcloud_audit_worker_manager.py index 58374333f..640aac193 100644 --- a/distributedcloud/dcmanager/audit/subcloud_audit_worker_manager.py +++ b/distributedcloud/dcmanager/audit/subcloud_audit_worker_manager.py @@ -27,6 +27,7 @@ from dcmanager.audit import firmware_audit from dcmanager.audit import kube_rootca_update_audit from dcmanager.audit import kubernetes_audit from dcmanager.audit import patch_audit +from dcmanager.audit import software_audit from dcmanager.audit.subcloud_audit_manager import HELM_APP_OPENSTACK from dcmanager.common import consts from dcmanager.common import context @@ -70,10 +71,13 @@ class SubcloudAuditWorkerManager(manager.Manager): self.context, self.state_rpc_client) self.kubernetes_audit = kubernetes_audit.KubernetesAudit( self.context, self.state_rpc_client) - self.kube_rootca_update_audit = \ + self.kube_rootca_update_audit = ( kube_rootca_update_audit.KubeRootcaUpdateAudit( - self.context, - self.state_rpc_client) + self.context, self.state_rpc_client + ) + ) + self.software_audit = software_audit.SoftwareAudit( + self.context, self.state_rpc_client) self.pid = os.getpid() def audit_subclouds(self, @@ -149,6 +153,7 @@ class SubcloudAuditWorkerManager(manager.Manager): do_kube_rootca_update_audit = \ subcloud_audits.kube_rootca_update_audit_requested update_subcloud_state = subcloud_audits.state_update_requested + do_software_audit = subcloud_audits.spare_audit_requested # Create a new greenthread for each subcloud to allow the audits # to be done in parallel. If there are not enough greenthreads @@ -167,7 +172,8 @@ class SubcloudAuditWorkerManager(manager.Manager): do_load_audit, do_firmware_audit, do_kubernetes_audit, - do_kube_rootca_update_audit) + do_kube_rootca_update_audit, + do_software_audit) def update_subcloud_endpoints(self, context, subcloud_name, endpoints): try: @@ -310,7 +316,8 @@ class SubcloudAuditWorkerManager(manager.Manager): do_load_audit, do_firmware_audit, do_kubernetes_audit, - do_kube_rootca_update_audit): + do_kube_rootca_update_audit, + do_software_audit): audits_done = list() failures = list() # Do the actual subcloud audit. @@ -328,7 +335,8 @@ class SubcloudAuditWorkerManager(manager.Manager): do_load_audit, do_firmware_audit, do_kubernetes_audit, - do_kube_rootca_update_audit) + do_kube_rootca_update_audit, + do_software_audit) except Exception: LOG.exception("Got exception auditing subcloud: %s" % subcloud.name) @@ -360,7 +368,8 @@ class SubcloudAuditWorkerManager(manager.Manager): do_load_audit, do_firmware_audit, do_kubernetes_audit, - do_kube_rootca_update_audit): + do_kube_rootca_update_audit, + do_software_audit): """Audit a single subcloud.""" avail_status_current = subcloud.availability_status @@ -492,13 +501,12 @@ class SubcloudAuditWorkerManager(manager.Manager): failmsg = "Audit failure subcloud: %s, endpoint: %s" # If we have patch audit data, audit the subcloud - if do_patch_audit and (patch_audit_data or software_audit_data): + if do_patch_audit and patch_audit_data: try: - self.patch_audit.subcloud_audit(subcloud_name, - subcloud_region, - patch_audit_data, - software_audit_data, - do_load_audit) + self.patch_audit.subcloud_patch_audit(subcloud_name, + subcloud_region, + patch_audit_data, + do_load_audit) audits_done.append('patch') if do_load_audit: audits_done.append('load') @@ -550,4 +558,13 @@ class SubcloudAuditWorkerManager(manager.Manager): except Exception: LOG.exception(failmsg % (subcloud.name, 'openstack')) failures.append('openstack') + # Perform software audit + if do_software_audit: + try: + self.software_audit.subcloud_software_audit( + subcloud_name, subcloud_region, software_audit_data) + audits_done.append('software') + except Exception: + LOG.exception(failmsg % (subcloud.name, 'software')) + failures.append('software') return audits_done, failures diff --git a/distributedcloud/dcmanager/audit/utils.py b/distributedcloud/dcmanager/audit/utils.py index 427ba7110..1b453d4ed 100644 --- a/distributedcloud/dcmanager/audit/utils.py +++ b/distributedcloud/dcmanager/audit/utils.py @@ -26,7 +26,8 @@ def request_subcloud_audits(context, audit_load=False, audit_firmware=False, audit_kubernetes=False, - audit_kube_rootca=False): + audit_kube_rootca=False, + audit_software=False,): values = {} if update_subcloud_state: values['state_update_requested'] = True @@ -40,4 +41,6 @@ def request_subcloud_audits(context, values['kubernetes_audit_requested'] = True if audit_kube_rootca: values['kube_rootca_update_audit_requested'] = True + if audit_software: + values['spare_audit_requested'] = True db_api.subcloud_audits_update_all(context, values) diff --git a/distributedcloud/dcmanager/db/sqlalchemy/api.py b/distributedcloud/dcmanager/db/sqlalchemy/api.py index 8c947d107..81a6cbb52 100644 --- a/distributedcloud/dcmanager/db/sqlalchemy/api.py +++ b/distributedcloud/dcmanager/db/sqlalchemy/api.py @@ -32,8 +32,7 @@ from oslo_utils import uuidutils import sqlalchemy from sqlalchemy import desc from sqlalchemy import or_ -from sqlalchemy.orm.exc import MultipleResultsFound -from sqlalchemy.orm.exc import NoResultFound +from sqlalchemy.orm import exc from sqlalchemy.orm import joinedload_all from sqlalchemy.orm import load_only from sqlalchemy.sql.expression import true @@ -204,7 +203,8 @@ def subcloud_audits_get_all_need_audit(context, last_audit_threshold): (models.SubcloudAudits.load_audit_requested == true()) | (models.SubcloudAudits.kube_rootca_update_audit_requested == true()) | - (models.SubcloudAudits.kubernetes_audit_requested == true())).\ + (models.SubcloudAudits.kubernetes_audit_requested == true()) | + (models.SubcloudAudits.spare_audit_requested == true())).\ all() return result @@ -239,6 +239,8 @@ def subcloud_audits_end_audit(context, subcloud_id, audits_done): subcloud_audits_ref.kube_rootca_update_audit_requested = False if 'kubernetes' in audits_done: subcloud_audits_ref.kubernetes_audit_requested = False + if 'software' in audits_done: + subcloud_audits_ref.spare_audit_requested = False subcloud_audits_ref.save(session) return subcloud_audits_ref @@ -260,6 +262,7 @@ def subcloud_audits_fix_expired_audits(context, last_audit_threshold, values['load_audit_requested'] = True values['kubernetes_audit_requested'] = True values['kube_rootca_update_audit_requested'] = True + values['spare_audit_requested'] = True with write_session() as session: result = session.query(models.SubcloudAudits).\ options(load_only("deleted", "audit_started_at", @@ -817,9 +820,9 @@ def system_peer_get(context, peer_id): filter_by(deleted=0). \ filter_by(id=peer_id). \ one() - except NoResultFound: + except exc.NoResultFound: raise exception.SystemPeerNotFound(peer_id=peer_id) - except MultipleResultsFound: + except exc.MultipleResultsFound: raise exception.InvalidParameterValue( err="Multiple entries found for system peer %s" % peer_id) @@ -833,9 +836,9 @@ def system_peer_get_by_name(context, name): filter_by(deleted=0). \ filter_by(peer_name=name). \ one() - except NoResultFound: + except exc.NoResultFound: raise exception.SystemPeerNameNotFound(name=name) - except MultipleResultsFound: + except exc.MultipleResultsFound: # This exception should never happen due to the UNIQUE setting for name raise exception.InvalidParameterValue( err="Multiple entries found for system peer %s" % name) @@ -850,9 +853,9 @@ def system_peer_get_by_uuid(context, uuid): filter_by(deleted=0). \ filter_by(peer_uuid=uuid). \ one() - except NoResultFound: + except exc.NoResultFound: raise exception.SystemPeerUUIDNotFound(uuid=uuid) - except MultipleResultsFound: + except exc.MultipleResultsFound: # This exception should never happen due to the UNIQUE setting for uuid raise exception.InvalidParameterValue( err="Multiple entries found for system peer %s" % uuid) @@ -973,9 +976,9 @@ def subcloud_group_get(context, group_id): filter_by(deleted=0). \ filter_by(id=group_id). \ one() - except NoResultFound: + except exc.NoResultFound: raise exception.SubcloudGroupNotFound(group_id=group_id) - except MultipleResultsFound: + except exc.MultipleResultsFound: raise exception.InvalidParameterValue( err="Multiple entries found for subcloud group %s" % group_id) @@ -989,9 +992,9 @@ def subcloud_group_get_by_name(context, name): filter_by(deleted=0). \ filter_by(name=name). \ one() - except NoResultFound: + except exc.NoResultFound: raise exception.SubcloudGroupNameNotFound(name=name) - except MultipleResultsFound: + except exc.MultipleResultsFound: # This exception should never happen due to the UNIQUE setting for name raise exception.InvalidParameterValue( err="Multiple entries found for subcloud group %s" % name) @@ -1109,9 +1112,9 @@ def subcloud_peer_group_get(context, group_id): filter_by(deleted=0). \ filter_by(id=group_id). \ one() - except NoResultFound: + except exc.NoResultFound: raise exception.SubcloudPeerGroupNotFound(group_id=group_id) - except MultipleResultsFound: + except exc.MultipleResultsFound: raise exception.InvalidParameterValue( err="Multiple entries found for subcloud peer group %s" % group_id) @@ -1149,9 +1152,9 @@ def subcloud_peer_group_get_by_name(context, name): filter_by(deleted=0). \ filter_by(peer_group_name=name). \ one() - except NoResultFound: + except exc.NoResultFound: raise exception.SubcloudPeerGroupNameNotFound(name=name) - except MultipleResultsFound: + except exc.MultipleResultsFound: # This exception should never happen due to the UNIQUE setting for name raise exception.InvalidParameterValue( err="Multiple entries found for subcloud peer group %s" % name) @@ -1288,10 +1291,10 @@ def peer_group_association_get(context, association_id): filter_by(deleted=0). \ filter_by(id=association_id). \ one() - except NoResultFound: + except exc.NoResultFound: raise exception.PeerGroupAssociationNotFound( association_id=association_id) - except MultipleResultsFound: + except exc.MultipleResultsFound: raise exception.InvalidParameterValue( err="Multiple entries found for peer group association %s" % association_id) @@ -1321,10 +1324,10 @@ def peer_group_association_get_by_peer_group_and_system_peer_id(context, filter_by(peer_group_id=peer_group_id). \ filter_by(system_peer_id=system_peer_id). \ one() - except NoResultFound: + except exc.NoResultFound: raise exception.PeerGroupAssociationCombinationNotFound( peer_group_id=peer_group_id, system_peer_id=system_peer_id) - except MultipleResultsFound: + except exc.MultipleResultsFound: # This exception should never happen due to the UNIQUE setting for name raise exception.InvalidParameterValue( err="Multiple entries found for peer group association %s,%s" % @@ -1485,9 +1488,9 @@ def _subcloud_alarms_get(context, name): try: return query.one() - except NoResultFound: + except exc.NoResultFound: raise exception.SubcloudNameNotFound(name=name) - except MultipleResultsFound: + except exc.MultipleResultsFound: raise exception.InvalidParameterValue( err="Multiple entries found for subcloud %s" % name) diff --git a/distributedcloud/dcmanager/db/sqlalchemy/models.py b/distributedcloud/dcmanager/db/sqlalchemy/models.py index f23e337d4..140597e21 100644 --- a/distributedcloud/dcmanager/db/sqlalchemy/models.py +++ b/distributedcloud/dcmanager/db/sqlalchemy/models.py @@ -203,20 +203,23 @@ class SubcloudAudits(BASE, DCManagerBase): __tablename__ = 'subcloud_audits' id = Column(Integer, primary_key=True, nullable=False) - subcloud_id = Column(Integer, - ForeignKey('subclouds.id', ondelete='CASCADE'), - unique=True) - audit_started_at = Column(DateTime(timezone=False), - default=datetime.datetime.min) - audit_finished_at = Column(DateTime(timezone=False), - default=datetime.datetime.min) + subcloud_id = Column( + Integer, ForeignKey('subclouds.id', ondelete='CASCADE'), unique=True + ) + audit_started_at = Column( + DateTime(timezone=False), default=datetime.datetime.min + ) + audit_finished_at = Column( + DateTime(timezone=False), default=datetime.datetime.min + ) state_update_requested = Column(Boolean, nullable=False, default=False) patch_audit_requested = Column(Boolean, nullable=False, default=False) load_audit_requested = Column(Boolean, nullable=False, default=False) firmware_audit_requested = Column(Boolean, nullable=False, default=False) kubernetes_audit_requested = Column(Boolean, nullable=False, default=False) - kube_rootca_update_audit_requested = Column(Boolean, nullable=False, - default=False) + kube_rootca_update_audit_requested = Column( + Boolean, nullable=False, default=False + ) spare_audit_requested = Column(Boolean, nullable=False, default=False) spare2_audit_requested = Column(Boolean, nullable=False, default=False) reserved = Column(Text) diff --git a/distributedcloud/dcmanager/orchestrator/patch_orch_thread.py b/distributedcloud/dcmanager/orchestrator/patch_orch_thread.py index 84cc21aa7..bf7f61bf9 100644 --- a/distributedcloud/dcmanager/orchestrator/patch_orch_thread.py +++ b/distributedcloud/dcmanager/orchestrator/patch_orch_thread.py @@ -28,8 +28,7 @@ from dcmanager.orchestrator.states.patch.finishing_patch_strategy import \ FinishingPatchStrategyState from dcmanager.orchestrator.states.patch.job_data import PatchJobData from dcmanager.orchestrator.states.patch.pre_check import PreCheckState -from dcmanager.orchestrator.states.patch.updating_patches import \ - UpdatingPatchesState +from dcmanager.orchestrator.states.patch.updating_patches import UpdatingPatchesState LOG = logging.getLogger(__name__) diff --git a/distributedcloud/dcmanager/orchestrator/software_orch_thread.py b/distributedcloud/dcmanager/orchestrator/software_orch_thread.py index f22dc97ab..b98eb1b57 100644 --- a/distributedcloud/dcmanager/orchestrator/software_orch_thread.py +++ b/distributedcloud/dcmanager/orchestrator/software_orch_thread.py @@ -7,16 +7,16 @@ from dccommon.drivers.openstack import vim from dcmanager.common import consts from dcmanager.orchestrator.orch_thread import OrchThread -from dcmanager.orchestrator.states.software.apply_vim_software_strategy \ - import ApplyVIMSoftwareStrategyState +from dcmanager.orchestrator.states.software.apply_vim_software_strategy import \ + ApplyVIMSoftwareStrategyState from dcmanager.orchestrator.states.software.cache.shared_cache_repository import \ SharedCacheRepository -from dcmanager.orchestrator.states.software.create_vim_software_strategy \ - import CreateVIMSoftwareStrategyState -from dcmanager.orchestrator.states.software.finish_strategy \ - import FinishStrategyState -from dcmanager.orchestrator.states.software.install_license \ - import InstallLicenseState +from dcmanager.orchestrator.states.software.create_vim_software_strategy import \ + CreateVIMSoftwareStrategyState +from dcmanager.orchestrator.states.software.finish_strategy import \ + FinishStrategyState +from dcmanager.orchestrator.states.software.install_license import \ + InstallLicenseState from dcmanager.orchestrator.states.software.pre_check import PreCheckState @@ -59,8 +59,8 @@ class SoftwareOrchThread(OrchThread): self._shared_caches.initialize_caches() def trigger_audit(self): - """Trigger an audit for upgrade (which is combined with patch audit)""" - self.audit_rpc_client.trigger_patch_audit(self.context) + """Trigger an audit for software""" + self.audit_rpc_client.trigger_software_audit(self.context) def pre_apply_setup(self): # Restart caches for next strategy diff --git a/distributedcloud/dcmanager/state/service.py b/distributedcloud/dcmanager/state/service.py index 9ec6a0617..816261233 100644 --- a/distributedcloud/dcmanager/state/service.py +++ b/distributedcloud/dcmanager/state/service.py @@ -18,15 +18,14 @@ # import functools -import six from oslo_config import cfg from oslo_log import log as logging import oslo_messaging from oslo_service import service +import six from dccommon import consts as dccommon_consts - from dcmanager.audit import rpcapi as dcmanager_audit_rpc_client from dcmanager.common import consts from dcmanager.common import context @@ -82,9 +81,9 @@ class DCManagerStateService(service.Service): LOG.info("Starting %s", self.__class__.__name__) utils.set_open_file_limit(cfg.CONF.worker_rlimit_nofile) self._init_managers() - target = oslo_messaging.Target(version=self.rpc_api_version, - server=self.host, - topic=self.topic) + target = oslo_messaging.Target( + version=self.rpc_api_version, server=self.host, topic=self.topic + ) self.target = target self._rpc_server = rpc_messaging.get_rpc_server(self.target, self) self._rpc_server.start() @@ -99,10 +98,9 @@ class DCManagerStateService(service.Service): try: self._rpc_server.stop() self._rpc_server.wait() - LOG.info('Engine service stopped successfully') + LOG.info("Engine service stopped successfully") except Exception as ex: - LOG.error('Failed to stop engine service: %s', - six.text_type(ex)) + LOG.error("Failed to stop engine service: %s", six.text_type(ex)) def stop(self): LOG.info("Stopping %s", self.__class__.__name__) @@ -113,57 +111,83 @@ class DCManagerStateService(service.Service): @request_context def update_subcloud_endpoint_status( - self, context, subcloud_name=None, subcloud_region=None, endpoint_type=None, - sync_status=dccommon_consts.SYNC_STATUS_OUT_OF_SYNC, alarmable=True, - ignore_endpoints=None + self, + context, + subcloud_name=None, + subcloud_region=None, + endpoint_type=None, + sync_status=dccommon_consts.SYNC_STATUS_OUT_OF_SYNC, + alarmable=True, + ignore_endpoints=None, ): # Updates subcloud endpoint sync status - LOG.info("Handling update_subcloud_endpoint_status request for " - "subcloud: (%s) endpoint: (%s) status:(%s) " - % (subcloud_name, endpoint_type, sync_status)) + LOG.info( + "Handling update_subcloud_endpoint_status request for " + "subcloud: (%s) endpoint: (%s) status:(%s) " + % (subcloud_name, endpoint_type, sync_status) + ) - self.subcloud_state_manager. \ - update_subcloud_endpoint_status(context, - subcloud_region, - endpoint_type, - sync_status, - alarmable, - ignore_endpoints) + self.subcloud_state_manager.update_subcloud_endpoint_status( + context, + subcloud_region, + endpoint_type, + sync_status, + alarmable, + ignore_endpoints, + ) # If the patching sync status is being set to unknown, trigger the # patching audit so it can update the sync status ASAP. - if (endpoint_type == dccommon_consts.ENDPOINT_TYPE_PATCHING - or endpoint_type == dccommon_consts.ENDPOINT_TYPE_SOFTWARE) and \ - sync_status == dccommon_consts.SYNC_STATUS_UNKNOWN: + if ( + endpoint_type == dccommon_consts.ENDPOINT_TYPE_PATCHING + and sync_status == dccommon_consts.SYNC_STATUS_UNKNOWN + ): self.audit_rpc_client.trigger_patch_audit(context) + # If the software sync status is being set to unknown, trigger the + # software audit so it can update the sync status ASAP. + if ( + endpoint_type == dccommon_consts.ENDPOINT_TYPE_SOFTWARE + and sync_status == dccommon_consts.SYNC_STATUS_UNKNOWN + ): + self.audit_rpc_client.trigger_software_audit(context) + # If the firmware sync status is being set to unknown, trigger the # firmware audit so it can update the sync status ASAP. - if endpoint_type == dccommon_consts.ENDPOINT_TYPE_FIRMWARE and \ - sync_status == dccommon_consts.SYNC_STATUS_UNKNOWN: + if ( + endpoint_type == dccommon_consts.ENDPOINT_TYPE_FIRMWARE + and sync_status == dccommon_consts.SYNC_STATUS_UNKNOWN + ): self.audit_rpc_client.trigger_firmware_audit(context) # If the kubernetes sync status is being set to unknown, trigger the # kubernetes audit so it can update the sync status ASAP. - if endpoint_type == dccommon_consts.ENDPOINT_TYPE_KUBERNETES and \ - sync_status == dccommon_consts.SYNC_STATUS_UNKNOWN: + if ( + endpoint_type == dccommon_consts.ENDPOINT_TYPE_KUBERNETES + and sync_status == dccommon_consts.SYNC_STATUS_UNKNOWN + ): self.audit_rpc_client.trigger_kubernetes_audit(context) return @request_context - def update_subcloud_availability(self, context, - subcloud_name, - subcloud_region, - availability_status, - update_state_only=False, - audit_fail_count=None): + def update_subcloud_availability( + self, + context, + subcloud_name, + subcloud_region, + availability_status, + update_state_only=False, + audit_fail_count=None, + ): # Updates subcloud availability - LOG.info("Handling update_subcloud_availability request for: %s" % - subcloud_name) + LOG.info( + "Handling update_subcloud_availability request for: %s" % subcloud_name + ) self.subcloud_state_manager.update_subcloud_availability( context, subcloud_region, availability_status, update_state_only, - audit_fail_count) + audit_fail_count, + ) diff --git a/distributedcloud/dcmanager/tests/unit/audit/test_firmware_audit_manager.py b/distributedcloud/dcmanager/tests/unit/audit/test_firmware_audit_manager.py index d753ea024..83a85e738 100644 --- a/distributedcloud/dcmanager/tests/unit/audit/test_firmware_audit_manager.py +++ b/distributedcloud/dcmanager/tests/unit/audit/test_firmware_audit_manager.py @@ -413,10 +413,6 @@ class TestFirmwareAudit(base.DCManagerTestCase): self.mock_audit_worker_api.return_value = self.fake_audit_worker_api self.addCleanup(p.stop) - p = mock.patch.object(patch_audit, 'SoftwareClient') - self.mock_patch_audit_sc = p.start() - self.addCleanup(p.stop) - def _rpc_convert(self, object_list): # Convert to dict like what would happen calling via RPC dict_results = [] @@ -425,8 +421,13 @@ class TestFirmwareAudit(base.DCManagerTestCase): return dict_results def get_fw_audit_data(self, am): - patch_audit_data, firmware_audit_data, kubernetes_audit_data, kube_root,\ - software_audit_data = am._get_audit_data(True, True, True, True) + ( + _, + firmware_audit_data, + _, + _, + _ + ) = am._get_audit_data(True, True, True, True, True) # Convert to dict like what would happen calling via RPC firmware_audit_data = self._rpc_convert(firmware_audit_data) diff --git a/distributedcloud/dcmanager/tests/unit/audit/test_kube_audit_manager.py b/distributedcloud/dcmanager/tests/unit/audit/test_kube_audit_manager.py index 84307c8ba..40af6e37c 100644 --- a/distributedcloud/dcmanager/tests/unit/audit/test_kube_audit_manager.py +++ b/distributedcloud/dcmanager/tests/unit/audit/test_kube_audit_manager.py @@ -110,10 +110,6 @@ class TestKubernetesAudit(base.DCManagerTestCase): self.mock_patch_audit_pc.return_value = mock.MagicMock() self.addCleanup(p.stop) - p = mock.patch.object(patch_audit, 'SoftwareClient') - self.mock_patch_audit_sc = p.start() - self.addCleanup(p.stop) - p = mock.patch.object(firmware_audit, 'OpenStackDriver') self.mock_firmware_audit_driver = p.start() self.mock_firmware_audit_driver.return_value = mock.MagicMock() @@ -147,8 +143,13 @@ class TestKubernetesAudit(base.DCManagerTestCase): return dict_results def get_kube_audit_data(self, am): - patch_audit_data, firmware_audit_data, kubernetes_audit_data, kube_rootca, \ - software_audit_data = am._get_audit_data(True, True, True, True) + ( + _, + _, + kubernetes_audit_data, + _, + _ + ) = am._get_audit_data(True, True, True, True, True) # Convert to dict like what would happen calling via RPC kubernetes_audit_data = self._rpc_convert(kubernetes_audit_data) return kubernetes_audit_data diff --git a/distributedcloud/dcmanager/tests/unit/audit/test_kube_rootca_update_audit_manager.py b/distributedcloud/dcmanager/tests/unit/audit/test_kube_rootca_update_audit_manager.py index 6c5cbd73f..fa2072c64 100644 --- a/distributedcloud/dcmanager/tests/unit/audit/test_kube_rootca_update_audit_manager.py +++ b/distributedcloud/dcmanager/tests/unit/audit/test_kube_rootca_update_audit_manager.py @@ -9,7 +9,6 @@ import mock from dccommon import consts as dccommon_consts from dcmanager.audit import kube_rootca_update_audit from dcmanager.audit import subcloud_audit_manager - from dcmanager.tests import base from dcmanager.tests import utils @@ -106,9 +105,13 @@ class TestKubeRootcaUpdateAudit(base.DCManagerTestCase): self.rootca_fm_client.get_alarms_by_ids.return_value = None def get_rootca_audit_data(self, am): - patch_audit_data, firmware_audit_data, kubernetes_audit_data, \ - kube_rootca_audit_data, software_audit_data = am._get_audit_data( - True, True, True, True) + ( + _, + _, + _, + kube_rootca_audit_data, + _ + ) = am._get_audit_data(True, True, True, True, True) return kube_rootca_audit_data diff --git a/distributedcloud/dcmanager/tests/unit/audit/test_patch_audit_manager.py b/distributedcloud/dcmanager/tests/unit/audit/test_patch_audit_manager.py index 45475230e..8731e6d49 100644 --- a/distributedcloud/dcmanager/tests/unit/audit/test_patch_audit_manager.py +++ b/distributedcloud/dcmanager/tests/unit/audit/test_patch_audit_manager.py @@ -263,16 +263,14 @@ class TestPatchAudit(base.DCManagerTestCase): self.mock_audit_worker_api.return_value = self.fake_audit_worker_api self.addCleanup(p.stop) - # Mock the Software Client - p = mock.patch.object(patch_audit, 'SoftwareClient') - self.mock_patch_audit_sc = p.start() - self.addCleanup(p.stop) - def get_patch_audit_data(self, am): - (patch_audit_data, firmware_audit_data, - kubernetes_audit_data, kube_rootca_data, - software_audit_data) = \ - am._get_audit_data(True, True, True, True) + ( + patch_audit_data, + _, + _, + _, + _ + ) = am._get_audit_data(True, True, True, True, True) # Convert to dict like what would happen calling via RPC patch_audit_data = patch_audit_data.to_dict() return patch_audit_data diff --git a/distributedcloud/dcmanager/tests/unit/audit/test_subcloud_audit_worker_manager.py b/distributedcloud/dcmanager/tests/unit/audit/test_subcloud_audit_worker_manager.py index a286fa093..cc658d0de 100644 --- a/distributedcloud/dcmanager/tests/unit/audit/test_subcloud_audit_worker_manager.py +++ b/distributedcloud/dcmanager/tests/unit/audit/test_subcloud_audit_worker_manager.py @@ -18,9 +18,8 @@ import copy import random import sys -import mock - from keystoneauth1 import exceptions as keystone_exceptions +import mock from dccommon import consts as dccommon_consts from dcmanager.audit import subcloud_audit_manager @@ -412,13 +411,20 @@ class TestAuditWorkerManager(base.DCManagerTestCase): do_firmware_audit = True do_kubernetes_audit = True do_kube_rootca_update_audit = True - (patch_audit_data, firmware_audit_data, - kubernetes_audit_data, kube_rootca_update_audit_data, - software_audit_data) = \ - am._get_audit_data(do_patch_audit, - do_firmware_audit, - do_kubernetes_audit, - do_kube_rootca_update_audit) + do_software_audit = True + ( + patch_audit_data, + firmware_audit_data, + kubernetes_audit_data, + kube_rootca_update_audit_data, + software_audit_data, + ) = am._get_audit_data( + do_patch_audit, + do_firmware_audit, + do_kubernetes_audit, + do_kube_rootca_update_audit, + do_software_audit, + ) # Convert to dict like what would happen calling via RPC # Note: the other data should also be converted... patch_audit_data = patch_audit_data.to_dict() @@ -434,7 +440,8 @@ class TestAuditWorkerManager(base.DCManagerTestCase): do_load_audit, do_firmware_audit, do_kubernetes_audit, - do_kube_rootca_update_audit) + do_kube_rootca_update_audit, + do_software_audit) # Verify the subcloud was set to online self.fake_dcmanager_state_api.update_subcloud_availability.\ @@ -457,9 +464,8 @@ class TestAuditWorkerManager(base.DCManagerTestCase): subcloud.name, self.fake_openstack_client.fm_client) # Verify patch audit is called - self.fake_patch_audit.subcloud_audit.assert_called_with( - subcloud.name, subcloud.region_name, patch_audit_data, - software_audit_data, do_load_audit) + self.fake_patch_audit.subcloud_patch_audit.assert_called_with( + subcloud.name, subcloud.region_name, patch_audit_data, do_load_audit) # Verify firmware audit is called self.fake_firmware_audit.subcloud_firmware_audit.assert_called_with( @@ -494,13 +500,20 @@ class TestAuditWorkerManager(base.DCManagerTestCase): do_firmware_audit = True do_kubernetes_audit = True do_kube_rootca_update_audit = True - (patch_audit_data, firmware_audit_data, - kubernetes_audit_data, kube_rootca_update_audit_data, - software_audit_data) = \ - am._get_audit_data(do_patch_audit, - do_firmware_audit, - do_kubernetes_audit, - do_kube_rootca_update_audit) + do_software_audit = True + ( + patch_audit_data, + firmware_audit_data, + kubernetes_audit_data, + kube_rootca_update_audit_data, + software_audit_data, + ) = am._get_audit_data( + do_patch_audit, + do_firmware_audit, + do_kubernetes_audit, + do_kube_rootca_update_audit, + do_software_audit, + ) # Convert to dict like what would happen calling via RPC # Note: the other data should also be converted... patch_audit_data = patch_audit_data.to_dict() @@ -516,7 +529,8 @@ class TestAuditWorkerManager(base.DCManagerTestCase): do_load_audit, do_firmware_audit, do_kubernetes_audit, - do_kube_rootca_update_audit) + do_kube_rootca_update_audit, + do_software_audit) # Verify the subcloud was set to online self.fake_dcmanager_state_api.update_subcloud_availability.\ @@ -566,13 +580,20 @@ class TestAuditWorkerManager(base.DCManagerTestCase): do_firmware_audit = True do_kubernetes_audit = True do_kube_rootca_update_audit = True - (patch_audit_data, firmware_audit_data, - kubernetes_audit_data, kube_rootca_update_audit_data, - software_audit_data) = \ - am._get_audit_data(do_patch_audit, - do_firmware_audit, - do_kubernetes_audit, - do_kube_rootca_update_audit) + do_software_audit = True + ( + patch_audit_data, + firmware_audit_data, + kubernetes_audit_data, + kube_rootca_update_audit_data, + software_audit_data, + ) = am._get_audit_data( + do_patch_audit, + do_firmware_audit, + do_kubernetes_audit, + do_kube_rootca_update_audit, + do_software_audit, + ) # Convert to dict like what would happen calling via RPC # Note: the other data should also be converted... patch_audit_data = patch_audit_data.to_dict() @@ -581,7 +602,7 @@ class TestAuditWorkerManager(base.DCManagerTestCase): firmware_audit_data, kubernetes_audit_data, kube_rootca_update_audit_data, software_audit_data, do_patch_audit, do_load_audit, do_firmware_audit, do_kubernetes_audit, - do_kube_rootca_update_audit + do_kube_rootca_update_audit, do_software_audit ) # Verify the subcloud was set to online @@ -639,7 +660,8 @@ class TestAuditWorkerManager(base.DCManagerTestCase): do_load_audit=False, do_firmware_audit=False, do_kubernetes_audit=False, - do_kube_rootca_update_audit=False) + do_kube_rootca_update_audit=False, + do_software_audit=False) # Verify the subcloud state was not updated self.fake_dcmanager_state_api.update_subcloud_availability.\ @@ -683,7 +705,8 @@ class TestAuditWorkerManager(base.DCManagerTestCase): do_load_audit=False, do_firmware_audit=False, do_kubernetes_audit=False, - do_kube_rootca_update_audit=False) + do_kube_rootca_update_audit=False, + do_software_audit=False) # Verify the subcloud state was updated even though no change self.fake_dcmanager_state_api.update_subcloud_availability.\ @@ -744,13 +767,20 @@ class TestAuditWorkerManager(base.DCManagerTestCase): do_firmware_audit = True do_kubernetes_audit = True do_kube_rootca_update_audit = True - (patch_audit_data, firmware_audit_data, - kubernetes_audit_data, kube_rootca_update_audit_data, - software_audit_data) = \ - am._get_audit_data(do_patch_audit, - do_firmware_audit, - do_kubernetes_audit, - do_kube_rootca_update_audit) + do_software_audit = True + ( + patch_audit_data, + firmware_audit_data, + kubernetes_audit_data, + kube_rootca_update_audit_data, + software_audit_data, + ) = am._get_audit_data( + do_patch_audit, + do_firmware_audit, + do_kubernetes_audit, + do_kube_rootca_update_audit, + do_software_audit, + ) # Convert to dict like what would happen calling via RPC patch_audit_data = patch_audit_data.to_dict() wm._audit_subcloud( @@ -763,15 +793,16 @@ class TestAuditWorkerManager(base.DCManagerTestCase): do_patch_audit=do_patch_audit, do_load_audit=do_load_audit, do_firmware_audit=do_firmware_audit, do_kubernetes_audit=do_kubernetes_audit, - do_kube_rootca_update_audit=do_kube_rootca_update_audit) + do_kube_rootca_update_audit=do_kube_rootca_update_audit, + do_software_audit=do_software_audit) # Verify alarm update is called once self.fake_alarm_aggr.update_alarm_summary.assert_called_once_with( subcloud.name, self.fake_openstack_client.fm_client) # Verify patch audit is called once - self.fake_patch_audit.subcloud_audit.assert_called_once_with( - subcloud.name, subcloud.region_name, mock.ANY, mock.ANY, True) + self.fake_patch_audit.subcloud_patch_audit.assert_called_once_with( + subcloud.name, subcloud.region_name, mock.ANY, True) # Verify firmware audit is called once self.fake_firmware_audit.subcloud_firmware_audit.assert_called_once_with( @@ -811,7 +842,8 @@ class TestAuditWorkerManager(base.DCManagerTestCase): do_patch_audit=do_patch_audit, do_load_audit=do_load_audit, do_firmware_audit=do_firmware_audit, do_kubernetes_audit=do_kubernetes_audit, - do_kube_rootca_update_audit=do_kube_rootca_update_audit) + do_kube_rootca_update_audit=do_kube_rootca_update_audit, + do_software_audit=do_software_audit) audit_fail_count = audit_fail_count + 1 @@ -827,7 +859,7 @@ class TestAuditWorkerManager(base.DCManagerTestCase): self.fake_alarm_aggr.update_alarm_summary.assert_called_once() # Verify patch audit is called only once - self.fake_patch_audit.subcloud_audit.assert_called_once() + self.fake_patch_audit.subcloud_patch_audit.assert_called_once() # Verify firmware audit is only called once self.fake_firmware_audit.subcloud_firmware_audit.assert_called_once() @@ -861,13 +893,20 @@ class TestAuditWorkerManager(base.DCManagerTestCase): do_firmware_audit = True do_kubernetes_audit = True do_kube_rootca_update_audit = True - (patch_audit_data, firmware_audit_data, - kubernetes_audit_data, kube_rootca_update_audit_data, - software_audit_data) = \ - am._get_audit_data(do_patch_audit, - do_firmware_audit, - do_kubernetes_audit, - do_kube_rootca_update_audit) + do_software_audit = True + ( + patch_audit_data, + firmware_audit_data, + kubernetes_audit_data, + kube_rootca_update_audit_data, + software_audit_data, + ) = am._get_audit_data( + do_patch_audit, + do_firmware_audit, + do_kubernetes_audit, + do_kube_rootca_update_audit, + do_software_audit, + ) # Convert to dict like what would happen calling via RPC patch_audit_data = patch_audit_data.to_dict() wm._audit_subcloud( @@ -880,7 +919,8 @@ class TestAuditWorkerManager(base.DCManagerTestCase): do_patch_audit=do_patch_audit, do_load_audit=do_load_audit, do_firmware_audit=do_firmware_audit, do_kubernetes_audit=do_kubernetes_audit, - do_kube_rootca_update_audit=do_kube_rootca_update_audit) + do_kube_rootca_update_audit=do_kube_rootca_update_audit, + do_software_audit=do_software_audit) # Verify the subcloud state was not updated self.fake_dcmanager_state_api.update_subcloud_availability.\ @@ -969,13 +1009,20 @@ class TestAuditWorkerManager(base.DCManagerTestCase): do_firmware_audit = True do_kubernetes_audit = True do_kube_rootca_update_audit = True - (patch_audit_data, firmware_audit_data, - kubernetes_audit_data, kube_rootca_update_audit_data, - software_audit_data) = \ - am._get_audit_data(do_patch_audit, - do_firmware_audit, - do_kubernetes_audit, - do_kube_rootca_update_audit) + do_software_audit = True + ( + patch_audit_data, + firmware_audit_data, + kubernetes_audit_data, + kube_rootca_update_audit_data, + software_audit_data, + ) = am._get_audit_data( + do_patch_audit, + do_firmware_audit, + do_kubernetes_audit, + do_kube_rootca_update_audit, + do_software_audit, + ) # Convert to dict like what would happen calling via RPC patch_audit_data = patch_audit_data.to_dict() wm._audit_subcloud( @@ -988,7 +1035,8 @@ class TestAuditWorkerManager(base.DCManagerTestCase): do_patch_audit=do_patch_audit, do_load_audit=do_load_audit, do_firmware_audit=do_firmware_audit, do_kubernetes_audit=do_kubernetes_audit, - do_kube_rootca_update_audit=do_kube_rootca_update_audit) + do_kube_rootca_update_audit=do_kube_rootca_update_audit, + do_software_audit=do_software_audit) # Verify that the subcloud was updated to offline audit_fail_count = 2 @@ -1053,14 +1101,20 @@ class TestAuditWorkerManager(base.DCManagerTestCase): do_firmware_audit = True do_kubernetes_audit = True do_kube_rootca_update_audit = True - (patch_audit_data, firmware_audit_data, - kubernetes_audit_data, kube_rootca_update_audit_data, - software_audit_data - ) = \ - am._get_audit_data(do_patch_audit, - do_firmware_audit, - do_kubernetes_audit, - do_kube_rootca_update_audit) + do_software_audit = True + ( + patch_audit_data, + firmware_audit_data, + kubernetes_audit_data, + kube_rootca_update_audit_data, + software_audit_data, + ) = am._get_audit_data( + do_patch_audit, + do_firmware_audit, + do_kubernetes_audit, + do_kube_rootca_update_audit, + do_software_audit, + ) # Convert to dict like what would happen calling via RPC patch_audit_data = patch_audit_data.to_dict() wm._audit_subcloud( @@ -1073,7 +1127,8 @@ class TestAuditWorkerManager(base.DCManagerTestCase): do_patch_audit=do_patch_audit, do_load_audit=do_load_audit, do_firmware_audit=do_firmware_audit, do_kubernetes_audit=do_kubernetes_audit, - do_kube_rootca_update_audit=do_kube_rootca_update_audit) + do_kube_rootca_update_audit=do_kube_rootca_update_audit, + do_software_audit=do_software_audit) # Verify the audit fail count was updated in the DB. subcloud = db_api.subcloud_get(self.ctx, subcloud.id) @@ -1137,18 +1192,19 @@ class TestAuditWorkerManager(base.DCManagerTestCase): # Audit the subcloud wm._audit_subcloud(subcloud, - False, # update_subcloud_state - True, # do_audit_openstack - None, # patch_audit_data - None, # firmware_audit_data - None, # kubernetes_audit_data - None, # kube_rootca_update_audit_data - None, # software_audit_data - False, # do_patch_audit - False, # do_load_audit - False, # do_firmware_audit - False, # do_kubernetes_audit - False) # do_kube_rootca_audit + False, # update_subcloud_state + True, # do_audit_openstack + None, # patch_audit_data + None, # firmware_audit_data + None, # kubernetes_audit_data + None, # kube_rootca_update_audit_data + None, # software_audit_data + False, # do_patch_audit + False, # do_load_audit + False, # do_firmware_audit + False, # do_kubernetes_audit + False, # do_kube_rootca_audit + False,) # do_software_audit # Verify the subcloud state was not updated self.fake_dcmanager_state_api.update_subcloud_availability.\ @@ -1213,7 +1269,8 @@ class TestAuditWorkerManager(base.DCManagerTestCase): False, # do_load_audit, False, # do_firmware_audit False, # do_kubernetes_audit - False) # do_kube_rootca_update_audit + False, # do_kube_rootca_update_audit + False) # do_software_audit # Verify the subcloud state was not updated self.fake_dcmanager_state_api.update_subcloud_availability.\ @@ -1277,7 +1334,8 @@ class TestAuditWorkerManager(base.DCManagerTestCase): False, # do_load_audit False, # do_firmware_audit False, # do_kubernetes_audit - False) # do_kube_rootca_update_audit + False, # do_kube_rootca_update_audit + False) # do_software_audit # Verify the subcloud state was not updated self.fake_dcmanager_state_api.update_subcloud_availability.\ @@ -1329,13 +1387,21 @@ class TestAuditWorkerManager(base.DCManagerTestCase): do_firmware_audit = False do_kubernetes_audit = False do_kube_rootca_audit = False - (patch_audit_data, firmware_audit_data, - kubernetes_audit_data, kube_rootca_update_audit_data, - software_audit_data) = \ - am._get_audit_data(do_patch_audit, - do_firmware_audit, - do_kubernetes_audit, - do_kube_rootca_audit) + do_kube_rootca_update_audit = False + do_software_audit = False + ( + patch_audit_data, + firmware_audit_data, + kubernetes_audit_data, + kube_rootca_update_audit_data, + software_audit_data, + ) = am._get_audit_data( + do_patch_audit, + do_firmware_audit, + do_kubernetes_audit, + do_kube_rootca_update_audit, + do_software_audit, + ) # Convert to dict like what would happen calling via RPC patch_audit_data = patch_audit_data.to_dict() @@ -1362,12 +1428,12 @@ class TestAuditWorkerManager(base.DCManagerTestCase): do_load_audit, do_firmware_audit, do_kubernetes_audit, - do_kube_rootca_audit) + do_kube_rootca_audit, + do_software_audit) # Verify patch audit is called - self.fake_patch_audit.subcloud_audit.assert_called_with( - subcloud.name, subcloud.region_name, patch_audit_data, - software_audit_data, do_load_audit) + self.fake_patch_audit.subcloud_patch_audit.assert_called_with( + subcloud.name, subcloud.region_name, patch_audit_data, do_load_audit) # Verify the _update_subcloud_audit_fail_count is not called with mock.patch.object(wm, '_update_subcloud_audit_fail_count') as \ diff --git a/distributedcloud/test-requirements.txt b/distributedcloud/test-requirements.txt index cf7e82db2..95823427c 100644 --- a/distributedcloud/test-requirements.txt +++ b/distributedcloud/test-requirements.txt @@ -1,20 +1,18 @@ # The order of packages is significant, because pip processes them in the order # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -hacking>=1.1.0,<=2.0.0 # Apache-2.0 -cliff>=2.6.0 # Apache-2.0 + coverage!=4.4,>=4.0 # Apache-2.0 -fixtures>=3.0.0 # Apache-2.0/BSD +flake8 # MIT +hacking>=1.1.0,<=2.0.0 # Apache-2.0 mock>=2.0 # BSD +oslotest>=1.10.0 # Apache-2.0 +pylint==2.14.1 # GPLv2 python-subunit>=0.0.18 # Apache-2.0/BSD redfish # BSD requests-mock>=1.1 # Apache-2.0 testrepository>=0.0.18 # Apache-2.0/BSD -testtools>=1.4.0 # MIT testresources>=0.2.4 # Apache-2.0/BSD testscenarios>=0.4 # Apache-2.0/BSD +testtools>=1.4.0 # MIT WebTest>=2.0 # MIT -oslotest>=1.10.0 # Apache-2.0 -pylint==2.14.1;python_version>"3.7" # GPLv2 -PyYAML>=3.1.0 -yamllint<1.26.1;python_version>="3.0" # GPLv2