From abe9efce41d3b94c12aeaff14f013c5c25b76d42 Mon Sep 17 00:00:00 2001 From: Tao Liu Date: Sun, 9 Feb 2020 20:47:16 -0500 Subject: [PATCH] Keystone token and client caching This update is to change the platform SDK driver to cache the keystone token and client sessions, also to add the fm client to the platform SDK in order to remove the sdk.py once nova, neutron and cinder references are removed (as part of story 2006588). The keystone client/token is cached per-region vs. the client session, which is cached per-region & per-thread. This is because, simultaneous access from different greenthreads to the client socket is not allowed. The initial sync does not use the cached clients as requesting a new token is required for each region. In addition, this update modifies the EndpointCache class to load the auth plugin once. Test cases: 1. Manage/unmanage subclouds 2. Platform resources and alarm summary sync 3. Platform resources and alarm summary audit 4. Verify the keystone client/token is cached until the token is expired 5. Add/delete subclouds 6. Managed subcloud goes offline/online (power off/on) 7. Managed subcloud goes offline/online (delete/add a static route) Story: 2007267 Task: 38709 Change-Id: I0842a79838ea0f7a6c16f3b1e69ad0eb1357018a Signed-off-by: Tao Liu --- .../dcorch/api/proxy/common/utils.py | 8 +- .../dcorch/common/endpoint_cache.py | 18 +- .../dcorch/drivers/openstack/fm.py | 7 +- .../dcorch/drivers/openstack/sdk_platform.py | 144 ++++-- .../dcorch/drivers/openstack/sysinv_v1.py | 2 +- .../dcorch/engine/alarm_aggregate_manager.py | 32 +- .../dcorch/engine/fernet_key_manager.py | 19 +- .../dcorch/engine/sync_services/sysinv.py | 476 ++++++------------ distributedcloud/dcorch/engine/sync_thread.py | 9 +- 9 files changed, 315 insertions(+), 400 deletions(-) diff --git a/distributedcloud/dcorch/api/proxy/common/utils.py b/distributedcloud/dcorch/api/proxy/common/utils.py index dbd969213..470ba6c92 100644 --- a/distributedcloud/dcorch/api/proxy/common/utils.py +++ b/distributedcloud/dcorch/api/proxy/common/utils.py @@ -138,7 +138,8 @@ def set_request_forward_environ(req, remote_host, remote_port): def _get_fernet_keys(): """Get fernet keys from sysinv.""" - os_client = sdk.OpenStackDriver(consts.CLOUD_0) + os_client = sdk.OpenStackDriver(region_name=consts.CLOUD_0, + thread_name='proxy') try: key_list = os_client.sysinv_client.get_fernet_keys() return [str(getattr(key, 'key')) for key in key_list] @@ -146,11 +147,12 @@ def _get_fernet_keys(): keystone_exceptions.ConnectFailure) as e: LOG.info("get_fernet_keys: cloud {} is not reachable [{}]" .format(consts.CLOUD_0, str(e))) - os_client.delete_region_clients(consts.CLOUD_0) + sdk.OpenStackDriver.delete_region_clients(consts.CLOUD_0) return None except (AttributeError, TypeError) as e: LOG.info("get_fernet_keys error {}".format(e)) - os_client.delete_region_clients(consts.CLOUD_0, clear_token=True) + sdk.OpenStackDriver.delete_region_clients(consts.CLOUD_0, + clear_token=True) return None except Exception as e: LOG.exception(e) diff --git a/distributedcloud/dcorch/common/endpoint_cache.py b/distributedcloud/dcorch/common/endpoint_cache.py index 94a341a62..1de57518e 100644 --- a/distributedcloud/dcorch/common/endpoint_cache.py +++ b/distributedcloud/dcorch/common/endpoint_cache.py @@ -14,6 +14,7 @@ # under the License. import collections +import threading from keystoneauth1 import loading from keystoneauth1 import session @@ -29,6 +30,10 @@ LOG = logging.getLogger(__name__) class EndpointCache(object): + + plugin_loader = None + plugin_lock = threading.Lock() + def __init__(self, region_name=None, auth_url=None): self.endpoint_map = collections.defaultdict(dict) self.admin_session = None @@ -46,10 +51,12 @@ class EndpointCache(object): self._update_endpoints() def _initialize_keystone_client(self, region_name=None, auth_url=None): - loader = loading.get_plugin_loader( - cfg.CONF.keystone_authtoken.auth_type) + with EndpointCache.plugin_lock: + if EndpointCache.plugin_loader is None: + EndpointCache.plugin_loader = loading.get_plugin_loader( + cfg.CONF.keystone_authtoken.auth_type) - auth = loader.load_from_options( + auth = EndpointCache.plugin_loader.load_from_options( auth_url=self.external_auth_url, username=cfg.CONF.cache.admin_username, user_domain_name=cfg.CONF.cache.admin_user_domain_name, @@ -83,11 +90,10 @@ class EndpointCache(object): except IndexError: LOG.error("Cannot find identity auth_url for %s", region_name) raise - sc_loader = loading.get_plugin_loader( - cfg.CONF.keystone_authtoken.auth_type) + # We assume that the Admin user names and passwords are the same # on this subcloud since this is an audited resource - sc_auth = sc_loader.load_from_options( + sc_auth = EndpointCache.plugin_loader.load_from_options( auth_url=sc_auth_url, username=cfg.CONF.cache.admin_username, user_domain_name=cfg.CONF.cache.admin_user_domain_name, diff --git a/distributedcloud/dcorch/drivers/openstack/fm.py b/distributedcloud/dcorch/drivers/openstack/fm.py index 43b4258fe..11dd5783d 100644 --- a/distributedcloud/dcorch/drivers/openstack/fm.py +++ b/distributedcloud/dcorch/drivers/openstack/fm.py @@ -46,8 +46,7 @@ class FmClient(base.DriverBase): LOG.info("get_alarm_summary region %s" % self.region_name) alarms = self.fm.alarm.summary() - return alarms except Exception as e: - LOG.error("get_alarm_summary exception=%s" % e) - pass - return {} + LOG.error("get_alarm_summary exception={}".format(e)) + raise e + return alarms diff --git a/distributedcloud/dcorch/drivers/openstack/sdk_platform.py b/distributedcloud/dcorch/drivers/openstack/sdk_platform.py index 623659e97..9bafbd8ff 100644 --- a/distributedcloud/dcorch/drivers/openstack/sdk_platform.py +++ b/distributedcloud/dcorch/drivers/openstack/sdk_platform.py @@ -22,6 +22,7 @@ from oslo_log import log from oslo_utils import timeutils from dcorch.common import consts +from dcorch.drivers.openstack.fm import FmClient from dcorch.drivers.openstack.keystone_v3 import KeystoneClient from dcorch.drivers.openstack.sysinv_v1 import SysinvClient @@ -30,90 +31,137 @@ STALE_TOKEN_DURATION = 60 LOG = log.getLogger(__name__) +LOCK_NAME = 'dcorch-openstackdriver-platform' + class OpenStackDriver(object): - @lockutils.synchronized('dcorch-openstackdriver-platform') - def __init__(self, region_name=consts.VIRTUAL_MASTER_CLOUD, + os_clients_dict = collections.defaultdict(dict) + _identity_tokens = {} + + @lockutils.synchronized(LOCK_NAME) + def __init__(self, region_name=consts.CLOUD_0, thread_name='dcorch', auth_url=None): # Check if objects are cached and try to use those - self.os_clients_dict = collections.defaultdict(dict) - self._identity_tokens = {} - self.region_name = region_name + self.sysinv_client = None + self.fm_client = None - if ((region_name in self.os_clients_dict) and + if ((region_name in OpenStackDriver.os_clients_dict) and ('keystone' in self.os_clients_dict[region_name]) and self._is_token_valid(region_name)): self.keystone_client = \ self.os_clients_dict[region_name]['keystone'] else: LOG.info("get new keystone client for subcloud %s", region_name) - self.keystone_client = KeystoneClient(region_name, auth_url) - self.os_clients_dict[region_name]['keystone'] = \ - self.keystone_client - if ((region_name in self.os_clients_dict) and - ('sysinv' in self.os_clients_dict[region_name]) and - self._is_token_valid(region_name)): - LOG.info('Using cached OS client objects %s' % region_name) - self.sysinv_client = self.os_clients_dict[ - region_name]['sysinv'] - else: - # Create new objects and cache them - LOG.debug("Creating fresh OS Clients objects %s" % region_name) - self.os_clients_dict[ - region_name] = collections.defaultdict(dict) + try: + self.keystone_client = KeystoneClient(region_name, auth_url) + OpenStackDriver.os_clients_dict[region_name]['keystone'] =\ + self.keystone_client + except Exception as exception: + LOG.error('keystone_client region %s error: %s' % + (region_name, exception.message)) + if ((region_name in OpenStackDriver.os_clients_dict) and + (thread_name in OpenStackDriver.os_clients_dict[region_name])): + + if ('sysinv' in OpenStackDriver.os_clients_dict[region_name] + [thread_name]): + LOG.debug('Using cached OS sysinv client objects %s %s' % + (region_name, thread_name)) + self.sysinv_client = OpenStackDriver.os_clients_dict[ + region_name][thread_name]['sysinv'] + + if ('fm' in OpenStackDriver.os_clients_dict[region_name] + [thread_name]): + LOG.debug('Using cached OS fm client objects %s %s' % + (region_name, thread_name)) + self.fm_client = OpenStackDriver.os_clients_dict[ + region_name][thread_name]['fm'] + else: + OpenStackDriver.os_clients_dict[region_name][thread_name] = {} + + if self.sysinv_client is None: + # Create new sysinv client object and cache it try: self.sysinv_client = SysinvClient(region_name, self.keystone_client.session) - self.os_clients_dict[region_name][ - 'sysinv'] = self.sysinv_client - except Exception as exception: - LOG.error('sysinv_client region %s error: %s' % - (region_name, exception.message)) + (OpenStackDriver.os_clients_dict[region_name][thread_name] + ['sysinv']) = self.sysinv_client - @lockutils.synchronized('dcorch-openstackdriver-platform') - def delete_region_clients(self, region_name, clear_token=False): + except Exception as exception: + LOG.error('sysinv_client region %s thread %s error: %s' % + (region_name, thread_name, exception.message)) + + if self.fm_client is None: + # Create new fm client object and cache it + try: + self.fm_client = FmClient( + region_name, + self.keystone_client.session, + endpoint_type=consts.KS_ENDPOINT_DEFAULT) + (OpenStackDriver.os_clients_dict[region_name][thread_name] + ['fm']) = self.fm_client + except Exception as exception: + LOG.error('fm_client region %s thread %s error: %s' % + (region_name, thread_name, exception.message)) + + @classmethod + @lockutils.synchronized(LOCK_NAME) + def delete_region_clients(cls, region_name, clear_token=False): LOG.warn("delete_region_clients=%s, clear_token=%s" % (region_name, clear_token)) - if region_name in self.os_clients_dict: - del self.os_clients_dict[region_name] + if region_name in cls.os_clients_dict: + del cls.os_clients_dict[region_name] if clear_token: - self._identity_tokens[region_name] = None + cls._identity_tokens[region_name] = None + + @classmethod + @lockutils.synchronized(LOCK_NAME) + def delete_region_clients_for_thread(cls, region_name, thread_name): + LOG.debug("delete_region_clients=%s, thread_name=%s" % + (region_name, thread_name)) + if (region_name in cls.os_clients_dict and + thread_name in cls.os_clients_dict[region_name]): + del cls.os_clients_dict[region_name][thread_name] def _is_token_valid(self, region_name): try: keystone = \ - self.os_clients_dict[region_name]['keystone'].keystone_client - if (not self._identity_tokens - or region_name not in self._identity_tokens - or not self._identity_tokens[region_name]): - self._identity_tokens[region_name] = \ + OpenStackDriver.os_clients_dict[region_name]['keystone'].\ + keystone_client + if (not OpenStackDriver._identity_tokens + or region_name not in OpenStackDriver._identity_tokens + or not OpenStackDriver._identity_tokens[region_name]): + OpenStackDriver._identity_tokens[region_name] = \ keystone.tokens.validate(keystone.session.get_token()) LOG.info("Get new token for subcloud %s expires_at=%s" % (region_name, - self._identity_tokens[region_name]['expires_at'])) + OpenStackDriver._identity_tokens[region_name] + ['expires_at'])) # Reset the cached dictionary - self.os_clients_dict[region_name] = \ + OpenStackDriver.os_clients_dict[region_name] = \ collections.defaultdict(dict) return False token = \ - keystone.tokens.validate(self._identity_tokens[region_name]) - if token != self._identity_tokens[region_name]: + keystone.tokens.validate(OpenStackDriver._identity_tokens + [region_name]) + if token != OpenStackDriver._identity_tokens[region_name]: LOG.info("updating token %s to %s" % - (self._identity_tokens[region_name], token)) - self._identity_tokens[region_name] = token - self.os_clients_dict[region_name] = \ + (OpenStackDriver._identity_tokens[region_name], + token)) + OpenStackDriver._identity_tokens[region_name] = token + OpenStackDriver.os_clients_dict[region_name] = \ collections.defaultdict(dict) return False except Exception as exception: LOG.info('_is_token_valid handle: %s', exception.message) # Reset the cached dictionary - self.os_clients_dict[region_name] = collections.defaultdict(dict) - self._identity_tokens[region_name] = None + OpenStackDriver.os_clients_dict[region_name] = \ + collections.defaultdict(dict) + OpenStackDriver._identity_tokens[region_name] = None return False expiry_time = timeutils.normalize_time(timeutils.parse_isotime( @@ -122,10 +170,12 @@ class OpenStackDriver(object): LOG.info("The cached keystone token for subcloud %s " "will expire soon %s" % (region_name, - self._identity_tokens[region_name]['expires_at'])) + OpenStackDriver._identity_tokens[region_name] + ['expires_at'])) # Reset the cached dictionary - self.os_clients_dict[region_name] = collections.defaultdict(dict) - self._identity_tokens[region_name] = None + OpenStackDriver.os_clients_dict[region_name] = \ + collections.defaultdict(dict) + OpenStackDriver._identity_tokens[region_name] = None return False else: return True diff --git a/distributedcloud/dcorch/drivers/openstack/sysinv_v1.py b/distributedcloud/dcorch/drivers/openstack/sysinv_v1.py index e977191af..7a0ca36d0 100644 --- a/distributedcloud/dcorch/drivers/openstack/sysinv_v1.py +++ b/distributedcloud/dcorch/drivers/openstack/sysinv_v1.py @@ -188,7 +188,7 @@ class SysinvClient(base.DriverBase): self.region_name, trapdest_ip_address)) self.client.itrapdest.delete(trapdest_ip_address) except HTTPNotFound: - LOG.info("snmp_trapdest_delete NotFound %s for region: {}".format( + LOG.info("snmp_trapdest_delete NotFound {} for region: {}".format( trapdest_ip_address, self.region_name)) raise exceptions.TrapDestNotFound(region_name=self.region_name, ip_address=trapdest_ip_address) diff --git a/distributedcloud/dcorch/engine/alarm_aggregate_manager.py b/distributedcloud/dcorch/engine/alarm_aggregate_manager.py index 0713193bf..7dde7d848 100644 --- a/distributedcloud/dcorch/engine/alarm_aggregate_manager.py +++ b/distributedcloud/dcorch/engine/alarm_aggregate_manager.py @@ -21,8 +21,11 @@ from dcorch.common import exceptions from dcorch.common.i18n import _ from dcorch.common import manager from dcorch.db import api as db_api -from dcorch.drivers.openstack import sdk -from dcorch.drivers.openstack import sdk_platform + +from dcorch.drivers.openstack.fm import FmClient +from dcorch.drivers.openstack.keystone_v3 import KeystoneClient +from dcorch.drivers.openstack import sdk_platform as sdk +from dcorch.drivers.openstack.sysinv_v1 import SysinvClient from oslo_config import cfg from oslo_log import log as logging @@ -53,12 +56,17 @@ class AlarmAggregateManager(manager.Manager): def enable_snmp(self, ctxt, subcloud_name): LOG.info("Enabling fm-aggregation trap for region_name=%s" % subcloud_name) - os_client = sdk_platform.OpenStackDriver(subcloud_name) + payload = {"ip_address": CONF.snmp.snmp_ip, "community": CONF.snmp.snmp_comm_str} try: - os_client.sysinv_client.snmp_trapdest_create(payload) - self.update_alarm_summary(self.context, subcloud_name) + ks_client = KeystoneClient(subcloud_name) + sysinv_client = SysinvClient(subcloud_name, ks_client.session) + fm_client = FmClient(subcloud_name, ks_client.session, + consts.KS_ENDPOINT_DEFAULT) + sysinv_client.snmp_trapdest_create(payload) + self.update_alarm_summary(self.context, subcloud_name, + fm_client=fm_client) except (exceptions.ConnectionRefused, exceptions.NotAuthorized, exceptions.TimeOut): LOG.info("snmp_trapdest_create exception Timeout region_name=%s" % @@ -77,11 +85,16 @@ class AlarmAggregateManager(manager.Manager): subcloud_name) pass - def update_alarm_summary(self, cntx, region_name): + def update_alarm_summary(self, cntx, region_name, thread_name=None, + fm_client=None): LOG.info("Updating alarm summary for %s" % region_name) try: - os_client = sdk.OpenStackDriver(region_name) - alarms = os_client.fm_client.get_alarm_summary() + if fm_client is not None: + alarms = fm_client.get_alarm_summary() + else: + os_client = sdk.OpenStackDriver(region_name=region_name, + thread_name=thread_name) + alarms = os_client.fm_client.get_alarm_summary() alarm_updates = {'critical_alarms': alarms[0].critical, 'major_alarms': alarms[0].major, 'minor_alarms': alarms[0].minor, @@ -144,7 +157,8 @@ class PeriodicAlarmUpdate(threading.Thread): dcm_consts.AVAILABILITY_ONLINE: self.parent.\ update_alarm_summary(self.context, - subcloud['region_name']) + subcloud['region_name'], + self.name) except Exception: pass time.sleep(1.0) diff --git a/distributedcloud/dcorch/engine/fernet_key_manager.py b/distributedcloud/dcorch/engine/fernet_key_manager.py index bf2debda3..b6e36eff9 100644 --- a/distributedcloud/dcorch/engine/fernet_key_manager.py +++ b/distributedcloud/dcorch/engine/fernet_key_manager.py @@ -26,8 +26,8 @@ from dcorch.common import exceptions from dcorch.common.i18n import _ from dcorch.common import manager from dcorch.common import utils -from dcorch.drivers.openstack import sdk_platform as sdk - +from dcorch.drivers.openstack.keystone_v3 import KeystoneClient +from dcorch.drivers.openstack.sysinv_v1 import SysinvClient FERNET_REPO_MASTER_ID = "keys" KEY_ROTATE_CMD = "/usr/bin/keystone-fernet-keys-rotate-active" @@ -89,8 +89,12 @@ class FernetKeyManager(manager.Manager): """get the keys from the local fernet key repo""" keys = [] try: - os_client = sdk.OpenStackDriver(consts.CLOUD_0) - keys = os_client.sysinv_client.get_fernet_keys() + # No cached client is required as it is called during the initial + # sync and after weekly key rotation + ks_client = KeystoneClient(consts.CLOUD_0) + sysinv_client = SysinvClient(consts.CLOUD_0, + ks_client.session) + keys = sysinv_client.get_fernet_keys() except (exceptions.ConnectionRefused, exceptions.NotAuthorized, exceptions.TimeOut): LOG.info(_("Retrieving the fernet keys from %s timeout") % @@ -130,8 +134,11 @@ class FernetKeyManager(manager.Manager): @staticmethod def update_fernet_repo(subcloud_name, key_list=None): try: - os_client = sdk.OpenStackDriver(subcloud_name) - os_client.sysinv_client.post_fernet_repo(key_list) + # No cached client is required as it is only called during the + # initial sync + ks_client = KeystoneClient(subcloud_name) + sysinv_client = SysinvClient(subcloud_name, ks_client.session) + sysinv_client.post_fernet_repo(key_list) except (exceptions.ConnectionRefused, exceptions.NotAuthorized, exceptions.TimeOut): LOG.info(_("Update the fernet repo on %s timeout") % diff --git a/distributedcloud/dcorch/engine/sync_services/sysinv.py b/distributedcloud/dcorch/engine/sync_services/sysinv.py index 9acf8bda7..33ae43684 100644 --- a/distributedcloud/dcorch/engine/sync_services/sysinv.py +++ b/distributedcloud/dcorch/engine/sync_services/sysinv.py @@ -53,18 +53,20 @@ class SysinvSyncThread(SyncThread): self.endpoint_type = consts.ENDPOINT_TYPE_PLATFORM self.sync_handler_map = { - consts.RESOURCE_TYPE_SYSINV_DNS: self.sync_dns, + consts.RESOURCE_TYPE_SYSINV_DNS: + self.sync_platform_resource, consts.RESOURCE_TYPE_SYSINV_SNMP_COMM: - self.sync_snmp_community, + self.sync_platform_resource, consts.RESOURCE_TYPE_SYSINV_SNMP_TRAPDEST: - self.sync_snmp_trapdest, + self.sync_platform_resource, consts.RESOURCE_TYPE_SYSINV_REMOTE_LOGGING: - self.sync_remotelogging, + self.sync_platform_resource, consts.RESOURCE_TYPE_SYSINV_CERTIFICATE: - self.sync_certificate, - consts.RESOURCE_TYPE_SYSINV_USER: self.sync_user, + self.sync_platform_resource, + consts.RESOURCE_TYPE_SYSINV_USER: + self.sync_platform_resource, consts.RESOURCE_TYPE_SYSINV_FERNET_REPO: - self.sync_fernet_resources + self.sync_platform_resource } self.region_name = self.subcloud_engine.subcloud.region_name self.log_extra = {"instance": "{}/{}: ".format( @@ -84,31 +86,49 @@ class SysinvSyncThread(SyncThread): super(SysinvSyncThread, self).initialize() LOG.info("SysinvSyncThread initialized", extra=self.log_extra) - def update_dns(self, nameservers): + def sync_platform_resource(self, request, rsrc): try: - s_os_client = sdk.OpenStackDriver(self.region_name) - idns = s_os_client.sysinv_client.update_dns(nameservers) - return idns - except (exceptions.ConnectionRefused, exceptions.NotAuthorized, - exceptions.TimeOut): - LOG.info("update_dns exception Timeout", - extra=self.log_extra) - s_os_client.delete_region_clients(self.region_name) + s_os_client = sdk.OpenStackDriver(region_name=self.region_name, + thread_name=self.thread.name) + # invoke the sync method for the requested resource_type + # I.e. sync_idns + s_func_name = "sync_" + rsrc.resource_type + getattr(self, s_func_name)(s_os_client, request, rsrc) + except AttributeError: + LOG.error("{} not implemented for {}" + .format(request.orch_job.operation_type, + rsrc.resource_type)) + raise exceptions.SyncRequestFailed + except (exceptions.ConnectionRefused, exceptions.TimeOut, + keystone_exceptions.connection.ConnectTimeout, + keystone_exceptions.ConnectFailure) as e: + LOG.info("{} {} region_name {} exception {}".format( + request.orch_job.operation_type, rsrc.resource_type, + self.region_name, str(e)), extra=self.log_extra) raise exceptions.SyncRequestTimeout - except (AttributeError, TypeError) as e: - LOG.info("update_dns error {} region_name".format(e), - extra=self.log_extra) - s_os_client.delete_region_clients(self.region_name, - clear_token=True) + except exceptions.NotAuthorized: + LOG.info("{} {} region_name {} not authorized".format( + request.orch_job.operation_type, rsrc.resource_type, + self.region_name), extra=self.log_extra) + sdk.OpenStackDriver.delete_region_clients(self.region_name) raise exceptions.SyncRequestFailedRetry except Exception as e: LOG.exception(e) raise exceptions.SyncRequestFailedRetry - def sync_dns(self, request, rsrc): + def update_dns(self, s_os_client, nameservers): + try: + idns = s_os_client.sysinv_client.update_dns(nameservers) + return idns + except (AttributeError, TypeError) as e: + LOG.info("update_dns error {}".format(e), + extra=self.log_extra) + raise exceptions.SyncRequestFailedRetry + + def sync_idns(self, s_os_client, request, rsrc): # The system is created with default dns; thus there # is a prepopulated dns entry. - LOG.info("sync_dns resource_info={}".format( + LOG.info("sync_idns resource_info={}".format( request.orch_job.resource_info), extra=self.log_extra) dns_dict = jsonutils.loads(request.orch_job.resource_info) @@ -119,21 +139,21 @@ class SysinvSyncThread(SyncThread): for ipayload in payload: if ipayload.get('path') == '/nameservers': nameservers = ipayload.get('value') - LOG.debug("sync_dns nameservers = {}".format(nameservers), + LOG.debug("sync_idns nameservers = {}".format(nameservers), extra=self.log_extra) break else: nameservers = payload.get('nameservers') - LOG.debug("sync_dns nameservers from dict={}".format(nameservers), + LOG.debug("sync_idns nameservers from dict={}".format(nameservers), extra=self.log_extra) if nameservers is None: - LOG.info("sync_dns No nameservers update found in resource_info" + LOG.info("sync_idns No nameservers update found in resource_info" "{}".format(request.orch_job.resource_info), extra=self.log_extra) nameservers = "" - idns = self.update_dns(nameservers) + idns = self.update_dns(s_os_client, nameservers) # Ensure subcloud resource is persisted to the DB for later subcloud_rsrc_id = self.persist_db_subcloud_resource( @@ -142,7 +162,7 @@ class SysinvSyncThread(SyncThread): .format(rsrc.id, subcloud_rsrc_id, nameservers), extra=self.log_extra) - def sync_snmp_trapdest(self, request, rsrc): + def sync_itrapdest(self, s_os_client, request, rsrc): switcher = { consts.OPERATION_TYPE_POST: self.snmp_trapdest_create, consts.OPERATION_TYPE_CREATE: self.snmp_trapdest_create, @@ -151,18 +171,12 @@ class SysinvSyncThread(SyncThread): func = switcher[request.orch_job.operation_type] try: - func(request, rsrc) - except (keystone_exceptions.connection.ConnectTimeout, - keystone_exceptions.ConnectFailure) as e: - LOG.info("sync_snmp_trapdest: subcloud {} is not reachable [{}]" - .format(self.subcloud_engine.subcloud.region_name, - str(e)), extra=self.log_extra) - raise exceptions.SyncRequestTimeout + func(s_os_client, request, rsrc) except Exception as e: LOG.exception(e) - raise exceptions.SyncRequestFailedRetry + raise e - def snmp_trapdest_create(self, request, rsrc): + def snmp_trapdest_create(self, s_os_client, request, rsrc): LOG.info("snmp_trapdest_create region {} resource_info={}".format( self.subcloud_engine.subcloud.region_name, request.orch_job.resource_info), @@ -172,23 +186,14 @@ class SysinvSyncThread(SyncThread): if not payload: payload = resource_info_dict - s_os_client = sdk.OpenStackDriver(self.region_name) try: itrapdest = s_os_client.sysinv_client.snmp_trapdest_create( payload) itrapdest_id = itrapdest.uuid ip_address = itrapdest.ip_address - except (exceptions.ConnectionRefused, exceptions.NotAuthorized, - exceptions.TimeOut): - LOG.info("snmp_trapdest_create exception Timeout", - extra=self.log_extra) - s_os_client.delete_region_clients(self.region_name) - raise exceptions.SyncRequestTimeout except (AttributeError, TypeError) as e: LOG.info("snmp_trapdest_create error {}".format(e), extra=self.log_extra) - s_os_client.delete_region_clients(self.region_name, - clear_token=True) raise exceptions.SyncRequestFailedRetry # Now persist the subcloud resource to the DB for later @@ -200,11 +205,11 @@ class SysinvSyncThread(SyncThread): extra=self.log_extra) return itrapdest - def snmp_trapdest_delete(self, request, rsrc): + def snmp_trapdest_delete(self, s_os_client, request, rsrc): subcloud_rsrc = self.get_db_subcloud_resource(rsrc.id) if not subcloud_rsrc: return - s_os_client = sdk.OpenStackDriver(self.region_name) + try: s_os_client.sysinv_client.snmp_trapdest_delete( subcloud_rsrc.subcloud_resource_id) @@ -212,17 +217,9 @@ class SysinvSyncThread(SyncThread): # SNMP trapdest already deleted in subcloud, carry on. LOG.info("SNMP trapdest not in subcloud, may be already deleted", extra=self.log_extra) - except (exceptions.ConnectionRefused, exceptions.NotAuthorized, - exceptions.TimeOut): - LOG.info("snmp_trapdest_delete exception Timeout", - extra=self.log_extra) - s_os_client.delete_region_clients(self.region_name) - raise exceptions.SyncRequestTimeout except (AttributeError, TypeError) as e: LOG.info("snmp_trapdest_delete error {}".format(e), extra=self.log_extra) - s_os_client.delete_region_clients(self.region_name, - clear_token=True) raise exceptions.SyncRequestFailedRetry subcloud_rsrc.delete() @@ -233,7 +230,7 @@ class SysinvSyncThread(SyncThread): subcloud_rsrc.subcloud_resource_id), extra=self.log_extra) - def sync_snmp_community(self, request, rsrc): + def sync_icommunity(self, s_os_client, request, rsrc): switcher = { consts.OPERATION_TYPE_POST: self.snmp_community_create, consts.OPERATION_TYPE_CREATE: self.snmp_community_create, @@ -242,18 +239,12 @@ class SysinvSyncThread(SyncThread): func = switcher[request.orch_job.operation_type] try: - func(request, rsrc) - except (keystone_exceptions.connection.ConnectTimeout, - keystone_exceptions.ConnectFailure) as e: - LOG.info("sync_snmp_community: subcloud {} is not reachable [{}]" - .format(self.subcloud_engine.subcloud.region_name, - str(e)), extra=self.log_extra) - raise exceptions.SyncRequestTimeout + func(s_os_client, request, rsrc) except Exception as e: LOG.exception(e) raise exceptions.SyncRequestFailedRetry - def snmp_community_create(self, request, rsrc): + def snmp_community_create(self, s_os_client, request, rsrc): LOG.info("snmp_community_create region {} resource_info={}".format( self.subcloud_engine.subcloud.region_name, request.orch_job.resource_info), @@ -263,23 +254,14 @@ class SysinvSyncThread(SyncThread): if not payload: payload = resource_info_dict - s_os_client = sdk.OpenStackDriver(self.region_name) try: icommunity = s_os_client.sysinv_client.snmp_community_create( payload) icommunity_id = icommunity.uuid community = icommunity.community - except (exceptions.ConnectionRefused, exceptions.NotAuthorized, - exceptions.TimeOut): - LOG.info("snmp_community_create exception Timeout", - extra=self.log_extra) - s_os_client.delete_region_clients(self.region_name) - raise exceptions.SyncRequestTimeout except (AttributeError, TypeError) as e: LOG.info("snmp_community_create error {}".format(e), extra=self.log_extra) - s_os_client.delete_region_clients(self.region_name, - clear_token=True) raise exceptions.SyncRequestFailedRetry # Now persist the subcloud resource to the DB for later @@ -291,11 +273,10 @@ class SysinvSyncThread(SyncThread): extra=self.log_extra) return icommunity - def snmp_community_delete(self, request, rsrc): + def snmp_community_delete(self, s_os_client, request, rsrc): subcloud_rsrc = self.get_db_subcloud_resource(rsrc.id) if not subcloud_rsrc: return - s_os_client = sdk.OpenStackDriver(self.region_name) try: s_os_client.sysinv_client.snmp_community_delete( subcloud_rsrc.subcloud_resource_id) @@ -303,17 +284,9 @@ class SysinvSyncThread(SyncThread): # Community already deleted in subcloud, carry on. LOG.info("SNMP community not in subcloud, may be already deleted", extra=self.log_extra) - except (exceptions.ConnectionRefused, exceptions.NotAuthorized, - exceptions.TimeOut): - LOG.info("snmp_community_delete exception Timeout", - extra=self.log_extra) - s_os_client.delete_region_clients(self.region_name) - raise exceptions.SyncRequestTimeout except (AttributeError, TypeError) as e: LOG.info("snmp_community_delete error {}".format(e), extra=self.log_extra) - s_os_client.delete_region_clients(self.region_name, - clear_token=True) raise exceptions.SyncRequestFailedRetry subcloud_rsrc.delete() @@ -324,30 +297,18 @@ class SysinvSyncThread(SyncThread): subcloud_rsrc.subcloud_resource_id), extra=self.log_extra) - def update_remotelogging(self, values): + def update_remotelogging(self, s_os_client, values): - s_os_client = sdk.OpenStackDriver(self.region_name) try: iremotelogging = s_os_client.sysinv_client.update_remotelogging( values) return iremotelogging - except (exceptions.ConnectionRefused, exceptions.NotAuthorized, - exceptions.TimeOut): - LOG.info("update_remotelogging exception Timeout", - extra=self.log_extra) - s_os_client.delete_region_clients(self.region_name) - raise exceptions.SyncRequestTimeout except (AttributeError, TypeError) as e: LOG.info("update_remotelogging error {} region_name".format(e), extra=self.log_extra) - s_os_client.delete_region_clients(self.region_name, - clear_token=True) - raise exceptions.SyncRequestFailedRetry - except Exception as e: - LOG.exception(e) raise exceptions.SyncRequestFailedRetry - def sync_remotelogging(self, request, rsrc): + def sync_remotelogging(self, s_os_client, request, rsrc): # The system is created with default remotelogging; thus there # is a prepopulated remotelogging entry. LOG.info("sync_remotelogging resource_info={}".format( @@ -362,7 +323,7 @@ class SysinvSyncThread(SyncThread): extra=self.log_extra) return - iremotelogging = self.update_remotelogging(payload) + iremotelogging = self.update_remotelogging(s_os_client, payload) # Ensure subcloud resource is persisted to the DB for later subcloud_rsrc_id = self.persist_db_subcloud_resource( @@ -373,27 +334,16 @@ class SysinvSyncThread(SyncThread): iremotelogging.uuid), extra=self.log_extra) - def update_certificate(self, signature, certificate=None, data=None): + def update_certificate(self, s_os_client, signature, + certificate=None, data=None): - s_os_client = sdk.OpenStackDriver(self.region_name) try: icertificate = s_os_client.sysinv_client.update_certificate( signature, certificate=certificate, data=data) return icertificate - except (exceptions.ConnectionRefused, exceptions.NotAuthorized, - exceptions.TimeOut): - LOG.info("update_certificate exception Timeout", - extra=self.log_extra) - s_os_client.delete_region_clients(self.region_name) - raise exceptions.SyncRequestTimeout except (AttributeError, TypeError) as e: LOG.info("update_certificate error {} region_name".format(e), extra=self.log_extra) - s_os_client.delete_region_clients(self.region_name, - clear_token=True) - raise exceptions.SyncRequestFailedRetry - except Exception as e: - LOG.exception(e) raise exceptions.SyncRequestFailedRetry @staticmethod @@ -427,7 +377,7 @@ class SysinvSyncThread(SyncThread): metadata)) return certificate, metadata - def sync_certificate(self, request, rsrc): + def sync_certificates(self, s_os_client, request, rsrc): LOG.info("sync_certificate resource_info={}".format( request.orch_job.resource_info), extra=self.log_extra) @@ -454,6 +404,7 @@ class SysinvSyncThread(SyncThread): signature = rsrc.master_id if signature and signature != self.CERTIFICATE_SIG_NULL: icertificate = self.update_certificate( + s_os_client, signature, certificate=certificate, data=metadata) @@ -471,34 +422,23 @@ class SysinvSyncThread(SyncThread): subcloud_rsrc_id, isignature, signature), extra=self.log_extra) - def update_user(self, passwd_hash, root_sig, passwd_expiry_days): + def update_user(self, s_os_client, passwd_hash, + root_sig, passwd_expiry_days): LOG.info("update_user={} {} {}".format( passwd_hash, root_sig, passwd_expiry_days), extra=self.log_extra) try: - s_os_client = sdk.OpenStackDriver(self.region_name) iuser = s_os_client.sysinv_client.update_user(passwd_hash, root_sig, passwd_expiry_days) return iuser - except (exceptions.ConnectionRefused, exceptions.NotAuthorized, - exceptions.TimeOut): - LOG.info("update_user exception Timeout", - extra=self.log_extra) - s_os_client.delete_region_clients(self.region_name) - raise exceptions.SyncRequestTimeout except (AttributeError, TypeError) as e: LOG.info("update_user error {} region_name".format(e), extra=self.log_extra) - s_os_client.delete_region_clients(self.region_name, - clear_token=True) - raise exceptions.SyncRequestFailedRetry - except Exception as e: - LOG.exception(e) raise exceptions.SyncRequestFailedRetry - def sync_user(self, request, rsrc): + def sync_iuser(self, s_os_client, request, rsrc): # The system is populated with user entry for wrsroot. LOG.info("sync_user resource_info={}".format( request.orch_job.resource_info), @@ -531,7 +471,8 @@ class SysinvSyncThread(SyncThread): extra=self.log_extra) return - iuser = self.update_user(passwd_hash, root_sig, passwd_expiry_days) + iuser = self.update_user(s_os_client, passwd_hash, root_sig, + passwd_expiry_days) # Ensure subcloud resource is persisted to the DB for later subcloud_rsrc_id = self.persist_db_subcloud_resource( @@ -540,7 +481,7 @@ class SysinvSyncThread(SyncThread): .format(rsrc.id, subcloud_rsrc_id, passwd_hash), extra=self.log_extra) - def sync_fernet_resources(self, request, rsrc): + def sync_fernet_repo(self, s_os_client, request, rsrc): switcher = { consts.OPERATION_TYPE_PUT: self.update_fernet_repo, consts.OPERATION_TYPE_PATCH: self.update_fernet_repo, @@ -549,7 +490,7 @@ class SysinvSyncThread(SyncThread): func = switcher[request.orch_job.operation_type] try: - func(request, rsrc) + func(s_os_client, request, rsrc) except (keystone_exceptions.connection.ConnectTimeout, keystone_exceptions.ConnectFailure) as e: LOG.info("sync_fernet_resources: subcloud {} is not reachable [{}]" @@ -560,64 +501,44 @@ class SysinvSyncThread(SyncThread): LOG.exception(e) raise exceptions.SyncRequestFailedRetry - def create_fernet_repo(self, request, rsrc): + def create_fernet_repo(self, s_os_client, request, rsrc): LOG.info("create_fernet_repo region {} resource_info={}".format( self.subcloud_engine.subcloud.region_name, request.orch_job.resource_info), extra=self.log_extra) resource_info = jsonutils.loads(request.orch_job.resource_info) - s_os_client = sdk.OpenStackDriver(self.region_name) try: s_os_client.sysinv_client.post_fernet_repo( FernetKeyManager.from_resource_info(resource_info)) # Ensure subcloud resource is persisted to the DB for later subcloud_rsrc_id = self.persist_db_subcloud_resource( rsrc.id, rsrc.master_id) - except (exceptions.ConnectionRefused, exceptions.NotAuthorized, - exceptions.TimeOut): - LOG.info("create_fernet_repo Timeout,{}:{}".format( - rsrc.id, subcloud_rsrc_id)) - s_os_client.delete_region_clients(self.region_name, - clear_token=True) - raise exceptions.SyncRequestTimeout except (AttributeError, TypeError) as e: LOG.info("create_fernet_repo error {}".format(e), extra=self.log_extra) - s_os_client.delete_region_clients(self.region_name, - clear_token=True) raise exceptions.SyncRequestFailedRetry LOG.info("fernet_repo {} {} {} created".format(rsrc.id, subcloud_rsrc_id, resource_info), extra=self.log_extra) - def update_fernet_repo(self, request, rsrc): + def update_fernet_repo(self, s_os_client, request, rsrc): LOG.info("update_fernet_repo region {} resource_info={}".format( self.subcloud_engine.subcloud.region_name, request.orch_job.resource_info), extra=self.log_extra) resource_info = jsonutils.loads(request.orch_job.resource_info) - s_os_client = sdk.OpenStackDriver(self.region_name) try: s_os_client.sysinv_client.put_fernet_repo( FernetKeyManager.from_resource_info(resource_info)) # Ensure subcloud resource is persisted to the DB for later subcloud_rsrc_id = self.persist_db_subcloud_resource( rsrc.id, rsrc.master_id) - except (exceptions.ConnectionRefused, exceptions.NotAuthorized, - exceptions.TimeOut): - LOG.info("update_fernet_repo Timeout,{}:{}".format( - rsrc.id, subcloud_rsrc_id)) - s_os_client.delete_region_clients(self.region_name, - clear_token=True) - raise exceptions.SyncRequestTimeout except (AttributeError, TypeError) as e: LOG.info("update_fernet_repo error {}".format(e), extra=self.log_extra) - s_os_client.delete_region_clients(self.region_name, - clear_token=True) raise exceptions.SyncRequestFailedRetry LOG.info("fernet_repo {} {} {} update".format(rsrc.id, @@ -626,193 +547,104 @@ class SysinvSyncThread(SyncThread): # SysInv Audit Related def get_master_resources(self, resource_type): - os_client = sdk.OpenStackDriver(consts.CLOUD_0) - if resource_type == consts.RESOURCE_TYPE_SYSINV_DNS: - return [self.get_dns_resource(os_client)] - elif resource_type == consts.RESOURCE_TYPE_SYSINV_SNMP_COMM: - return self.get_snmp_community_resources(os_client) - elif resource_type == consts.RESOURCE_TYPE_SYSINV_SNMP_TRAPDEST: - return self.get_snmp_trapdest_resources(os_client) - elif resource_type == consts.RESOURCE_TYPE_SYSINV_REMOTE_LOGGING: - return [self.get_remotelogging_resource(os_client)] - elif resource_type == consts.RESOURCE_TYPE_SYSINV_CERTIFICATE: - return self.get_certificates_resources(os_client) - elif resource_type == consts.RESOURCE_TYPE_SYSINV_USER: - return [self.get_user_resource(os_client)] - elif resource_type == consts.RESOURCE_TYPE_SYSINV_FERNET_REPO: - return [self.get_fernet_resources(os_client)] - else: - LOG.error("Wrong resource type {}".format(resource_type), - extra=self.log_extra) + try: + os_client = sdk.OpenStackDriver(region_name=consts.CLOUD_0, + thread_name=self.audit_thread.name) + if resource_type == consts.RESOURCE_TYPE_SYSINV_DNS: + return [self.get_dns_resource(os_client)] + elif resource_type == consts.RESOURCE_TYPE_SYSINV_SNMP_COMM: + return self.get_snmp_community_resources(os_client) + elif resource_type == consts.RESOURCE_TYPE_SYSINV_SNMP_TRAPDEST: + return self.get_snmp_trapdest_resources(os_client) + elif resource_type == consts.RESOURCE_TYPE_SYSINV_REMOTE_LOGGING: + return [self.get_remotelogging_resource(os_client)] + elif resource_type == consts.RESOURCE_TYPE_SYSINV_CERTIFICATE: + return self.get_certificates_resources(os_client) + elif resource_type == consts.RESOURCE_TYPE_SYSINV_USER: + return [self.get_user_resource(os_client)] + elif resource_type == consts.RESOURCE_TYPE_SYSINV_FERNET_REPO: + return [self.get_fernet_resources(os_client)] + else: + LOG.error("Wrong resource type {}".format(resource_type), + extra=self.log_extra) + return None + except Exception as e: + LOG.exception(e) return None def get_subcloud_resources(self, resource_type): - os_client = sdk.OpenStackDriver(self.region_name) - if resource_type == consts.RESOURCE_TYPE_SYSINV_DNS: - return [self.get_dns_resource(os_client)] - elif resource_type == consts.RESOURCE_TYPE_SYSINV_SNMP_COMM: - return self.get_snmp_community_resources(os_client) - elif resource_type == consts.RESOURCE_TYPE_SYSINV_SNMP_TRAPDEST: - return self.get_snmp_trapdest_resources(os_client) - elif resource_type == consts.RESOURCE_TYPE_SYSINV_REMOTE_LOGGING: - return [self.get_remotelogging_resource(os_client)] - elif resource_type == consts.RESOURCE_TYPE_SYSINV_CERTIFICATE: - return self.get_certificates_resources(os_client) - elif resource_type == consts.RESOURCE_TYPE_SYSINV_USER: - return [self.get_user_resource(os_client)] - elif resource_type == consts.RESOURCE_TYPE_SYSINV_FERNET_REPO: - return [self.get_fernet_resources(os_client)] - else: - LOG.error("Wrong resource type {}".format(resource_type), - extra=self.log_extra) + try: + os_client = sdk.OpenStackDriver(region_name=self.region_name, + thread_name=self.audit_thread.name) + if resource_type == consts.RESOURCE_TYPE_SYSINV_DNS: + return [self.get_dns_resource(os_client)] + elif resource_type == consts.RESOURCE_TYPE_SYSINV_SNMP_COMM: + return self.get_snmp_community_resources(os_client) + elif resource_type == consts.RESOURCE_TYPE_SYSINV_SNMP_TRAPDEST: + return self.get_snmp_trapdest_resources(os_client) + elif resource_type == consts.RESOURCE_TYPE_SYSINV_REMOTE_LOGGING: + return [self.get_remotelogging_resource(os_client)] + elif resource_type == consts.RESOURCE_TYPE_SYSINV_CERTIFICATE: + return self.get_certificates_resources(os_client) + elif resource_type == consts.RESOURCE_TYPE_SYSINV_USER: + return [self.get_user_resource(os_client)] + elif resource_type == consts.RESOURCE_TYPE_SYSINV_FERNET_REPO: + return [self.get_fernet_resources(os_client)] + else: + LOG.error("Wrong resource type {}".format(resource_type), + extra=self.log_extra) + return None + except (exceptions.ConnectionRefused, exceptions.TimeOut, + keystone_exceptions.connection.ConnectTimeout, + keystone_exceptions.ConnectFailure) as e: + LOG.info("get subcloud_resources {}: subcloud {} is not reachable" + "[{}]".format(resource_type, + self.subcloud_engine.subcloud.region_name, + str(e)), extra=self.log_extra) + # None will force skip of audit return None + except exceptions.NotAuthorized as e: + LOG.info("get subcloud_resources {}: subcloud {} not authorized" + "[{}]".format(resource_type, + self.subcloud_engine.subcloud.region_name, + str(e)), extra=self.log_extra) + sdk.OpenStackDriver.delete_region_clients(self.region_name) + return None + except (AttributeError, TypeError) as e: + LOG.info("get subcloud_resources {} error {}".format( + resource_type, e), extra=self.log_extra) + return None + except Exception as e: + LOG.exception(e) + return None + + def post_audit(self): + sdk.OpenStackDriver.delete_region_clients_for_thread( + self.region_name, self.audit_thread.name) + sdk.OpenStackDriver.delete_region_clients_for_thread( + consts.CLOUD_0, self.audit_thread.name) def get_dns_resource(self, os_client): - try: - idns = os_client.sysinv_client.get_dns() - return idns - except (keystone_exceptions.connection.ConnectTimeout, - keystone_exceptions.ConnectFailure) as e: - LOG.info("get_dns: subcloud {} is not reachable [{}]" - .format(self.subcloud_engine.subcloud.region_name, - str(e)), extra=self.log_extra) - # TODO(knasim-wrs): This is a bad design to delete the - # client here as the parent may be passing in a shared - # client. Return error here and let parent - # (get_master_resources or get_subcloud_resources) clean - # it up. - os_client.delete_region_clients(self.region_name) - # None will force skip of audit - return None - except (AttributeError, TypeError) as e: - LOG.info("get_dns_resources error {}".format(e), - extra=self.log_extra) - os_client.delete_region_clients(self.region_name, clear_token=True) - return None - except Exception as e: - LOG.exception(e) - return None + return os_client.sysinv_client.get_dns() def get_snmp_trapdest_resources(self, os_client): - try: - itrapdests = os_client.sysinv_client.snmp_trapdest_list() - return itrapdests - except (keystone_exceptions.connection.ConnectTimeout, - keystone_exceptions.ConnectFailure) as e: - LOG.info("snmp_trapdest_list: subcloud {} is not reachable [{}]" - .format(self.subcloud_engine.subcloud.region_name, - str(e)), extra=self.log_extra) - return None - except (AttributeError, TypeError) as e: - LOG.info("get_snmp_trapdest_resources error {}".format(e), - extra=self.log_extra) - os_client.delete_region_clients(self.region_name, clear_token=True) - return None - except Exception as e: - LOG.exception(e) - return None + return os_client.sysinv_client.snmp_trapdest_list() def get_snmp_community_resources(self, os_client): - try: - icommunitys = os_client.sysinv_client.snmp_community_list() - return icommunitys - except (keystone_exceptions.connection.ConnectTimeout, - keystone_exceptions.ConnectFailure) as e: - LOG.info("snmp_community_list: subcloud {} is not reachable [{}]" - .format(self.subcloud_engine.subcloud.region_name, - str(e)), extra=self.log_extra) - return None - except (AttributeError, TypeError) as e: - LOG.info("get_snmp_community_resources error {}".format(e), - extra=self.log_extra) - os_client.delete_region_clients(self.region_name, clear_token=True) - return None - except Exception as e: - LOG.exception(e) - return None + return os_client.sysinv_client.snmp_community_list() def get_remotelogging_resource(self, os_client): - try: - iremotelogging = os_client.sysinv_client.get_remotelogging() - return iremotelogging - except (keystone_exceptions.connection.ConnectTimeout, - keystone_exceptions.ConnectFailure) as e: - LOG.info("get_remotelogging: subcloud {} is not reachable [{}]" - .format(self.subcloud_engine.subcloud.region_name, - str(e)), extra=self.log_extra) - # None will force skip of audit - os_client.delete_region_clients(self.region_name) - return None - except (AttributeError, TypeError) as e: - LOG.info("get_remotelogging_resource error {}".format(e), - extra=self.log_extra) - os_client.delete_region_clients(self.region_name, clear_token=True) - return None - except Exception as e: - LOG.exception(e) - return None + return os_client.sysinv_client.get_remotelogging() def get_certificates_resources(self, os_client): - try: - return os_client.sysinv_client.get_certificates() - except (keystone_exceptions.connection.ConnectTimeout, - keystone_exceptions.ConnectFailure) as e: - LOG.info("get_certificates: subcloud {} is not reachable [{}]" - .format(self.subcloud_engine.subcloud.region_name, - str(e)), extra=self.log_extra) - # None will force skip of audit - os_client.delete_region_clients(self.region_name) - return None - except (AttributeError, TypeError) as e: - LOG.info("get_certificates_resources error {}".format(e), - extra=self.log_extra) - os_client.delete_region_clients(self.region_name, clear_token=True) - return None - except Exception as e: - LOG.exception(e) - return None + return os_client.sysinv_client.get_certificates() def get_user_resource(self, os_client): - try: - iuser = os_client.sysinv_client.get_user() - return iuser - except (keystone_exceptions.connection.ConnectTimeout, - keystone_exceptions.ConnectFailure) as e: - LOG.info("get_user: subcloud {} is not reachable [{}]" - .format(self.subcloud_engine.subcloud.region_name, - str(e)), extra=self.log_extra) - # None will force skip of audit - os_client.delete_region_clients(self.region_name) - return None - except (AttributeError, TypeError) as e: - LOG.info("get_user_resources error {}".format(e), - extra=self.log_extra) - os_client.delete_region_clients(self.region_name, clear_token=True) - return None - except Exception as e: - LOG.exception(e) - return None + return os_client.sysinv_client.get_user() def get_fernet_resources(self, os_client): - try: - keys = os_client.sysinv_client.get_fernet_keys() - return FernetKeyManager.to_resource_info(keys) - except (keystone_exceptions.connection.ConnectTimeout, - keystone_exceptions.ConnectFailure) as e: - LOG.info("get_fernet_resource: subcloud {} is not reachable [{}]" - .format(self.subcloud_engine.subcloud.region_name, - str(e)), extra=self.log_extra) - os_client.delete_region_clients(self.region_name) - # None will force skip of audit - return None - except (AttributeError, TypeError) as e: - LOG.info("get_fernet_resource error {}".format(e), - extra=self.log_extra) - os_client.delete_region_clients(self.region_name, clear_token=True) - return None - except Exception as e: - LOG.exception(e) - return None + keys = os_client.sysinv_client.get_fernet_keys() + return FernetKeyManager.to_resource_info(keys) def get_resource_id(self, resource_type, resource): if resource_type == consts.RESOURCE_TYPE_SYSINV_SNMP_COMM: @@ -852,7 +684,7 @@ class SysinvSyncThread(SyncThread): resource_type, resource.uuid)) return resource.uuid else: - LOG.info("get_resource_id {} NO uuid resource_type={}".format( + LOG.info("get_resource_id NO uuid resource_type={}".format( resource_type)) return self.RESOURCE_UUID_NULL # master_id cannot be None diff --git a/distributedcloud/dcorch/engine/sync_thread.py b/distributedcloud/dcorch/engine/sync_thread.py index 925017e6c..6b09e6ea4 100644 --- a/distributedcloud/dcorch/engine/sync_thread.py +++ b/distributedcloud/dcorch/engine/sync_thread.py @@ -140,12 +140,12 @@ class SyncThread(object): # keystone client self.ks_client = keystoneclient.Client( session=self.admin_session, - region_name=consts.VIRTUAL_MASTER_CLOUD) + region_name=consts.CLOUD_0) # dcdbsync client self.dbs_client = dbsyncclient.Client( endpoint_type=consts.DBS_ENDPOINT_INTERNAL, session=self.admin_session, - region_name=consts.VIRTUAL_MASTER_CLOUD) + region_name=consts.CLOUD_0) def initialize_sc_clients(self): # base implementation of initializing the subcloud specific @@ -485,6 +485,11 @@ class SyncThread(object): LOG.debug("{}: done sync audit".format(self.audit_thread.name), extra=self.log_extra) + self.post_audit() + + def post_audit(self): + # The specific SyncThread subclasses may perform post audit actions + return def audit_find_missing(self, resource_type, m_resources, db_resources, sc_resources,