diff --git a/dcorch/common/config.py b/dcorch/common/config.py index df69e4620..f32a8461d 100644 --- a/dcorch/common/config.py +++ b/dcorch/common/config.py @@ -197,6 +197,12 @@ snmp_server_opts = [ help='interval of periodic updates in seconds') ] +fernet_opts = [ + cfg.IntOpt('key_rotation_interval', + default=168, + help='Hours between running fernet key rotation tasks.') +] + scheduler_opt_group = cfg.OptGroup('scheduler', title='Scheduler options for periodic job') # The group stores DC Orchestrator global limit for all the projects. @@ -212,6 +218,9 @@ cache_opt_group = cfg.OptGroup(name='cache', snmp_opt_group = cfg.OptGroup(name='snmp', title='SNMP Options') +fernet_opt_group = cfg.OptGroup(name='fernet', + title='Fernet Options') + def list_opts(): yield default_quota_group.name, nova_quotas @@ -221,6 +230,7 @@ def list_opts(): yield scheduler_opt_group.name, scheduler_opts yield pecan_group.name, pecan_opts yield snmp_opt_group.name, snmp_server_opts + yield fernet_opt_group.name, fernet_opts yield None, global_opts yield None, common_opts diff --git a/dcorch/common/consts.py b/dcorch/common/consts.py index 902b292eb..63a3edab7 100644 --- a/dcorch/common/consts.py +++ b/dcorch/common/consts.py @@ -90,6 +90,7 @@ RESOURCE_TYPE_SYSINV_REMOTE_LOGGING = "remotelogging" RESOURCE_TYPE_SYSINV_SNMP_COMM = "icommunity" RESOURCE_TYPE_SYSINV_SNMP_TRAPDEST = "itrapdest" RESOURCE_TYPE_SYSINV_USER = "iuser" +RESOURCE_TYPE_SYSINV_FERNET_REPO = "fernet_repo" # Compute Resources RESOURCE_TYPE_COMPUTE_FLAVOR = "flavor" @@ -178,3 +179,5 @@ ACTION_EXTRASPECS_DELETE = "extra_specs_delete" ALARM_OK_STATUS = "OK" ALARM_DEGRADED_STATUS = "degraded" ALARM_CRITICAL_STATUS = "critical" + +SECONDS_IN_HOUR = 3600 diff --git a/dcorch/drivers/openstack/sysinv_v1.py b/dcorch/drivers/openstack/sysinv_v1.py index a0f1c5cf4..76192fc84 100644 --- a/dcorch/drivers/openstack/sysinv_v1.py +++ b/dcorch/drivers/openstack/sysinv_v1.py @@ -708,3 +708,50 @@ class SysinvClient(base.DriverBase): raise exceptions.SyncRequestFailedRetry() return iuser + + def create_fernet_repo(self, key_list): + """Add the fernet keys for this region + + :param: key list payload + :return: Nothing + """ + + # Example key_list: + # [{"id": 0, "key": "GgDAOfmyr19u0hXdm5r_zMgaMLjglVFpp5qn_N4GBJQ="}, + # {"id": 1, "key": "7WfL_z54p67gWAkOmQhLA9P0ZygsbbJcKgff0uh28O8="}, + # {"id": 2, "key": ""5gsUQeOZ2FzZP58DN32u8pRKRgAludrjmrZFJSOHOw0="}] + LOG.info("create_fernet_repo driver region={} " + "fernet_repo_list={}".format(self.region_name, key_list)) + try: + self.client.fernet.create(key_list) + except Exception as e: + LOG.error("create_fernet_repo exception={}".format(e)) + raise exceptions.SyncRequestFailedRetry() + + def update_fernet_repo(self, key_list): + """Update the fernet keys for this region + + :param: key list payload + :return: Nothing + """ + LOG.info("update_fernet_repo driver region={} " + "fernet_repo_list={}".format(self.region_name, key_list)) + try: + self.client.fernet.put(key_list) + except Exception as e: + LOG.error("update_fernet_repo exception={}".format(e)) + raise exceptions.SyncRequestFailedRetry() + + def get_fernet_keys(self): + """Retrieve the fernet keys for this region + + :return: a list of fernet keys + """ + + try: + keys = self.client.fernet.list() + except Exception as e: + LOG.error("get_fernet_keys exception={}".format(e)) + raise exceptions.SyncRequestFailedRetry() + + return keys diff --git a/dcorch/engine/fernet_key_manager.py b/dcorch/engine/fernet_key_manager.py new file mode 100644 index 000000000..a35ebdc5d --- /dev/null +++ b/dcorch/engine/fernet_key_manager.py @@ -0,0 +1,125 @@ +# Copyright 2018 Wind River +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess + +from oslo_config import cfg +from oslo_log import log as logging +from oslo_serialization import jsonutils + +from dcorch.common import consts +from dcorch.common import context +from dcorch.common import exceptions +from dcorch.common.i18n import _ +from dcorch.common import manager +from dcorch.common import utils +from dcorch.drivers.openstack import sdk_platform as sdk +from dcorch.objects import subcloud as subcloud_obj + + +FERNET_REPO_MASTER_ID = "keys" +KEY_ROTATE_CMD = "/usr/bin/keystone-fernet-keys-rotate-active" + +CONF = cfg.CONF + +LOG = logging.getLogger(__name__) + + +class FernetKeyManager(manager.Manager): + """Manages tasks related to fernet key management""" + + def __init__(self, gsm, *args, **kwargs): + LOG.debug(_('FernetKeyManager initialization...')) + + super(FernetKeyManager, self).__init__(service_name="fernet_manager", + *args, **kwargs) + self.gsm = gsm + self.context = context.get_admin_context() + self.endpoint_type = consts.ENDPOINT_TYPE_PLATFORM + self.resource_type = consts.RESOURCE_TYPE_SYSINV_FERNET_REPO + + @classmethod + def to_resource_info(cls, key_list): + return dict((getattr(key, 'id'), getattr(key, 'key')) + for key in key_list) + + @classmethod + def from_resource_info(cls, keys): + key_list = [dict(id=k, key=v) for k, v in keys.items()] + return key_list + + @classmethod + def get_resource_hash(cls, resource_info): + return hash(tuple(sorted(hash(x) for x in resource_info.items()))) + + def _schedule_work(self, operation_type, subcloud=None): + keys = self._get_master_keys() + if not keys: + LOG.info(_("No fernet keys returned from %s") % consts.CLOUD_0) + return + try: + resource_info = FernetKeyManager.to_resource_info(keys) + utils.enqueue_work(self.context, + self.endpoint_type, + self.resource_type, + FERNET_REPO_MASTER_ID, + operation_type, + resource_info=jsonutils.dumps(resource_info), + subcloud=subcloud) + # wake up sync thread + if self.gsm: + self.gsm.sync_request(self.context, self.endpoint_type) + except Exception as e: + LOG.error(_("Exception in schedule_work: %s") % e.message) + + @staticmethod + def _get_master_keys(): + """get the keys from the local fernet key repo""" + keys = [] + try: + os_client = sdk.OpenStackDriver(consts.CLOUD_0) + keys = os_client.sysinv_client.get_fernet_keys() + except (exceptions.ConnectionRefused, exceptions.NotAuthorized, + exceptions.TimeOut): + LOG.info(_("Retrieving the fernet keys from %s timeout") % + consts.CLOUD_0) + except Exception as e: + LOG.info(_("Fail to retrieve the master fernet keys: %s") % + e.message) + return keys + + def rotate_fernet_keys(self): + """Rotate fernet keys.""" + + with open(os.devnull, "w") as fnull: + try: + subprocess.check_call(KEY_ROTATE_CMD, + stdout=fnull, + stderr=fnull) + except subprocess.CalledProcessError: + msg = _("Failed to rotate the keys") + LOG.exception(msg) + raise exceptions.InternalError(msg) + + self._schedule_work(consts.OPERATION_TYPE_PUT) + + def distribute_keys(self, ctxt, subcloud_name): + subclouds = subcloud_obj.SubcloudList.get_all(ctxt) + for sc in subclouds: + if sc.region_name == subcloud_name: + subcloud = sc + self._schedule_work(consts.OPERATION_TYPE_CREATE, subcloud) + break diff --git a/dcorch/engine/service.py b/dcorch/engine/service.py index a0fd5489e..8250697a2 100644 --- a/dcorch/engine/service.py +++ b/dcorch/engine/service.py @@ -25,6 +25,7 @@ from dcorch.common import exceptions from dcorch.common.i18n import _ from dcorch.common import messaging as rpc_messaging from dcorch.engine.alarm_aggregate_manager import AlarmAggregateManager +from dcorch.engine.fernet_key_manager import FernetKeyManager from dcorch.engine.generic_sync_manager import GenericSyncManager from dcorch.engine.quota_manager import QuotaManager from dcorch.engine import scheduler @@ -77,6 +78,7 @@ class EngineService(service.Service): self.qm = None self.gsm = None self.aam = None + self.fkm = None def init_tgm(self): self.TG = scheduler.ThreadGroupManager() @@ -92,12 +94,16 @@ class EngineService(service.Service): def init_aam(self): self.aam = AlarmAggregateManager() + def init_fkm(self): + self.fkm = FernetKeyManager(self.gsm) + def start(self): self.engine_id = uuidutils.generate_uuid() self.init_tgm() self.init_qm() self.init_gsm() self.init_aam() + self.init_fkm() target = oslo_messaging.Target(version=self.rpc_api_version, server=self.host, topic=self.topic) @@ -118,6 +124,11 @@ class EngineService(service.Service): self.TG.add_timer(self.periodic_interval, self.periodic_sync_audit, initial_delay=self.periodic_interval / 2) + self.TG.add_timer(CONF.fernet.key_rotation_interval * + consts.SECONDS_IN_HOUR, + self.periodic_key_rotation, + initial_delay=(CONF.fernet.key_rotation_interval + * consts.SECONDS_IN_HOUR)) def service_registry_report(self): ctx = context.get_admin_context() @@ -182,6 +193,7 @@ class EngineService(service.Service): # keep equivalent functionality for now if (management_state == dcm_consts.MANAGEMENT_MANAGED) and \ (availability_status == dcm_consts.AVAILABILITY_ONLINE): + self.fkm.distribute_keys(ctxt, subcloud_name) self.aam.enable_snmp(ctxt, subcloud_name) self.gsm.enable_subcloud(ctxt, subcloud_name) else: @@ -233,3 +245,8 @@ class EngineService(service.Service): # Terminate the engine process LOG.info("All threads were gone, terminating engine") super(EngineService, self).stop() + + def periodic_key_rotation(self): + """Periodic key rotation.""" + LOG.info("Periodic key rotation started at: %s", time.strftime("%c")) + return self.fkm.rotate_fernet_keys() diff --git a/dcorch/engine/sync_services/sysinv.py b/dcorch/engine/sync_services/sysinv.py index 5224fe4a8..060afc54e 100644 --- a/dcorch/engine/sync_services/sysinv.py +++ b/dcorch/engine/sync_services/sysinv.py @@ -22,7 +22,8 @@ from oslo_serialization import jsonutils from dcorch.common import consts from dcorch.common import exceptions from dcorch.drivers.openstack import sdk_platform as sdk - +from dcorch.engine.fernet_key_manager import FERNET_REPO_MASTER_ID +from dcorch.engine.fernet_key_manager import FernetKeyManager from dcorch.engine.sync_thread import AUDIT_RESOURCE_MISSING from dcorch.engine.sync_thread import SyncThread @@ -37,13 +38,15 @@ class SysinvSyncThread(SyncThread): consts.RESOURCE_TYPE_SYSINV_PTP, consts.RESOURCE_TYPE_SYSINV_REMOTE_LOGGING, consts.RESOURCE_TYPE_SYSINV_USER, + consts.RESOURCE_TYPE_SYSINV_FERNET_REPO ] SYSINV_ADD_DELETE_RESOURCES = [consts.RESOURCE_TYPE_SYSINV_SNMP_COMM, consts.RESOURCE_TYPE_SYSINV_SNMP_TRAPDEST] SYSINV_CREATE_RESOURCES = [consts.RESOURCE_TYPE_SYSINV_FIREWALL_RULES, - consts.RESOURCE_TYPE_SYSINV_CERTIFICATE] + consts.RESOURCE_TYPE_SYSINV_CERTIFICATE, + consts.RESOURCE_TYPE_SYSINV_FERNET_REPO] FIREWALL_SIG_NULL = 'NoCustomFirewallRules' CERTIFICATE_SIG_NULL = 'NoCertificate' @@ -68,6 +71,8 @@ class SysinvSyncThread(SyncThread): consts.RESOURCE_TYPE_SYSINV_CERTIFICATE: self.sync_certificate, consts.RESOURCE_TYPE_SYSINV_USER: self.sync_user, + consts.RESOURCE_TYPE_SYSINV_FERNET_REPO: + self.sync_fernet_resources } self.region_name = self.subcloud_engine.subcloud.region_name self.log_extra = {"instance": "{}/{}: ".format( @@ -83,6 +88,7 @@ class SysinvSyncThread(SyncThread): consts.RESOURCE_TYPE_SYSINV_SNMP_COMM, consts.RESOURCE_TYPE_SYSINV_SNMP_TRAPDEST, consts.RESOURCE_TYPE_SYSINV_USER, + consts.RESOURCE_TYPE_SYSINV_FERNET_REPO, ] # initialize the master clients @@ -764,6 +770,90 @@ class SysinvSyncThread(SyncThread): .format(rsrc.id, subcloud_rsrc_id, passwd_hash), extra=self.log_extra) + def sync_fernet_resources(self, request, rsrc): + switcher = { + consts.OPERATION_TYPE_PUT: self.update_fernet_repo, + consts.OPERATION_TYPE_PATCH: self.update_fernet_repo, + consts.OPERATION_TYPE_CREATE: self.create_fernet_repo, + } + + func = switcher[request.orch_job.operation_type] + try: + func(request, rsrc) + except (keystone_exceptions.connection.ConnectTimeout, + keystone_exceptions.ConnectFailure) as e: + LOG.info("sync_fernet_resources: subcloud {} is not reachable [{}]" + .format(self.subcloud_engine.subcloud.region_name, + str(e)), extra=self.log_extra) + raise exceptions.SyncRequestTimeout + except Exception as e: + LOG.exception(e) + raise exceptions.SyncRequestFailedRetry + + def create_fernet_repo(self, request, rsrc): + LOG.info("create_fernet_repo region {} resource_info={}".format( + self.subcloud_engine.subcloud.region_name, + request.orch_job.resource_info), + extra=self.log_extra) + resource_info = jsonutils.loads(request.orch_job.resource_info) + + s_os_client = sdk.OpenStackDriver(self.region_name) + try: + s_os_client.sysinv_client.create_fernet_repo( + FernetKeyManager.from_resource_info(resource_info)) + # Ensure subcloud resource is persisted to the DB for later + subcloud_rsrc_id = self.persist_db_subcloud_resource( + rsrc.id, rsrc.master_id) + except (exceptions.ConnectionRefused, exceptions.NotAuthorized, + exceptions.TimeOut): + LOG.info("create_fernet_repo Timeout,{}:{}".format( + rsrc.id, subcloud_rsrc_id)) + s_os_client.delete_region_clients(self.region_name, + clear_token=True) + raise exceptions.SyncRequestTimeout + except (AttributeError, TypeError) as e: + LOG.info("create_fernet_repo error {}".format(e), + extra=self.log_extra) + s_os_client.delete_region_clients(self.region_name, + clear_token=True) + raise exceptions.SyncRequestFailedRetry + + LOG.info("fernet_repo {} {} {} created".format(rsrc.id, + subcloud_rsrc_id, resource_info), + extra=self.log_extra) + + def update_fernet_repo(self, request, rsrc): + LOG.info("update_fernet_repo region {} resource_info={}".format( + self.subcloud_engine.subcloud.region_name, + request.orch_job.resource_info), + extra=self.log_extra) + resource_info = jsonutils.loads(request.orch_job.resource_info) + + s_os_client = sdk.OpenStackDriver(self.region_name) + try: + s_os_client.sysinv_client.update_fernet_repo( + FernetKeyManager.from_resource_info(resource_info)) + # Ensure subcloud resource is persisted to the DB for later + subcloud_rsrc_id = self.persist_db_subcloud_resource( + rsrc.id, rsrc.master_id) + except (exceptions.ConnectionRefused, exceptions.NotAuthorized, + exceptions.TimeOut): + LOG.info("update_fernet_repo Timeout,{}:{}".format( + rsrc.id, subcloud_rsrc_id)) + s_os_client.delete_region_clients(self.region_name, + clear_token=True) + raise exceptions.SyncRequestTimeout + except (AttributeError, TypeError) as e: + LOG.info("update_fernet_repo error {}".format(e), + extra=self.log_extra) + s_os_client.delete_region_clients(self.region_name, + clear_token=True) + raise exceptions.SyncRequestFailedRetry + + LOG.info("fernet_repo {} {} {} update".format(rsrc.id, + subcloud_rsrc_id, resource_info), + extra=self.log_extra) + # SysInv Audit Related def get_master_resources(self, resource_type): os_client = sdk.OpenStackDriver(consts.CLOUD_0) @@ -785,6 +875,8 @@ class SysinvSyncThread(SyncThread): return self.get_certificates_resources(os_client) elif resource_type == consts.RESOURCE_TYPE_SYSINV_USER: return [self.get_user_resource(os_client)] + elif resource_type == consts.RESOURCE_TYPE_SYSINV_FERNET_REPO: + return [self.get_fernet_resources(os_client)] else: LOG.error("Wrong resource type {}".format(resource_type), extra=self.log_extra) @@ -810,6 +902,8 @@ class SysinvSyncThread(SyncThread): return self.get_certificates_resources(os_client) elif resource_type == consts.RESOURCE_TYPE_SYSINV_USER: return [self.get_user_resource(os_client)] + elif resource_type == consts.RESOURCE_TYPE_SYSINV_FERNET_REPO: + return [self.get_fernet_resources(os_client)] else: LOG.error("Wrong resource type {}".format(resource_type), extra=self.log_extra) @@ -1004,6 +1098,27 @@ class SysinvSyncThread(SyncThread): LOG.exception(e) return None + def get_fernet_resources(self, os_client): + try: + keys = os_client.sysinv_client.get_fernet_keys() + return FernetKeyManager.to_resource_info(keys) + except (keystone_exceptions.connection.ConnectTimeout, + keystone_exceptions.ConnectFailure) as e: + LOG.info("get_fernet_resource: subcloud {} is not reachable [{}]" + .format(self.subcloud_engine.subcloud.region_name, + str(e)), extra=self.log_extra) + os_client.delete_region_clients(self.region_name) + # None will force skip of audit + return None + except (AttributeError, TypeError) as e: + LOG.info("get_fernet_resource error {}".format(e), + extra=self.log_extra) + os_client.delete_region_clients(self.region_name, clear_token=True) + return None + except Exception as e: + LOG.exception(e) + return None + def get_resource_id(self, resource_type, resource): if resource_type == consts.RESOURCE_TYPE_SYSINV_SNMP_COMM: LOG.debug("get_resource_id for community {}".format(resource)) @@ -1047,6 +1162,10 @@ class SysinvSyncThread(SyncThread): else: LOG.error("no get_resource_id for certificate") return self.CERTIFICATE_SIG_NULL + elif resource_type == consts.RESOURCE_TYPE_SYSINV_FERNET_REPO: + LOG.info("get_resource_id {} resource={}".format( + resource_type, resource)) + return FERNET_REPO_MASTER_ID else: if hasattr(resource, 'uuid'): LOG.info("get_resource_id {} uuid={}".format( @@ -1155,6 +1274,15 @@ class SysinvSyncThread(SyncThread): same_user = False return same_user + def same_fernet_key(self, i1, i2): + LOG.info("same_fernet_repo i1={}, i2={}".format(i1, i2), + extra=self.log_extra) + same_fernet = True + if (FernetKeyManager.get_resource_hash(i1) != + FernetKeyManager.get_resource_hash(i2)): + same_fernet = False + return same_fernet + def same_resource(self, resource_type, m_resource, sc_resource): if resource_type == consts.RESOURCE_TYPE_SYSINV_DNS: return self.same_dns(m_resource, sc_resource) @@ -1172,8 +1300,10 @@ class SysinvSyncThread(SyncThread): return self.same_firewallrules(m_resource, sc_resource) elif resource_type == consts.RESOURCE_TYPE_SYSINV_CERTIFICATE: return self.same_certificate(m_resource, sc_resource) - if resource_type == consts.RESOURCE_TYPE_SYSINV_USER: + elif resource_type == consts.RESOURCE_TYPE_SYSINV_USER: return self.same_user(m_resource, sc_resource) + elif resource_type == consts.RESOURCE_TYPE_SYSINV_FERNET_REPO: + return self.same_fernet_key(m_resource, sc_resource) else: LOG.warn("same_resource() unexpected resource_type {}".format( resource_type), @@ -1279,6 +1409,10 @@ class SysinvSyncThread(SyncThread): resource_type, dumps), extra=self.log_extra) return dumps + elif resource_type == consts.RESOURCE_TYPE_SYSINV_FERNET_REPO: + LOG.info("get_resource_info resource_type={} resource={}".format( + resource_type, resource), extra=self.log_extra) + return jsonutils.dumps(resource) else: LOG.warn("get_resource_info unsupported resource {}".format( resource_type),