Fernet key synchronization

This update contains the following changes for Distributed
Cloud Fernet Key Synching & Management:

1.Disable key rotation cron job for distributed cloud
2.Add a fernet key repo config option in puppet sysinv
3.Add fernet repo sysinv APIs for create/update/retrieve keys
4.Add a fernet operator to create/update/retrieve the keys

Story: 2002842
Task: 22786

Change-Id: Ia14caeef067fa481e3a4159c1658289250632779
Signed-off-by: Tao Liu <tao.liu@windriver.com>

puppet-manifests/src/modules/openstack/manifests/keystone.pp

@@ -110,19 +110,21 @@ class openstack::keystone (
 
     include ::keystone::ldap
 
-    # Set up cron job that will rotate fernet keys. This is done every month on
-    # the first day of the month at 00:25 by default. The cron job only runs on
-    # the active controller.
-    cron { 'keystone-fernet-keys-rotater':
-      ensure      => 'present',
-      command     => '/usr/bin/keystone-fernet-keys-rotate-active',
-      environment => 'PATH=/bin:/usr/bin:/usr/sbin',
-      minute      => $fernet_keys_rotation_minute,
-      hour        => $fernet_keys_rotation_hour,
-      month       => $fernet_keys_rotation_month,
-      monthday    => $fernet_keys_rotation_monthday,
-      weekday     => $fernet_keys_rotation_weekday,
-      user        => 'root',
+    if $::platform::params::distributed_cloud_role == undef {
+      # Set up cron job that will rotate fernet keys. This is done every month on
+      # the first day of the month at 00:25 by default. The cron job runs on both
+      # controllers, but the script will only take action on the active controller.
+      cron { 'keystone-fernet-keys-rotater':
+        ensure      => 'present',
+        command     => '/usr/bin/keystone-fernet-keys-rotate-active',
+        environment => 'PATH=/bin:/usr/bin:/usr/sbin',
+        minute      => $fernet_keys_rotation_minute,
+        hour        => $fernet_keys_rotation_hour,
+        month       => $fernet_keys_rotation_month,
+        monthday    => $fernet_keys_rotation_monthday,
+        weekday     => $fernet_keys_rotation_weekday,
+        user        => 'root',
+      }
     }
   } else {
       class { '::keystone':

puppet-manifests/src/modules/platform/manifests/sysinv.pp

@@ -12,10 +12,13 @@ class platform::sysinv
 
   include ::platform::params
   include ::platform::amqp::params
+  include ::platform::drbd::cgcs::params
 
   # sysinv-agent is started on all hosts
   include ::sysinv::agent
 
+  $keystone_key_repo_path = "${::platform::drbd::cgcs::params::mountpoint}/keystone"
+
   group { 'sysinv':
     ensure => 'present',
     gid    => '168',
@@ -47,6 +50,7 @@ class platform::sysinv
     rabbit_userid => $::platform::amqp::params::auth_user,
     rabbit_password => $::platform::amqp::params::auth_password,
     fm_catalog_info => $fm_catalog_info,
+    fernet_key_repository => "$keystone_key_repo_path/fernet-keys",
   }
 
   # Note: The log format strings are prefixed with "sysinv" because it is

puppet-modules-wrs/puppet-sysinv/src/sysinv/manifests/init.pp

@@ -71,6 +71,7 @@ class sysinv (
   $nova_region_name            = 'RegionOne',
   $magnum_region_name          = 'RegionOne',
   $fm_catalog_info             = undef,
+  $fernet_key_repository       = undef,
 ) {
 
   include sysinv::params
@@ -205,9 +206,10 @@ class sysinv (
   sysinv_config {
     'fm/catalog_info':    value => $fm_catalog_info;
     'fm/os_region_name':  value => $region_name;
+    'fernet_repo/key_repository':  value => $fernet_key_repository;
   }
 
- sysinv_api_paste_ini {
+  sysinv_api_paste_ini {
     'filter:authtoken/region_name': value => $region_name;
- }
+  }
 }

sysinv/cgts-client/cgts-client/cgtsclient/v1/client.py

@@ -26,6 +26,7 @@ from cgtsclient.v1 import cluster
 from cgtsclient.v1 import controller_fs
 from cgtsclient.v1 import drbdconfig
 from cgtsclient.v1 import ethernetport
+from cgtsclient.v1 import fernet
 from cgtsclient.v1 import firewallrules
 from cgtsclient.v1 import health
 from cgtsclient.v1 import helm
@@ -150,3 +151,4 @@ class Client(http.HTTPClient):
             storage_ceph_external.StorageCephExternalManager(self)
         self.helm = helm.HelmManager(self)
         self.label = label.KubernetesLabelManager(self)
+        self.fernet = fernet.FernetManager(self)

sysinv/cgts-client/cgts-client/cgtsclient/v1/fernet.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+#
+# Copyright (c) 2018 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+from cgtsclient.common import base
+
+
+class FernetKey(base.Resource):
+    def __repr__(self):
+        return "<keys %s>" % self._info
+
+
+class FernetManager(base.Manager):
+    resource_class = FernetKey
+
+    @staticmethod
+    def _path(id=None):
+        return '/v1/fernet_repo/%s' % id if id else '/v1/fernet_repo'
+
+    def list(self):
+        return self._list(self._path(), "keys")
+
+    def get(self, id):
+        try:
+            return self._list(self._path(id))[0]
+        except IndexError:
+            return None
+
+    def create(self, data):
+        return self._create(self._path(), data)
+
+    def put(self, patch, id=None):
+        return self._update(self._path(id), patch, http_method='PUT')

sysinv/sysinv/sysinv/sysinv/api/controllers/v1/__init__.py

@@ -32,6 +32,7 @@ from sysinv.api.controllers.v1 import disk
 from sysinv.api.controllers.v1 import dns
 from sysinv.api.controllers.v1 import drbdconfig
 from sysinv.api.controllers.v1 import ethernet_port
+from sysinv.api.controllers.v1 import fernet_repo
 from sysinv.api.controllers.v1 import firewallrules
 from sysinv.api.controllers.v1 import health
 from sysinv.api.controllers.v1 import helm_charts
@@ -233,6 +234,9 @@ class V1(base.APIBase):
     label = [link.Link]
     "Links to the label resource "
 
+    fernet_repo = [link.Link]
+    "Links to the fernet repo resource"
+
     @classmethod
     def convert(self):
         v1 = V1()
@@ -726,6 +730,14 @@ class V1(base.APIBase):
                                          pecan.request.host_url,
                                          'labels', '',
                                          bookmark=True)]
+
+        v1.fernet_repo = [link.Link.make_link('self', pecan.request.host_url,
+                                              'fernet_repo', ''),
+                          link.Link.make_link('bookmark',
+                                              pecan.request.host_url,
+                                              'fernet_repo', '',
+                                              bookmark=True)
+                          ]
         return v1
 
 
@@ -790,6 +802,7 @@ class Controller(rest.RestController):
     firewallrules = firewallrules.FirewallRulesController()
     license = license.LicenseController()
     labels = label.LabelController()
+    fernet_repo = fernet_repo.FernetKeyController()
 
     @wsme_pecan.wsexpose(V1)
     def get(self):

sysinv/sysinv/sysinv/sysinv/api/controllers/v1/fernet_repo.py

@@ -0,0 +1,137 @@
+#
+# Copyright (c) 2018 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+
+import pecan
+import wsme
+import wsmeext.pecan as wsme_pecan
+from six.moves import http_client
+
+from pecan import rest
+from sysinv.api.controllers.v1 import base
+from sysinv.api.controllers.v1 import collection
+from sysinv.api.controllers.v1 import link
+from sysinv.api.controllers.v1 import types
+from sysinv.openstack.common import log
+from sysinv.common import utils as cutils
+from sysinv.openstack.common.gettextutils import _
+from wsme import types as wtypes
+
+LOG = log.getLogger(__name__)
+
+LOCK_NAME = 'FernetKeyController'
+
+
+class FernetKey(base.APIBase):
+    """API representation of a Fernet Key.
+
+    This class enforces type checking and value constraints, and converts
+    between the internal object model and the API representation of
+    a Fernet Key.
+    """
+
+    uuid = types.uuid
+    "The UUID of the fernet key"
+
+    id = int
+    "The id of the fernet key"
+
+    key = wtypes.text
+    "Represents the fernet key"
+
+    links = [link.Link]
+    "A list containing a self link and associated key links"
+
+    def __init__(self, **kwargs):
+        self.fields = ["id", "key"]
+        for k in self.fields:
+            setattr(self, k, kwargs.get(k))
+
+    @classmethod
+    def from_dict(cls, obj_dict):
+        """Convert a dictionary to an API object."""
+        return cls(**obj_dict)
+
+    @classmethod
+    def convert_with_links(cls, rpc_fernet, expand=True):
+        repo = FernetKey.from_dict(rpc_fernet)
+        return repo
+
+
+class FernetKeyCollection(collection.Collection):
+    """API representation of a collection of fernet key."""
+
+    keys = [FernetKey]
+    "A list containing fernet key objects"
+
+    def __init__(self, **kwargs):
+        self._type = 'keys'
+
+    @classmethod
+    def convert_with_links(cls, keys, **kwargs):
+        keys = sorted(keys, key=lambda x: x['id'])
+        collection = FernetKeyCollection()
+        collection.keys = [FernetKey.convert_with_links(k)
+                           for k in keys]
+        return collection
+
+
+class FernetKeyController(rest.RestController):
+    """REST controller for Fernet Keys."""
+
+    def __init__(self):
+        self._api_token = None
+
+    @wsme_pecan.wsexpose(FernetKeyCollection)
+    def get_all(self):
+        """Provides all keys under the Fernet Repo"""
+        try:
+            output = pecan.request.rpcapi.get_fernet_keys(
+                pecan.request.context)
+        except Exception as e:
+            LOG.exception(e)
+            raise wsme.exc.ClientSideError(_(
+                "Unable to perform fernet key query."))
+
+        return FernetKeyCollection.convert_with_links(output)
+
+    @wsme_pecan.wsexpose(FernetKey, wtypes.text)
+    def get_one(self, key):
+        """Provide a key under the Fernet Repo"""
+        try:
+            success, output = pecan.request.rpcapi.get_fernet_keys(
+                pecan.request.context, key_id=int(key))
+        except Exception as e:
+            LOG.exception(e)
+            raise wsme.exc.ClientSideError(_(
+                "Unable to perform fernet key query."))
+        return FernetKey.convert_with_links(output[0])
+
+    @cutils.synchronized(LOCK_NAME)
+    @wsme_pecan.wsexpose(None, body=[FernetKey],
+                         status_code=http_client.CREATED)
+    def post(self, keys):
+        key_list = [k.as_dict() for k in keys]
+        try:
+            pecan.request.rpcapi.update_fernet_keys(pecan.request.context,
+                                                    key_list)
+        except Exception as e:
+            LOG.exception(e)
+            raise wsme.exc.ClientSideError(_(
+                "Unable to create fernet keys."))
+
+    @cutils.synchronized(LOCK_NAME)
+    @wsme_pecan.wsexpose(None, body=[FernetKey],
+                         status_code=http_client.ACCEPTED)
+    def put(self, keys):
+        key_list = [k.as_dict() for k in keys]
+        try:
+            pecan.request.rpcapi.update_fernet_keys(pecan.request.context,
+                                                    key_list)
+        except Exception as e:
+            LOG.exception(e)
+            raise wsme.exc.ClientSideError(_(
+                "Unable to update fernet keys."))

sysinv/sysinv/sysinv/sysinv/common/fernet.py

@@ -0,0 +1,170 @@
+#
+# Copyright (c) 2018 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+import os
+from grp import getgrnam
+from pwd import getpwnam
+
+from oslo_config import cfg
+from sysinv.common import exception
+from sysinv.openstack.common import log as logging
+from sysinv.openstack.common.gettextutils import _
+
+CONF = cfg.CONF
+
+LOG = logging.getLogger(__name__)
+
+fernet_group = cfg.OptGroup(
+    'fernet_repo',
+    title='fernet repo Options',
+    help="Configuration options for the fernet key repository")
+
+fernet_opts = [
+    cfg.StrOpt('key_repository',
+               default='/etc/keystone/fernet-keys',
+               help="The fernet key repository."),
+]
+
+CONF.register_group(fernet_group)
+CONF.register_opts(fernet_opts, group=fernet_group)
+
+KEYSTONE_USER = 'keystone'
+KEYSTONE_GROUP = 'keystone'
+
+
+class FernetOperator(object):
+    """Class to encapsulate Fernet Key operations for System Inventory"""
+
+    def __init__(self, keystone_user_id=None, keystone_group_id=None):
+        self.key_repository = CONF.fernet_repo.key_repository
+        self.keystone_user_id = keystone_user_id
+        self.keystone_group_id = keystone_group_id
+
+    def _set_user_group(self):
+        if self.keystone_user_id is None:
+            self.keystone_user_id = getpwnam(KEYSTONE_USER).pw_uid
+
+        if self.keystone_group_id is None:
+            self.keystone_group_id = getgrnam(KEYSTONE_GROUP).gr_gid
+
+    def _check_key_directory(self):
+        """Check if the key directory exists and attempt to create it if it
+           doesn't.
+        """
+        if not os.access(self.key_repository, os.F_OK):
+            LOG.info(_("key_repository:(%s) does not exist; attempting to "
+                     "create it") % self.key_repository)
+            try:
+                os.makedirs(self.key_repository, 0o700)
+            except OSError:
+                LOG.error(_("Failed to create key_repository"))
+                return False
+
+            self._set_user_group()
+            os.chown(self.key_repository, self.keystone_user_id,
+                     self.keystone_group_id)
+        return True
+
+    def _create_key_file(self, id, key):
+        """Create a tmp key file."""
+
+        self._set_user_group()
+        old_umask = os.umask(0o177)
+        old_egid = os.getegid()
+        old_euid = os.geteuid()
+        os.setegid(self.keystone_group_id)
+        os.seteuid(self.keystone_user_id)
+
+        temp_key_file = os.path.join(self.key_repository, str(id) + '.tmp')
+        real_key_file = os.path.join(self.key_repository, str(id))
+        create = False
+        try:
+            with open(temp_key_file, 'w') as f:
+                f.write(key)
+                f.flush()
+                create = True
+        except IOError:
+            LOG.error(_('Failed to create new temporary key: %s',
+                        temp_key_file))
+            raise
+        finally:
+            # restore the umask, user and group identifiers
+            os.umask(old_umask)
+            os.seteuid(old_euid)
+            os.setegid(old_egid)
+            if not create and os.access(temp_key_file, os.F_OK):
+                os.remove(temp_key_file)
+                return False
+
+        os.rename(temp_key_file, real_key_file)
+        LOG.debug('Created a new key: %s', real_key_file)
+        return True
+
+    def _get_key_files(self):
+        # read the list of key files
+        key_files = dict()
+        for filename in os.listdir(self.key_repository):
+            path = os.path.join(self.key_repository, str(filename))
+            if os.path.isfile(path):
+                try:
+                    key_id = int(filename)
+                except ValueError:
+                    pass
+                else:
+                    key_files[key_id] = path
+        return key_files
+
+    def _validate_key_repository(self):
+        """Validate permissions on the key repository directory."""
+
+        # ensure current user has sufficient access to the key repository
+        is_valid = os.access(self.key_repository, os.R_OK)
+
+        if not is_valid:
+            LOG.error(_("Either (%s) key_repository does not exist or we "
+                      "don't have sufficient permission to access it." %
+                        self.key_repository))
+        return is_valid
+
+    def update_fernet_keys(self, new_keys):
+        new_key_ids = []
+
+        if not self._check_key_directory():
+            raise exception.SysinvException(_(
+                "Error checking key repository."))
+
+        try:
+            for key in new_keys:
+                self._create_key_file(key['id'], key['key'])
+                new_key_ids.append(key['id'])
+
+            # remove excess keys
+            key_files = self._get_key_files()
+            for key in key_files.keys():
+                if key not in new_key_ids:
+                    key_to_purge = key_files[key]
+                    LOG.info('Purge excess key: %s', key_to_purge)
+                    os.remove(key_to_purge)
+        except Exception as e:
+            msg = _("Failed to update fernet keys: %s") % e.message
+            LOG.exception(msg)
+            raise exception.SysinvException(msg)
+
+    def get_fernet_keys(self, key_id=None):
+        keys = []
+        if not self._validate_key_repository():
+            return keys
+
+        key_files = self._get_key_files()
+        for k, v in key_files.items():
+            key = dict()
+            key['id'] = k
+            with open(v, 'r') as key_file:
+                key['key'] = key_file.read()
+                keys.append(key)
+                if key_id is not None and key_id == k:
+                    break
+        return keys

sysinv/sysinv/sysinv/sysinv/conductor/manager.py

@@ -69,6 +69,7 @@ from sysinv.common import constants
 from sysinv.common import ceph as cceph
 from sysinv.common import exception
 from sysinv.common import fm
+from sysinv.common import fernet
 from sysinv.common import health
 from sysinv.common import kubernetes
 from sysinv.common import retrying
@@ -154,6 +155,7 @@ class ConductorManager(service.PeriodicService):
         self._ceph_api = ceph.CephWrapper(
             endpoint='http://localhost:5001/api/v0.1/')
         self._kube = None
+        self._fernet = None
 
         self._openstack = None
         self._api_token = None
@@ -180,6 +182,7 @@ class ConductorManager(service.PeriodicService):
         self._ceph = iceph.CephOperator(self.dbapi)
         self._helm = helm.HelmOperator(self.dbapi)
         self._kube = kubernetes.KubeOperator(self.dbapi)
+        self._fernet = fernet.FernetOperator()
 
         # create /var/run/sysinv if required. On DOR, the manifests
         # may not run to create this volatile directory.
@@ -10332,3 +10335,21 @@ class ConductorManager(service.PeriodicService):
 
         rpcapi = agent_rpcapi.AgentAPI()
         rpcapi.update_host_memory(context, host.uuid)
+
+    def update_fernet_keys(self, context, keys):
+        """Update the fernet repo with the new keys.
+
+          :param context: request context.
+          :param keys: a list of keys
+          :returns: nothing
+          """
+        self._fernet.update_fernet_keys(keys)
+
+    def get_fernet_keys(self, context, key_id=None):
+        """Get the keys from the fernet repo.
+
+          :param context: request context.
+          :param key_id: Optionally, it can be used to retrieve a specified key
+          :returns: a list of keys
+          """
+        return self._fernet.get_fernet_keys(key_id)

sysinv/sysinv/sysinv/sysinv/conductor/rpcapi.py

@@ -1708,3 +1708,22 @@ class ConductorAPI(sysinv.openstack.common.rpc.proxy.RpcProxy):
                  " host memory update request to conductor")
         return self.cast(context, self.make_msg('update_host_memory',
                                                 host_uuid=host_uuid))
+
+    def update_fernet_keys(self, context, keys):
+        """Synchronously, have the conductor update fernet keys.
+
+        :param context: request context.
+        :param keys: a list of fernet keys
+        """
+        return self.call(context, self.make_msg('update_fernet_keys',
+                                                keys=keys))
+
+    def get_fernet_keys(self, context, key_id=None):
+        """Synchronously, have the conductor to retrieve fernet keys.
+
+        :param context: request context.
+        :param key_id: (optional)
+        :returns: a list of fernet keys.
+        """
+        return self.call(context, self.make_msg('get_fernet_keys',
+                                                key_id=key_id))