Support multiple CA certificates synchronization in DC

This update enhanced dcorch and sysinv API proxy to support multiple CA
certificates synchronization in DC system. The support utilizes the
updated sysinv certificate install API and the new certificate
uninstall API.

Closes-Bug: 1861438
Closes-Bug: 1860995
Depends-On: https://review.opendev.org/#/c/711538/
Change-Id: I407314b913ae5a56bb714b39484aea3263a41d19
Signed-off-by: Andy Ning <andy.ning@windriver.com>

distributedcloud/centos/build_srpm.data

@@ -1,4 +1,4 @@
 SRC_DIR="."
 COPY_LIST="$FILES_BASE/*"
 
-TIS_PATCH_VER=0
+TIS_PATCH_VER=1

distributedcloud/dcorch/api/proxy/apps/controller.py

@@ -400,10 +400,19 @@ class SysinvAPIController(APIController):
         # certificate need special processing
         p_resource_info = 'suppressed'
         if resource_type == consts.RESOURCE_TYPE_SYSINV_CERTIFICATE:
-            resource_info['payload'] = request_body
+            if operation_type == consts.OPERATION_TYPE_DELETE:
-            resource_info['content_type'] = environ.get('CONTENT_TYPE')
+                resource_id = json.loads(response.body)['signature']
-            resource = json.loads(response.body)[resource_type]
+                resource_ids = [resource_id]
-            resource_id = resource['signature']
+            else:
+                resource_info['payload'] = request_body
+                resource_info['content_type'] = environ.get('CONTENT_TYPE')
+                resource = json.loads(response.body)[resource_type]
+                # For ssl_ca cert, the resource in response is a list
+                if isinstance(resource, list):
+                    resource_ids = [str(res.get('signature'))
+                                    for res in resource]
+                else:
+                    resource_ids = [resource.get('signature')]
         else:
             if (operation_type == consts.OPERATION_TYPE_POST and
                     resource_type in self.RESOURCE_ID_MAP):
@@ -413,22 +422,23 @@ class SysinvAPIController(APIController):
                 resource_id = json.loads(request_body)[rid]
             else:
                 resource_id = self.get_resource_id_from_link(request_header)
-
+            resource_ids = [resource_id]
             if operation_type != consts.OPERATION_TYPE_DELETE:
                 resource_info['payload'] = json.loads(request_body)
             p_resource_info = resource_info
 
-        LOG.info("Resource id: (%s), type: (%s), info: (%s)",
+        for resource_id in resource_ids:
-                 resource_id, resource_type, p_resource_info)
+            LOG.info("Resource id: (%s), type: (%s), info: (%s)",
-        try:
+                     resource_id, resource_type, p_resource_info)
-            utils.enqueue_work(self.ctxt,
+            try:
-                               self.ENDPOINT_TYPE,
+                utils.enqueue_work(self.ctxt,
-                               resource_type,
+                                   self.ENDPOINT_TYPE,
-                               resource_id,
+                                   resource_type,
-                               operation_type,
+                                   resource_id,
-                               json.dumps(resource_info))
+                                   operation_type,
-        except exception.ResourceNotFound as e:
+                                   json.dumps(resource_info))
-            raise webob.exc.HTTPNotFound(explanation=e.format_message())
+            except exception.ResourceNotFound as e:
+                raise webob.exc.HTTPNotFound(explanation=e.format_message())
 
 
 class IdentityAPIController(APIController):

distributedcloud/dcorch/api/proxy/common/constants.py

@@ -91,7 +91,8 @@ COMMUNITY_STRING_PATHS = [
 ]
 
 CERTIFICATE_PATHS = [
-    '/v1/certificate/certificate_install'
+    '/v1/certificate/certificate_install',
+    '/v1/certificate/{uuid}'
 ]
 
 USER_PATHS = [
@@ -315,7 +316,7 @@ ROUTE_METHOD_MAP = {
         consts.RESOURCE_TYPE_SYSINV_DNS: ['PATCH', 'PUT'],
         consts.RESOURCE_TYPE_SYSINV_SNMP_TRAPDEST: ['POST', 'DELETE'],
         consts.RESOURCE_TYPE_SYSINV_SNMP_COMM: ['POST', 'DELETE'],
-        consts.RESOURCE_TYPE_SYSINV_CERTIFICATE: ['POST'],
+        consts.RESOURCE_TYPE_SYSINV_CERTIFICATE: ['POST', 'DELETE'],
         consts.RESOURCE_TYPE_SYSINV_USER: ['PATCH', 'PUT'],
     },
     consts.ENDPOINT_TYPE_NETWORK: {

distributedcloud/dcorch/common/exceptions.py

@@ -226,3 +226,8 @@ class CommunityAlreadyExists(Conflict):
 
 class CommunityNotFound(NotFound):
     message = _("Community %(community)s not found in region=%(region_name)s")
+
+
+class CertificateNotFound(NotFound):
+    message = _("Certificate in region=%(region_name)s with signature "
+                "%(signature)s not found")

distributedcloud/dcorch/drivers/openstack/sysinv_v1.py

@@ -370,6 +370,21 @@ class SysinvClient(base.DriverBase):
 
         return icertificate
 
+    def delete_certificate(self, certificate):
+        """Delete the certificate for this region
+
+           :param: a CA certificate to delete
+        """
+        try:
+            LOG.info(" delete_certificate region {} certificate: {}".format(
+                     self.region_name, certificate.signature))
+            self.client.certificate.certificate_uninstall(certificate.uuid)
+        except HTTPNotFound:
+            LOG.info("delete_certificate NotFound {} for region: {}".format(
+                     certificate.signature, self.region_name))
+            raise exceptions.CertificateNotFound(
+                region_name=self.region_name, signature=certificate.signature)
+
     def get_user(self):
         """Get the user password info for this region
 

distributedcloud/dcorch/engine/sync_services/sysinv.py

@@ -24,6 +24,7 @@ from dcorch.common import exceptions
 from dcorch.drivers.openstack import sdk_platform as sdk
 from dcorch.engine.fernet_key_manager import FERNET_REPO_MASTER_ID
 from dcorch.engine.fernet_key_manager import FernetKeyManager
+from dcorch.engine.sync_thread import AUDIT_RESOURCE_EXTRA
 from dcorch.engine.sync_thread import AUDIT_RESOURCE_MISSING
 from dcorch.engine.sync_thread import SyncThread
 
@@ -336,15 +337,15 @@ class SysinvSyncThread(SyncThread):
             metadata))
         return certificate, metadata
 
-    def sync_certificates(self, s_os_client, request, rsrc):
+    def create_certificate(self, s_os_client, request, rsrc):
-        LOG.info("sync_certificate resource_info={}".format(
+        LOG.info("create_certificate resource_info={}".format(
                  request.orch_job.resource_info),
                  extra=self.log_extra)
         certificate_dict = jsonutils.loads(request.orch_job.resource_info)
         payload = certificate_dict.get('payload')
 
         if not payload:
-            LOG.info("sync_certificate No payload found in resource_info"
+            LOG.info("create_certificate No payload found in resource_info"
                      "{}".format(request.orch_job.resource_info),
                      extra=self.log_extra)
             return
@@ -359,7 +360,7 @@ class SysinvSyncThread(SyncThread):
         certificate, metadata = self._decode_certificate_payload(
             certificate_dict)
 
-        isignature = None
+        icertificate = None
         signature = rsrc.master_id
         if signature and signature != self.CERTIFICATE_SIG_NULL:
             icertificate = self.update_certificate(
@@ -367,9 +368,6 @@ class SysinvSyncThread(SyncThread):
                 signature,
                 certificate=certificate,
                 data=metadata)
-            cert_body = icertificate.get('certificates')
-            if cert_body:
-                isignature = cert_body.get('signature')
         else:
             LOG.info("skipping signature={}".format(signature))
 
@@ -377,10 +375,69 @@ class SysinvSyncThread(SyncThread):
         subcloud_rsrc_id = self.persist_db_subcloud_resource(
             rsrc.id, signature)
 
-        LOG.info("certificate {} {} [{}/{}] updated".format(rsrc.id,
+        cert_bodys = icertificate.get('certificates')
-                 subcloud_rsrc_id, isignature, signature),
+        sub_certs_updated = [str(cert_body.get('signature'))
+                             for cert_body in cert_bodys]
+
+        LOG.info("certificate {} {} [{}] updated with subcloud certificates:"
+                 " {}".format(rsrc.id, subcloud_rsrc_id, signature,
+                              sub_certs_updated),
                  extra=self.log_extra)
 
+    def delete_certificate(self, s_os_client, request, rsrc):
+        subcloud_rsrc = self.get_db_subcloud_resource(rsrc.id)
+        if not subcloud_rsrc:
+            return
+
+        try:
+            certificates = s_os_client.sysinv_client.get_certificates()
+            cert_to_delete = None
+            for certificate in certificates:
+                if certificate.signature == subcloud_rsrc.subcloud_resource_id:
+                    cert_to_delete = certificate
+                    break
+            if not cert_to_delete:
+                raise exceptions.CertificateNotFound(
+                    region_name=self.subcloud_engine.subcloud.region_name,
+                    signature=subcloud_rsrc.subcloud_resource_id)
+            s_os_client.sysinv_client.delete_certificate(cert_to_delete)
+        except exceptions.CertificateNotFound:
+            # Certificate already deleted in subcloud, carry on.
+            LOG.info("Certificate not in subcloud, may be already deleted",
+                     extra=self.log_extra)
+        except (AttributeError, TypeError) as e:
+            LOG.info("delete_certificate error {}".format(e),
+                     extra=self.log_extra)
+            raise exceptions.SyncRequestFailedRetry
+
+        subcloud_rsrc.delete()
+        # Master Resource can be deleted only when all subcloud resources
+        # are deleted along with corresponding orch_job and orch_requests.
+        LOG.info("Certificate {}:{} [{}] deleted".format(
+                 rsrc.id, subcloud_rsrc.id,
+                 subcloud_rsrc.subcloud_resource_id),
+                 extra=self.log_extra)
+
+    def sync_certificates(self, s_os_client, request, rsrc):
+        switcher = {
+            consts.OPERATION_TYPE_POST: self.create_certificate,
+            consts.OPERATION_TYPE_CREATE: self.create_certificate,
+            consts.OPERATION_TYPE_DELETE: self.delete_certificate,
+        }
+
+        func = switcher[request.orch_job.operation_type]
+        try:
+            func(s_os_client, request, rsrc)
+        except (keystone_exceptions.connection.ConnectTimeout,
+                keystone_exceptions.ConnectFailure) as e:
+            LOG.info("sync_certificates: subcloud {} is not reachable [{}]"
+                     .format(self.subcloud_engine.subcloud.region_name,
+                             str(e)), extra=self.log_extra)
+            raise exceptions.SyncRequestTimeout
+        except Exception as e:
+            LOG.exception(e)
+            raise exceptions.SyncRequestFailedRetry
+
     def update_user(self, s_os_client, passwd_hash,
                     root_sig, passwd_expiry_days):
         LOG.info("update_user={} {} {}".format(
@@ -774,15 +831,15 @@ class SysinvSyncThread(SyncThread):
             # resource can be either from dcorch DB or
             # fetched by OpenStack query
             resource_id = self.get_resource_id(resource_type, resource)
+            if resource_id == self.CERTIFICATE_SIG_NULL:
+                LOG.info("No certificate resource to sync")
+                return num_of_audit_jobs
+            elif resource_id == self.RESOURCE_UUID_NULL:
+                LOG.info("No resource to sync")
+                return num_of_audit_jobs
+
             if finding == AUDIT_RESOURCE_MISSING:
                 # default action is create for a 'missing' resource
-                if resource_id == self.CERTIFICATE_SIG_NULL:
-                    LOG.info("No certificate resource to sync")
-                    return num_of_audit_jobs
-                elif resource_id == self.RESOURCE_UUID_NULL:
-                    LOG.info("No resource to sync")
-                    return num_of_audit_jobs
-
                 self.schedule_work(
                     self.endpoint_type, resource_type,
                     resource_id,
@@ -791,6 +848,12 @@ class SysinvSyncThread(SyncThread):
                         resource_type, resource,
                         consts.OPERATION_TYPE_CREATE))
                 num_of_audit_jobs += 1
+            elif finding == AUDIT_RESOURCE_EXTRA:
+                # default action is delete for a 'extra' resource
+                self.schedule_work(self.endpoint_type, resource_type,
+                                   resource_id,
+                                   consts.OPERATION_TYPE_DELETE)
+                num_of_audit_jobs += 1
             return num_of_audit_jobs
         else:  # use default audit_action
             return super(SysinvSyncThread, self).audit_action(