Fix dc-cert sync status handling for unmanaged subcloud

When a subcloud is unmanaged and then managed again, it will
never go back in sync. This is because all sync_statuses are
set to unknown when the unmanage is done. However, the
dc-cert_sync_status is only set to in-sync when the subcloud
transitions from offline to online. So in this case the
dc-cert_sync_status stays as unknown.

The fix is to avoid setting the dc-cert sync status to unknown
when a subcloud is unmanaged. The dc-cert sync status will
still being audited for unmanaged subclouds so there is no
need to do this.

Change-Id: I7a53c11888b8866c6077679265d69c36faf4e264
Closes-Bug: 1887733
Signed-off-by: Bart Wensley <barton.wensley@windriver.com>
(cherry picked from commit 6a16d05061)

distributedcloud/dcmanager/manager/subcloud_manager.py

@@ -850,18 +850,21 @@ class SubcloudManager(manager.Manager):
 
             if management_state == consts.MANAGEMENT_UNMANAGED:
 
-                # set all endpoint statuses to unknown
+                # set all endpoint statuses to unknown, except the dc-cert
+                # endpoint which continues to be audited for unmanaged
+                # subclouds
                 self.update_subcloud_endpoint_status(
                     context,
                     subcloud_name=subcloud.name,
                     endpoint_type=None,
-                    sync_status=consts.SYNC_STATUS_UNKNOWN)
+                    sync_status=consts.SYNC_STATUS_UNKNOWN,
+                    ignore_endpoints=[dcorch_consts.ENDPOINT_TYPE_DC_CERT])
 
         return db_api.subcloud_db_model_to_dict(subcloud)
 
     def _update_online_managed_subcloud(self, context, subcloud_id,
                                         endpoint_type, sync_status,
-                                        alarmable):
+                                        alarmable, ignore_endpoints=None):
         """Update online/managed subcloud endpoint status
 
         :param context: request context object
@@ -869,8 +872,13 @@ class SubcloudManager(manager.Manager):
         :param endpoint_type: endpoint type to update
         :param sync_status: sync status to set
         :param alarmable: controls raising an alarm if applicable
+        :param ignore_endpoints: list of endpoints to ignore (only used if
+               endpoint_type is None)
         """
 
+        if ignore_endpoints is None:
+            ignore_endpoints = []
+
         subcloud_status_list = []
         subcloud = None
         # retrieve the info from the db for this subcloud.
@@ -955,11 +963,16 @@ class SubcloudManager(manager.Manager):
 
             else:
                 # update all endpoints on this subcloud
-                LOG.info("Updating all subclouds, endpoint: %s sync: %s" %
+                LOG.info("Updating all endpoints on subcloud: %s sync: %s "
-                         (endpoint_type, sync_status))
+                         "ignore_endpoints: %s" %
+                         (subcloud.name, sync_status, ignore_endpoints))
 
                 for entry in subcloud_status_list:
                     endpoint = entry[consts.ENDPOINT_TYPE]
+                    if endpoint in ignore_endpoints:
+                        # Do not update this endpoint
+                        continue
+
                     db_api.subcloud_status_update(context,
                                                   subcloud_id,
                                                   endpoint,
@@ -1014,7 +1027,8 @@ class SubcloudManager(manager.Manager):
             subcloud_name,
             endpoint_type=None,
             sync_status=consts.SYNC_STATUS_OUT_OF_SYNC,
-            alarmable=True):
+            alarmable=True,
+            ignore_endpoints=None):
         """Update subcloud endpoint status
 
         :param context: request context object
@@ -1022,8 +1036,13 @@ class SubcloudManager(manager.Manager):
         :param endpoint_type: endpoint type to update
         :param sync_status: sync status to set
         :param alarmable: controls raising an alarm if applicable
+        :param ignore_endpoints: list of endpoints to ignore (only used if
+               endpoint_type is None)
         """
 
+        if ignore_endpoints is None:
+            ignore_endpoints = []
+
         if not subcloud_name:
             raise exceptions.BadRequest(
                 resource='subcloud',
@@ -1053,21 +1072,24 @@ class SubcloudManager(manager.Manager):
                                                      subcloud.id,
                                                      endpoint_type,
                                                      sync_status,
-                                                     alarmable)
+                                                     alarmable,
+                                                     ignore_endpoints)
             except Exception as e:
                 LOG.exception(e)
                 raise e
         else:
-            LOG.info("Ignoring unmanaged/offline subcloud sync_status "
+            LOG.info("Ignoring subcloud sync_status update for subcloud:%s "
-                     "update for subcloud:%s endpoint:%s sync:%s" %
+                     "availability:%s management:%s endpoint:%s sync:%s" %
-                     (subcloud_name, endpoint_type, sync_status))
+                     (subcloud_name, subcloud.availability_status,
+                      subcloud.management_state, endpoint_type, sync_status))
 
     def update_subcloud_endpoint_status(
             self, context,
             subcloud_name=None,
             endpoint_type=None,
             sync_status=consts.SYNC_STATUS_OUT_OF_SYNC,
-            alarmable=True):
+            alarmable=True,
+            ignore_endpoints=None):
         """Update subcloud endpoint status
 
         :param context: request context object
@@ -1075,17 +1097,23 @@ class SubcloudManager(manager.Manager):
         :param endpoint_type: endpoint type to update
         :param sync_status: sync status to set
         :param alarmable: controls raising an alarm if applicable
+        :param ignore_endpoints: list of endpoints to ignore (only used if
+               endpoint_type is None)
         """
 
+        if ignore_endpoints is None:
+            ignore_endpoints = []
+
         if subcloud_name:
             self._update_subcloud_endpoint_status(
-                context, subcloud_name, endpoint_type, sync_status, alarmable)
+                context, subcloud_name, endpoint_type, sync_status, alarmable,
+                ignore_endpoints)
         else:
             # update all subclouds
             for subcloud in db_api.subcloud_get_all(context):
                 self._update_subcloud_endpoint_status(
                     context, subcloud.name, endpoint_type, sync_status,
-                    alarmable)
+                    alarmable, ignore_endpoints)
 
     def _update_subcloud_state(self, context, subcloud_name,
                                management_state, availability_status):

distributedcloud/dcmanager/tests/unit/manager/test_subcloud_manager.py

@@ -433,6 +433,51 @@ class TestSubcloudManager(base.DCManagerTestCase):
             self.assertEqual(updated_subcloud_status.sync_status,
                              consts.SYNC_STATUS_OUT_OF_SYNC)
 
+        # Attempt to update each status to be unknown for an offline/unmanaged
+        # subcloud. This is allowed.
+        sm.update_subcloud_endpoint_status(
+            self.ctx, subcloud_name=subcloud.name,
+            endpoint_type=None,
+            sync_status=consts.SYNC_STATUS_UNKNOWN)
+
+        for endpoint in [dcorch_consts.ENDPOINT_TYPE_PLATFORM,
+                         dcorch_consts.ENDPOINT_TYPE_IDENTITY,
+                         dcorch_consts.ENDPOINT_TYPE_PATCHING,
+                         dcorch_consts.ENDPOINT_TYPE_FM,
+                         dcorch_consts.ENDPOINT_TYPE_NFV,
+                         dcorch_consts.ENDPOINT_TYPE_DC_CERT]:
+            updated_subcloud_status = db_api.subcloud_status_get(
+                self.ctx, subcloud.id, endpoint)
+            self.assertIsNotNone(updated_subcloud_status)
+            self.assertEqual(updated_subcloud_status.sync_status,
+                             consts.SYNC_STATUS_UNKNOWN)
+
+        # Attempt to update each status to be out-of-sync for an
+        # offline/unmanaged subcloud. Exclude one endpoint. This is allowed.
+        sm.update_subcloud_endpoint_status(
+            self.ctx, subcloud_name=subcloud.name,
+            endpoint_type=None,
+            sync_status=consts.SYNC_STATUS_OUT_OF_SYNC,
+            ignore_endpoints=[dcorch_consts.ENDPOINT_TYPE_DC_CERT])
+
+        for endpoint in [dcorch_consts.ENDPOINT_TYPE_PLATFORM,
+                         dcorch_consts.ENDPOINT_TYPE_IDENTITY,
+                         dcorch_consts.ENDPOINT_TYPE_PATCHING,
+                         dcorch_consts.ENDPOINT_TYPE_FM,
+                         dcorch_consts.ENDPOINT_TYPE_NFV]:
+            updated_subcloud_status = db_api.subcloud_status_get(
+                self.ctx, subcloud.id, endpoint)
+            self.assertIsNotNone(updated_subcloud_status)
+            self.assertEqual(updated_subcloud_status.sync_status,
+                             consts.SYNC_STATUS_OUT_OF_SYNC)
+        # Verify the dc-sync endpoint did not change
+        endpoint = dcorch_consts.ENDPOINT_TYPE_DC_CERT
+        updated_subcloud_status = db_api.subcloud_status_get(
+            self.ctx, subcloud.id, endpoint)
+        self.assertIsNotNone(updated_subcloud_status)
+        self.assertEqual(updated_subcloud_status.sync_status,
+                         consts.SYNC_STATUS_UNKNOWN)
+
         # Set/verify the subcloud is online/unmanaged
         db_api.subcloud_update(
             self.ctx, subcloud.id,
@@ -463,6 +508,8 @@ class TestSubcloudManager(base.DCManagerTestCase):
             self.assertEqual(updated_subcloud_status.sync_status,
                              consts.SYNC_STATUS_OUT_OF_SYNC)
 
+        # Attempt to update dc-cert status to be in-sync for an
+        # online/unmanaged subcloud. This is allowed. Verify the change.
         endpoint = dcorch_consts.ENDPOINT_TYPE_DC_CERT
         sm.update_subcloud_endpoint_status(
             self.ctx, subcloud_name=subcloud.name,
@@ -472,7 +519,6 @@ class TestSubcloudManager(base.DCManagerTestCase):
         updated_subcloud_status = db_api.subcloud_status_get(
             self.ctx, subcloud.id, endpoint)
         self.assertIsNotNone(updated_subcloud_status)
-        # No change in status: Only online/managed clouds are updated
         self.assertEqual(updated_subcloud_status.sync_status,
                          consts.SYNC_STATUS_IN_SYNC)