From b6693ea74d00b33798bcb66298294e199a892fdb Mon Sep 17 00:00:00 2001 From: Hugo Brito Date: Thu, 18 Apr 2024 16:39:01 -0300 Subject: [PATCH] Improve subcloud_get_all_with_status query efficiency Refactor the subcloud_get_all_with_status function to query only the endpoint_type and sync_status from the subcloud_status table for improved efficiency. Test Plan: PASS: Get all subclouds with the right sync_status PASS: Create a dcmanager strategy - Fail if sync_status = Unknown Story: 2011106 Task: 49893 Change-Id: Ie99abc6cb820800a632f1fd90ee7d7e0869a8312 Signed-off-by: Hugo Brito --- .../dcmanager/api/controllers/v1/subclouds.py | 79 ++++------ .../dcmanager/db/sqlalchemy/api.py | 15 +- .../orchestrator/sw_update_manager.py | 137 ++++++++++-------- 3 files changed, 114 insertions(+), 117 deletions(-) diff --git a/distributedcloud/dcmanager/api/controllers/v1/subclouds.py b/distributedcloud/dcmanager/api/controllers/v1/subclouds.py index 68b528856..992a6a1d0 100644 --- a/distributedcloud/dcmanager/api/controllers/v1/subclouds.py +++ b/distributedcloud/dcmanager/api/controllers/v1/subclouds.py @@ -402,65 +402,40 @@ class SubcloudsController(object): if subcloud_ref is None: # List of subclouds requested subclouds = db_api.subcloud_get_all_with_status(context) - result = dict() - result['subclouds'] = [] - first_time = True - subcloud_list = [] - subcloud_status_list = [] + result = {'subclouds': []} + subcloud_dict = {} - # We get back a subcloud, subcloud_status pair for every - # subcloud_status entry corresponding to a subcloud. (Subcloud - # info repeats) - # Aggregate all the sync status for each of the - # endpoints per subcloud into an overall sync status - for subcloud, subcloud_status in subclouds: - subcloud_dict = db_api.subcloud_db_model_to_dict(subcloud) - subcloud_status_dict = db_api.subcloud_status_db_model_to_dict( - subcloud_status) - subcloud_dict.update(subcloud_status_dict) + for subcloud, endpoint_type, sync_status in subclouds: + subcloud_id = subcloud.id + if subcloud_id not in subcloud_dict: + subcloud_dict[subcloud_id] = db_api.subcloud_db_model_to_dict( + subcloud) + self._append_static_err_content(subcloud_dict[subcloud_id]) + subcloud_dict[subcloud_id].update( + {consts.SYNC_STATUS: sync_status} + ) + subcloud_dict[subcloud_id][consts.ENDPOINT_SYNC_STATUS] = [] - self._append_static_err_content(subcloud_dict) + subcloud_dict[subcloud_id][consts.ENDPOINT_SYNC_STATUS].append( + { + consts.ENDPOINT_TYPE: endpoint_type, + consts.SYNC_STATUS: sync_status + } + ) - if not first_time: - if subcloud_list[-1]['id'] == subcloud_dict['id']: - # We have a match for this subcloud id already, - # check if we have a same sync_status - if subcloud_list[-1][consts.SYNC_STATUS] != \ - subcloud_dict[consts.SYNC_STATUS]: - subcloud_list[-1][consts.SYNC_STATUS] = \ - dccommon_consts.SYNC_STATUS_OUT_OF_SYNC + # If any of the endpoint sync status is out of sync, then + # the subcloud sync status is out of sync + if sync_status != subcloud_dict[subcloud_id][consts.SYNC_STATUS]: + subcloud_dict[subcloud_id][consts.SYNC_STATUS] = ( + dccommon_consts.SYNC_STATUS_OUT_OF_SYNC + ) - if subcloud_status: - subcloud_status_list.append( - db_api.subcloud_endpoint_status_db_model_to_dict( - subcloud_status)) - subcloud_list[-1][ - consts.ENDPOINT_SYNC_STATUS] = subcloud_status_list - - else: - subcloud_status_list = [] - if subcloud_status: - subcloud_status_list.append( - db_api.subcloud_endpoint_status_db_model_to_dict( - subcloud_status)) - - subcloud_list.append(subcloud_dict) - else: - if subcloud_status: - subcloud_status_list.append( - db_api.subcloud_endpoint_status_db_model_to_dict( - subcloud_status)) - subcloud_list.append(subcloud_dict) - - first_time = False - - for s in subcloud_list: + for subcloud in subcloud_dict.values(): # This is to reduce changes on cert-mon # Overwrites the name value with region if utils.is_req_from_cert_mon_agent(request): - s['name'] = s['region-name'] - result['subclouds'].append(s) - + subcloud['name'] = subcloud['region-name'] + result['subclouds'].append(subcloud) return result else: # Single subcloud requested diff --git a/distributedcloud/dcmanager/db/sqlalchemy/api.py b/distributedcloud/dcmanager/db/sqlalchemy/api.py index 39e52ca12..481736e62 100644 --- a/distributedcloud/dcmanager/db/sqlalchemy/api.py +++ b/distributedcloud/dcmanager/db/sqlalchemy/api.py @@ -363,13 +363,14 @@ def subcloud_get_all_ordered_by_id(context): @require_context def subcloud_get_all_with_status(context): - result = model_query(context, models.Subcloud, models.SubcloudStatus). \ - outerjoin(models.SubcloudStatus, - (models.Subcloud.id == models.SubcloudStatus.subcloud_id) | - (not models.SubcloudStatus.subcloud_id)). \ - filter(models.Subcloud.deleted == 0). \ - order_by(models.Subcloud.id). \ - all() + result = model_query( + context, + models.Subcloud, + models.SubcloudStatus.endpoint_type, + models.SubcloudStatus.sync_status).join( + models.SubcloudStatus, + models.Subcloud.id == models.SubcloudStatus.subcloud_id).filter( + models.Subcloud.deleted == 0).order_by(models.Subcloud.id).all() return result diff --git a/distributedcloud/dcmanager/orchestrator/sw_update_manager.py b/distributedcloud/dcmanager/orchestrator/sw_update_manager.py index ce39f2931..a5270fd8d 100644 --- a/distributedcloud/dcmanager/orchestrator/sw_update_manager.py +++ b/distributedcloud/dcmanager/orchestrator/sw_update_manager.py @@ -459,91 +459,112 @@ class SwUpdateManager(manager.Manager): for sb in db_api.subcloud_get_for_group(context, single_group.id): statuses = db_api.subcloud_status_get_all(context, sb.id) for status in statuses: - subclouds.append((sb, status)) + subclouds.append((sb, status.endpoint_type, status.sync_status)) else: subclouds = db_api.subcloud_get_all_with_status(context) subclouds_processed = list() - for subcloud, subcloud_status in subclouds: - if (cloud_name and subcloud.name != cloud_name or - subcloud.management_state != dccommon_consts.MANAGEMENT_MANAGED): + for subcloud, endpoint_type, sync_status in subclouds: + if ( + cloud_name + and subcloud.name != cloud_name + or subcloud.management_state != dccommon_consts.MANAGEMENT_MANAGED + ): # We are not updating this subcloud continue if strategy_type == consts.SW_UPDATE_TYPE_UPGRADE: - if subcloud.availability_status != \ - dccommon_consts.AVAILABILITY_ONLINE: + if ( + subcloud.availability_status != + dccommon_consts.AVAILABILITY_ONLINE + ): if not force: continue - elif (subcloud_status.endpoint_type == - dccommon_consts.ENDPOINT_TYPE_LOAD and - subcloud_status.sync_status == - dccommon_consts.SYNC_STATUS_UNKNOWN): + elif ( + endpoint_type == dccommon_consts.ENDPOINT_TYPE_LOAD + and sync_status == dccommon_consts.SYNC_STATUS_UNKNOWN + ): raise exceptions.BadRequest( - resource='strategy', - msg='Upgrade sync status is unknown for one or more ' - 'subclouds') + resource="strategy", + msg="Upgrade sync status is unknown for one or more " + "subclouds", + ) elif strategy_type == consts.SW_UPDATE_TYPE_SOFTWARE: - if subcloud.availability_status != \ - dccommon_consts.AVAILABILITY_ONLINE: + if ( + subcloud.availability_status != + dccommon_consts.AVAILABILITY_ONLINE + ): if not force: continue - if (subcloud_status.endpoint_type == - dccommon_consts.ENDPOINT_TYPE_SOFTWARE and - subcloud_status.sync_status == - dccommon_consts.SYNC_STATUS_UNKNOWN): + if ( + endpoint_type == dccommon_consts.ENDPOINT_TYPE_SOFTWARE + and sync_status == dccommon_consts.SYNC_STATUS_UNKNOWN + ): raise exceptions.BadRequest( - resource='strategy', - msg='Software sync status is unknown for one or more ' - 'subclouds') + resource="strategy", + msg="Software sync status is unknown for one or more " + "subclouds", + ) elif strategy_type == consts.SW_UPDATE_TYPE_PATCH: - if subcloud.availability_status != \ - dccommon_consts.AVAILABILITY_ONLINE: + if ( + subcloud.availability_status != + dccommon_consts.AVAILABILITY_ONLINE + ): continue - elif (subcloud_status.endpoint_type == - dccommon_consts.ENDPOINT_TYPE_PATCHING and - subcloud_status.sync_status == - dccommon_consts.SYNC_STATUS_UNKNOWN): + elif ( + endpoint_type == dccommon_consts.ENDPOINT_TYPE_PATCHING + and sync_status == dccommon_consts.SYNC_STATUS_UNKNOWN + ): raise exceptions.BadRequest( - resource='strategy', - msg='Patching sync status is unknown for one or more ' - 'subclouds') + resource="strategy", + msg="Patching sync status is unknown for one or more " + "subclouds", + ) elif strategy_type == consts.SW_UPDATE_TYPE_FIRMWARE: - if subcloud.availability_status != \ - dccommon_consts.AVAILABILITY_ONLINE: + if ( + subcloud.availability_status != + dccommon_consts.AVAILABILITY_ONLINE + ): continue - elif (subcloud_status.endpoint_type == - dccommon_consts.ENDPOINT_TYPE_FIRMWARE and - subcloud_status.sync_status == - dccommon_consts.SYNC_STATUS_UNKNOWN): + elif ( + endpoint_type == dccommon_consts.ENDPOINT_TYPE_FIRMWARE + and sync_status == dccommon_consts.SYNC_STATUS_UNKNOWN + ): raise exceptions.BadRequest( - resource='strategy', - msg='Firmware sync status is unknown for one or more ' - 'subclouds') + resource="strategy", + msg="Firmware sync status is unknown for one or more " + "subclouds", + ) elif strategy_type == consts.SW_UPDATE_TYPE_KUBERNETES: - if subcloud.availability_status != \ - dccommon_consts.AVAILABILITY_ONLINE: + if ( + subcloud.availability_status != + dccommon_consts.AVAILABILITY_ONLINE + ): continue - elif (subcloud_status.endpoint_type == - dccommon_consts.ENDPOINT_TYPE_KUBERNETES and - subcloud_status.sync_status == - dccommon_consts.SYNC_STATUS_UNKNOWN): + elif ( + endpoint_type == dccommon_consts.ENDPOINT_TYPE_KUBERNETES + and sync_status == dccommon_consts.SYNC_STATUS_UNKNOWN + ): raise exceptions.BadRequest( - resource='strategy', - msg='Kubernetes sync status is unknown for one or more ' - 'subclouds') + resource="strategy", + msg="Kubernetes sync status is unknown for one or more " + "subclouds", + ) elif strategy_type == consts.SW_UPDATE_TYPE_KUBE_ROOTCA_UPDATE: - if subcloud.availability_status != \ - dccommon_consts.AVAILABILITY_ONLINE: + if ( + subcloud.availability_status != + dccommon_consts.AVAILABILITY_ONLINE + ): continue - elif (subcloud_status.endpoint_type == - dccommon_consts.ENDPOINT_TYPE_KUBE_ROOTCA and - subcloud_status.sync_status == - dccommon_consts.SYNC_STATUS_UNKNOWN): + elif ( + endpoint_type == dccommon_consts.ENDPOINT_TYPE_KUBE_ROOTCA + and sync_status == dccommon_consts.SYNC_STATUS_UNKNOWN + ): raise exceptions.BadRequest( - resource='strategy', - msg='Kube rootca update sync status is unknown for ' - 'one or more subclouds') + resource="strategy", + msg="Kube rootca update sync status is unknown for " + "one or more subclouds", + ) elif strategy_type == consts.SW_UPDATE_TYPE_PRESTAGE: if subcloud.name not in subclouds_processed: # Do initial validation for subcloud