From 477c9e6cb8be58cff9debd4893fe4abf197d1c45 Mon Sep 17 00:00:00 2001 From: "Zhang Rong(Jon)" Date: Thu, 24 Aug 2023 15:46:22 +0800 Subject: [PATCH] Add dcmanager peer group association management API support Add dcmanager peer-group-association management API. This commit adds peer-group-association APIs of create/delete/update/show/list/sync, and adds system-peer API of list subcloud-peer-groups. Test Plan: 1. PASS - Verify that cloud manage peer-group-association through api successfully. 2. PASS - Verify that cloud get associated subcloud-peer-group list with system-peer api successfully. 3. PASS - Check create without providing the must required parameters. 4. PASS - Check create with wrong peer_group_priority. 5. PASS - Check delete with a not existing association id. 6. PASS - Check system_peer_manager associate peer group, sync peer group and delete association. 7. PASS - Create an association with 50 subclouds need to sync. Check the sync status on peer site after it is synced. Story: 2010852 Task: 48506 Change-Id: I41c16a8ab13e60f5b1de5b05fbbc51821f7f8d57 Signed-off-by: Zhang Rong(Jon) --- api-ref/source/api-ref-dcmanager-v1.rst | 307 ++++++++++- api-ref/source/parameters.yaml | 43 ++ .../association-get-response.json | 10 + .../association-patch-request.json | 3 + .../association-patch-response.json | 10 + .../associations-get-response.json | 13 + .../associations-post-request.json | 5 + .../associations-post-response.json | 10 + .../system-peer-patch-request.json | 4 +- ...system-peers-get-peer-groups-response.json | 15 + .../system-peers-post-request.json | 4 +- .../drivers/openstack/dcmanager_v1.py | 9 +- distributedcloud/dccommon/exceptions.py | 5 + .../controllers/v1/peer_group_association.py | 360 +++++++++++++ .../dcmanager/api/controllers/v1/root.py | 3 + .../api/controllers/v1/subcloud_peer_group.py | 5 +- .../dcmanager/api/controllers/v1/subclouds.py | 17 +- .../api/controllers/v1/system_peers.py | 31 +- .../dcmanager/api/policies/__init__.py | 2 + .../api/policies/peer_group_association.py | 72 +++ .../dcmanager/api/policies/system_peers.py | 6 + distributedcloud/dcmanager/common/consts.py | 10 + .../dcmanager/common/exceptions.py | 20 + distributedcloud/dcmanager/db/api.py | 82 ++- .../dcmanager/db/sqlalchemy/api.py | 141 ++++- ...add_subcloud_peer_group_and_association.py | 27 + .../dcmanager/db/sqlalchemy/models.py | 14 + distributedcloud/dcmanager/manager/service.py | 17 + .../dcmanager/manager/system_peer_manager.py | 508 ++++++++++++++++++ distributedcloud/dcmanager/rpc/client.py | 18 + .../test_peer_group_association.py | 358 ++++++++++++ .../unit/manager/test_system_peer_manager.py | 389 ++++++++++++++ 32 files changed, 2492 insertions(+), 26 deletions(-) create mode 100644 api-ref/source/samples/peer-group-associations/association-get-response.json create mode 100644 api-ref/source/samples/peer-group-associations/association-patch-request.json create mode 100644 api-ref/source/samples/peer-group-associations/association-patch-response.json create mode 100644 api-ref/source/samples/peer-group-associations/associations-get-response.json create mode 100644 api-ref/source/samples/peer-group-associations/associations-post-request.json create mode 100644 api-ref/source/samples/peer-group-associations/associations-post-response.json create mode 100644 api-ref/source/samples/system-peers/system-peers-get-peer-groups-response.json create mode 100644 distributedcloud/dcmanager/api/controllers/v1/peer_group_association.py create mode 100644 distributedcloud/dcmanager/api/policies/peer_group_association.py create mode 100644 distributedcloud/dcmanager/manager/system_peer_manager.py create mode 100644 distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_peer_group_association.py create mode 100644 distributedcloud/dcmanager/tests/unit/manager/test_system_peer_manager.py diff --git a/api-ref/source/api-ref-dcmanager-v1.rst b/api-ref/source/api-ref-dcmanager-v1.rst index b951cda7a..caf1c8431 100644 --- a/api-ref/source/api-ref-dcmanager-v1.rst +++ b/api-ref/source/api-ref-dcmanager-v1.rst @@ -2457,6 +2457,52 @@ Response Example :language: json +***************************************************************** +Shows subcloud peer groups that are associated with a system peer +***************************************************************** + +.. rest_method:: GET /v1.0/system-peers/​{system-peer}​/subcloud-peer-groups + +**Normal response codes** + +200 + +**Error response codes** + +badRequest (400), unauthorized (401), forbidden (403), +itemNotFound (404), badMethod (405), HTTPUnprocessableEntity (422), +internalServerError (500), serviceUnavailable (503) + +**Request parameters** + +.. rest_parameters:: parameters.yaml + + - system-peer: system_peer_uri + +This operation does not accept a request body. + +**Response parameters** + +.. rest_parameters:: parameters.yaml + + - subcloud_peer_groups: subcloud_peer_groups + - id: subcloud_peer_group_id + - peer_group_name: subcloud_peer_group_name + - group_priority: subcloud_peer_group_priority + - group_state: subcloud_peer_group_administrative_state + - max_subcloud_rehoming: subcloud_peer_group_max_subcloud_rehoming + - system_leader_id: subcloud_peer_group_system_leader_id + - system_leader_name: subcloud_peer_group_system_leader_name + - created_at: created_at + - updated_at: updated_at + +Response Example +---------------- + +.. literalinclude:: samples/system-peers/system-peers-get-peer-groups-response.json + :language: json + + ******************************* Modifies a specific system peer ******************************* @@ -2895,4 +2941,263 @@ internalServerError (500), serviceUnavailable (503) - subcloud-peer-group: subcloud_peer_group_uri -This operation does not accept a request body. \ No newline at end of file +This operation does not accept a request body. + +---------------------- +Peer Group Association +---------------------- + +Peer Group Associations are logical connections managed by a central System Controller. +It's a linking of the subcloud peer group and the system peer to +establish associations with local subcloud peer groups and peer sites. + +********************************* +Lists all peer group associations +********************************* + +.. rest_method:: GET /v1.0/peer-group-associations + +This operation does not accept a request body. + +**Normal response codes** + +200 + +**Error response codes** + +badRequest (400), unauthorized (401), forbidden (403), +badMethod (405), HTTPUnprocessableEntity (422), +internalServerError (500), serviceUnavailable (503) + + +**Response parameters** + +.. rest_parameters:: parameters.yaml + + - peer-group-associations: peer_group_associations + - id: peer_group_association_id + - peer-group-id: association_peer_group_id + - system-peer-id: system_peer_id + - peer-group-priority: association_peer_group_priority + - sync-status: association_sync_status + - created-at: created_at + - updated-at: updated_at + +Response Example +---------------- + +.. literalinclude:: samples/peer-group-associations/associations-get-response.json + :language: json + + +******************************** +Creates a peer group association +******************************** + +.. rest_method:: POST /v1.0/peer-group-associations + +**Normal response codes** + +200 + +**Error response codes** + +badRequest (400), unauthorized (401), forbidden (403), badMethod (405), +HTTPUnprocessableEntity (422), internalServerError (500), +serviceUnavailable (503) + +**Request parameters** + +.. rest_parameters:: parameters.yaml + + - peer_group_id: association_peer_group_id + - system_peer_id: system_peer_id + - peer_group_priority: association_peer_group_priority + +Request Example +---------------- + +.. literalinclude:: samples/peer-group-associations/associations-post-request.json + :language: json + +**Response parameters** + +.. rest_parameters:: parameters.yaml + + - id: peer_group_association_id + - peer-group-id: association_peer_group_id + - system-peer-id: system_peer_id + - peer-group-priority: association_peer_group_priority + - sync-status: association_sync_status + - sync-message: association_sync_message + - created-at: created_at + - updated-at: updated_at + +Response Example +---------------- + +.. literalinclude:: samples/peer-group-associations/associations-post-response.json + :language: json + + +********************************************************* +Shows information about a specific peer group association +********************************************************* + +.. rest_method:: GET /v1.0/peer-group-associations/​{associate_id}​ + +**Normal response codes** + +200 + +**Error response codes** + +badRequest (400), unauthorized (401), forbidden (403), +itemNotFound (404), badMethod (405), HTTPUnprocessableEntity (422), +internalServerError (500), serviceUnavailable (503) + +**Request parameters** + +.. rest_parameters:: parameters.yaml + + - associate_id: peer_group_association_uri + +This operation does not accept a request body. + +**Response parameters** + +.. rest_parameters:: parameters.yaml + + - id: peer_group_association_id + - peer-group-id: association_peer_group_id + - system-peer-id: system_peer_id + - peer-group-priority: association_peer_group_priority + - sync-status: association_sync_status + - sync-message: association_sync_message + - created-at: created_at + - updated-at: updated_at + +Response Example +---------------- + +.. literalinclude:: samples/peer-group-associations/association-get-response.json + :language: json + + +********************************************** +Synchronizes a specific peer group association +********************************************** + +.. rest_method:: PATCH /v1.0/peer-group-associations/​{associate_id}​/sync + +**Normal response codes** + +200 + +**Error response codes** + +badRequest (400), unauthorized (401), forbidden (403), +itemNotFound (404), badMethod (405), HTTPUnprocessableEntity (422), +internalServerError (500), serviceUnavailable (503) + +**Request parameters** + +.. rest_parameters:: parameters.yaml + + - associate_id: peer_group_association_uri + +**Response parameters** + +.. rest_parameters:: parameters.yaml + + - id: peer_group_association_id + - peer-group-id: association_peer_group_id + - system-peer-id: system_peer_id + - peer-group-priority: association_peer_group_priority + - sync-status: association_sync_status + - sync-message: association_sync_message + - created-at: created_at + - updated-at: updated_at + +Response Example +---------------- + +.. literalinclude:: samples/peer-group-associations/association-patch-response.json + :language: json + + +****************************************** +Modifies a specific peer group association +****************************************** + +.. rest_method:: PATCH /v1.0/peer-group-associations/​{associate_id}​ + +The attributes of a subcloud peer group which are modifiable: + +- peer_group_priority + +**Normal response codes** + +200 + +**Error response codes** + +badRequest (400), unauthorized (401), forbidden (403), badMethod (405), +HTTPUnprocessableEntity (422), internalServerError (500), +serviceUnavailable (503) + +**Request parameters** + +.. rest_parameters:: parameters.yaml + + - associate_id: peer_group_association_uri + - peer_group_priority: association_peer_group_priority + +Request Example +---------------- +.. literalinclude:: samples/peer-group-associations/association-patch-request.json + :language: json + +**Response parameters** + +.. rest_parameters:: parameters.yaml + + - id: peer_group_association_id + - peer-group-id: association_peer_group_id + - system-peer-id: system_peer_id + - peer-group-priority: association_peer_group_priority + - sync-status: association_sync_status + - sync-message: association_sync_message + - created-at: created_at + - updated-at: updated_at + +Response Example +---------------- + +.. literalinclude:: samples/peer-group-associations/association-patch-response.json + :language: json + + +***************************************** +Deletes a specific peer group association +***************************************** + +.. rest_method:: DELETE /v1.0/peer-group-associations/​{associate_id}​ + +**Normal response codes** + +200 + +**Error response codes** + +badRequest (400), unauthorized (401), forbidden (403), +itemNotFound (404), badMethod (405), HTTPUnprocessableEntity (422), +internalServerError (500), serviceUnavailable (503) + +**Request parameters** + +.. rest_parameters:: parameters.yaml + + - associate_id: peer_group_association_uri + +This operation does not accept a request body. diff --git a/api-ref/source/parameters.yaml b/api-ref/source/parameters.yaml index f612a1116..ca14f7b39 100644 --- a/api-ref/source/parameters.yaml +++ b/api-ref/source/parameters.yaml @@ -6,6 +6,12 @@ backup_delete_release: in: path required: true type: string +peer_group_association_uri: + description: | + The peer group association reference id. + in: path + required: true + type: string release_uri: description: | The subcloud software version. @@ -77,6 +83,31 @@ alarm_summary_uuid: in: body required: true type: string +association_peer_group_id: + description: | + The ID of the subcloud peer group as an integer. + in: body + required: true + type: string +association_peer_group_priority: + description: | + The priority of the subcloud peer group in peer group association. + The lower the value, the higher the priority. + in: body + required: false + type: integer +association_sync_message: + description: | + The sync message for association. + in: body + required: true + type: string +association_sync_status: + description: | + The sync status for association. + in: body + required: true + type: string availability_status: description: | The availability status of the subcloud. @@ -397,6 +428,18 @@ peer_controller_gateway_address: in: body required: true type: string +peer_group_association_id: + description: | + The ID of a peer group association as an integer. + in: body + required: true + type: integer +peer_group_associations: + description: | + The list of ``peer-group-association`` objects. + in: body + required: true + type: array peer_name: description: | The name of a peer as a string. diff --git a/api-ref/source/samples/peer-group-associations/association-get-response.json b/api-ref/source/samples/peer-group-associations/association-get-response.json new file mode 100644 index 000000000..3aef416e5 --- /dev/null +++ b/api-ref/source/samples/peer-group-associations/association-get-response.json @@ -0,0 +1,10 @@ +{ + "id": 9, + "peer-group-id": 1, + "system-peer-id": 1, + "peer-group-priority": 1, + "sync-status": "synced", + "sync-message": null, + "created-at": "2023-08-21 09:24:07.394961", + "updated-at": null +} \ No newline at end of file diff --git a/api-ref/source/samples/peer-group-associations/association-patch-request.json b/api-ref/source/samples/peer-group-associations/association-patch-request.json new file mode 100644 index 000000000..653091482 --- /dev/null +++ b/api-ref/source/samples/peer-group-associations/association-patch-request.json @@ -0,0 +1,3 @@ +{ + "peer_group_priority": 99 +} \ No newline at end of file diff --git a/api-ref/source/samples/peer-group-associations/association-patch-response.json b/api-ref/source/samples/peer-group-associations/association-patch-response.json new file mode 100644 index 000000000..0d0b3917d --- /dev/null +++ b/api-ref/source/samples/peer-group-associations/association-patch-response.json @@ -0,0 +1,10 @@ +{ + "id": 9, + "peer-group-id": 1, + "system-peer-id": 1, + "peer-group-priority": 99, + "sync-status": "synced", + "sync-message": null, + "created-at": "2023-08-21 09:24:07.394961", + "updated-at": "2023-08-21 10:24:07.394961" +} \ No newline at end of file diff --git a/api-ref/source/samples/peer-group-associations/associations-get-response.json b/api-ref/source/samples/peer-group-associations/associations-get-response.json new file mode 100644 index 000000000..05152196e --- /dev/null +++ b/api-ref/source/samples/peer-group-associations/associations-get-response.json @@ -0,0 +1,13 @@ +{ + "peer_group_associations": [ + { + "id": 9, + "peer-group-id": 1, + "system-peer-id": 1, + "peer-group-priority": 1, + "sync-status": "synced", + "created-at": "2023-08-21 09:24:07.394961", + "updated-at": null + } + ] +} \ No newline at end of file diff --git a/api-ref/source/samples/peer-group-associations/associations-post-request.json b/api-ref/source/samples/peer-group-associations/associations-post-request.json new file mode 100644 index 000000000..b69a406a4 --- /dev/null +++ b/api-ref/source/samples/peer-group-associations/associations-post-request.json @@ -0,0 +1,5 @@ +{ + "peer_group_id": 1, + "system_peer_id": 1, + "peer_group_priority": 1 +} \ No newline at end of file diff --git a/api-ref/source/samples/peer-group-associations/associations-post-response.json b/api-ref/source/samples/peer-group-associations/associations-post-response.json new file mode 100644 index 000000000..d01bacea8 --- /dev/null +++ b/api-ref/source/samples/peer-group-associations/associations-post-response.json @@ -0,0 +1,10 @@ +{ + "id": 9, + "peer-group-id": 1, + "system-peer-id": 1, + "peer-group-priority": 1, + "sync-status": "syncing", + "sync-message": null, + "created-at": "2023-08-21 09:24:07.394961", + "updated-at": null +} \ No newline at end of file diff --git a/api-ref/source/samples/system-peers/system-peer-patch-request.json b/api-ref/source/samples/system-peers/system-peer-patch-request.json index 2518686ce..fe817a3a0 100644 --- a/api-ref/source/samples/system-peers/system-peer-patch-request.json +++ b/api-ref/source/samples/system-peers/system-peer-patch-request.json @@ -3,8 +3,8 @@ "peer_name": "PeerDistributedCloud1", "manager_endpoint": "http://128.128.128.1:5000/v3", "manager_username": "admin", - "manager-password": "V2luZDEyMyQ=", - "peer_controller_gateway-address": "192.168.204.1", + "manager_password": "V2luZDEyMyQ=", + "peer_controller_gateway_address": "192.168.204.1", "administrative_state": "enabled", "heartbeat_interval": 60, "heartbeat_failure_threshold": 3, diff --git a/api-ref/source/samples/system-peers/system-peers-get-peer-groups-response.json b/api-ref/source/samples/system-peers/system-peers-get-peer-groups-response.json new file mode 100644 index 000000000..2a752816a --- /dev/null +++ b/api-ref/source/samples/system-peers/system-peers-get-peer-groups-response.json @@ -0,0 +1,15 @@ +{ + "subcloud_peer_groups": [ + { + "id": 1, + "peer_group_name": "dc1-pg", + "group_priority": 0, + "group_state": "enabled", + "max_subcloud_rehoming": 10, + "system_leader_id": "ac62f555-9386-42f1-b3a1-51ecb709409d", + "system_leader_name": "dc1-name", + "created-at": "2023-07-26 00:51:01.396694", + "updated-at": "2023-08-07 06:09:04.086417" + } + ] +} \ No newline at end of file diff --git a/api-ref/source/samples/system-peers/system-peers-post-request.json b/api-ref/source/samples/system-peers/system-peers-post-request.json index 2518686ce..fe817a3a0 100644 --- a/api-ref/source/samples/system-peers/system-peers-post-request.json +++ b/api-ref/source/samples/system-peers/system-peers-post-request.json @@ -3,8 +3,8 @@ "peer_name": "PeerDistributedCloud1", "manager_endpoint": "http://128.128.128.1:5000/v3", "manager_username": "admin", - "manager-password": "V2luZDEyMyQ=", - "peer_controller_gateway-address": "192.168.204.1", + "manager_password": "V2luZDEyMyQ=", + "peer_controller_gateway_address": "192.168.204.1", "administrative_state": "enabled", "heartbeat_interval": 60, "heartbeat_failure_threshold": 3, diff --git a/distributedcloud/dccommon/drivers/openstack/dcmanager_v1.py b/distributedcloud/dccommon/drivers/openstack/dcmanager_v1.py index 76dd1300a..a6518b0df 100644 --- a/distributedcloud/dccommon/drivers/openstack/dcmanager_v1.py +++ b/distributedcloud/dccommon/drivers/openstack/dcmanager_v1.py @@ -33,13 +33,15 @@ class DcmanagerClient(base.DriverBase): self.token = session.get_token() self.timeout = timeout - def get_subcloud(self, subcloud_ref): + def get_subcloud(self, subcloud_ref, is_region_name=False): """Get subcloud.""" if subcloud_ref is None: raise ValueError("subcloud_ref is required.") url = f"{self.endpoint}/subclouds/{subcloud_ref}" headers = {"X-Auth-Token": self.token} + if is_region_name: + headers["User-Agent"] = "dcmanager/1.0" response = requests.get(url, headers=headers, timeout=self.timeout) if response.status_code == 200: @@ -267,6 +269,11 @@ class DcmanagerClient(base.DriverBase): 'Subcloud Peer Group not found' in response.text: raise exceptions.SubcloudPeerGroupNotFound( peer_group_ref=peer_group_ref) + elif response.status_code == 400 and \ + 'a peer group which is associated with a system peer' in \ + response.text: + raise exceptions.SubcloudPeerGroupDeleteFailedAssociated( + peer_group_ref=peer_group_ref) message = "Delete Subcloud Peer Group: peer_group_ref %s " \ "failed with RC: %d" % (peer_group_ref, response.status_code) LOG.error(message) diff --git a/distributedcloud/dccommon/exceptions.py b/distributedcloud/dccommon/exceptions.py index 694920def..b4670b111 100644 --- a/distributedcloud/dccommon/exceptions.py +++ b/distributedcloud/dccommon/exceptions.py @@ -135,3 +135,8 @@ class SubcloudNotFound(NotFound): class SubcloudPeerGroupNotFound(NotFound): message = _("Subcloud Peer Group %(peer_group_ref)s not found") + + +class SubcloudPeerGroupDeleteFailedAssociated(DCCommonException): + message = _("Subcloud Peer Group %(peer_group_ref)s delete failed " + "cause it is associated with a system peer.") diff --git a/distributedcloud/dcmanager/api/controllers/v1/peer_group_association.py b/distributedcloud/dcmanager/api/controllers/v1/peer_group_association.py new file mode 100644 index 000000000..79c3e7e1b --- /dev/null +++ b/distributedcloud/dcmanager/api/controllers/v1/peer_group_association.py @@ -0,0 +1,360 @@ +# +# Copyright (c) 2023 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +import http.client as httpclient +import json + +from oslo_config import cfg +from oslo_log import log as logging +from oslo_messaging import RemoteError +import pecan +from pecan import expose +from pecan import request + +from dccommon import consts as dccommon_consts +from dccommon.drivers.openstack.sysinv_v1 import SysinvClient +from dcmanager.api.controllers import restcomm +from dcmanager.api.policies import peer_group_association as \ + peer_group_association_policy +from dcmanager.api import policy +from dcmanager.common import consts +from dcmanager.common import exceptions as exception +from dcmanager.common.i18n import _ +from dcmanager.common import phased_subcloud_deploy as psd_common +from dcmanager.db import api as db_api +from dcmanager.rpc import client as rpc_client + + +CONF = cfg.CONF +LOG = logging.getLogger(__name__) + +MIN_PEER_GROUP_ASSOCIATION_PRIORITY = 1 +MAX_PEER_GROUP_ASSOCIATION_PRIORITY = 65536 + + +class PeerGroupAssociationsController(restcomm.GenericPathController): + + def __init__(self): + super(PeerGroupAssociationsController, self).__init__() + self.rpc_client = rpc_client.ManagerClient() + + @expose(generic=True, template='json') + def index(self): + # Route the request to specific methods with parameters + pass + + def _get_peer_group_association_list(self, context): + associations = db_api.peer_group_association_get_all(context) + association_list = [] + + for association in associations: + association_dict = db_api.peer_group_association_db_model_to_dict( + association) + # Remove the sync_message from the list response + association_dict.pop('sync-message', None) + association_list.append(association_dict) + + result = {'peer_group_associations': association_list} + return result + + @staticmethod + def _get_payload(request): + try: + payload = json.loads(request.body) + except Exception: + error_msg = 'Request body is malformed.' + LOG.exception(error_msg) + pecan.abort(400, _(error_msg)) + + if not isinstance(payload, dict): + pecan.abort(400, _('Invalid request body format')) + return payload + + def _validate_peer_group_leader_id(self, system_leader_id): + ks_client = psd_common.get_ks_client() + sysinv_client = SysinvClient( + dccommon_consts.DEFAULT_REGION_NAME, + ks_client.session, + endpoint=ks_client.endpoint_cache.get_endpoint('sysinv')) + system = sysinv_client.get_system() + return True if system.uuid == system_leader_id else False + + @index.when(method='GET', template='json') + def get(self, association_id=None): + """Get details about peer group association. + + :param association_id: ID of peer group association + """ + policy.authorize(peer_group_association_policy.POLICY_ROOT % "get", {}, + restcomm.extract_credentials_for_policy()) + context = restcomm.extract_context_from_environ() + + if association_id is None: + # List of peer group association requested + return self._get_peer_group_association_list(context) + elif not association_id.isdigit(): + pecan.abort(httpclient.BAD_REQUEST, + _('Peer Group Association ID must be an integer')) + + try: + association = db_api.peer_group_association_get(context, + association_id) + except exception.PeerGroupAssociationNotFound: + pecan.abort(httpclient.NOT_FOUND, + _('Peer Group Association not found')) + + return db_api.peer_group_association_db_model_to_dict(association) + + def _validate_peer_group_id(self, context, peer_group_id): + try: + db_api.subcloud_peer_group_get(context, peer_group_id) + except exception.SubcloudPeerGroupNotFound: + LOG.debug("Subcloud Peer Group Not Found, peer group id: %s" + % peer_group_id) + return False + except Exception as e: + LOG.warning("Get Subcloud Peer Group failed: %s; peer_group_id: %s" + % (e, peer_group_id)) + return False + return True + + def _validate_system_peer_id(self, context, system_peer_id): + try: + db_api.system_peer_get(context, system_peer_id) + except exception.SystemPeerNotFound: + LOG.debug("System Peer Not Found, system peer id: %s" + % system_peer_id) + return False + except Exception as e: + LOG.warning("Get System Peer failed: %s; system_peer_id: %s" + % (e, system_peer_id)) + return False + return True + + def _validate_peer_group_priority(self, peer_group_priority): + try: + # Check the value is an integer + val = int(peer_group_priority) + except ValueError: + LOG.debug("Peer Group Priority is not Integer: %s" + % peer_group_priority) + return False + # Less than min or greater than max priority is not supported. + if val < MIN_PEER_GROUP_ASSOCIATION_PRIORITY or \ + val > MAX_PEER_GROUP_ASSOCIATION_PRIORITY: + LOG.debug("Invalid Peer Group Priority out of support range: %s" + % peer_group_priority) + return False + return True + + @index.when(method='POST', template='json') + def post(self): + """Create a new peer group association.""" + policy.authorize(peer_group_association_policy.POLICY_ROOT % + "create", {}, + restcomm.extract_credentials_for_policy()) + context = restcomm.extract_context_from_environ() + + payload = self._get_payload(request) + if not payload: + pecan.abort(httpclient.BAD_REQUEST, _('Body required')) + + # Validate payload + peer_group_id = payload.get('peer_group_id') + if not self._validate_peer_group_id(context, peer_group_id): + pecan.abort(httpclient.BAD_REQUEST, _('Invalid peer_group_id')) + + system_peer_id = payload.get('system_peer_id') + if not self._validate_system_peer_id(context, system_peer_id): + pecan.abort(httpclient.BAD_REQUEST, _('Invalid system_peer_id')) + + peer_group_priority = payload.get('peer_group_priority') + peer_group = db_api.subcloud_peer_group_get(context, peer_group_id) + + if (peer_group.group_priority == 0 and not peer_group_priority) or \ + (peer_group.group_priority > 0 and peer_group_priority): + pecan.abort(httpclient.BAD_REQUEST, + _('Peer Group Association create with peer_group_' + 'priority is required when the subcloud peer group ' + 'priority is 0, and it is not allowed when the ' + 'subcloud peer group priority is greater than 0.')) + + if peer_group_priority and not self._validate_peer_group_priority( + peer_group_priority): + pecan.abort(httpclient.BAD_REQUEST, + _('Invalid peer_group_priority')) + + sync_enabled = peer_group.group_priority == 0 + + # only one combination of peer_group_id + system_peer_id can exists + association = None + try: + association = db_api.\ + peer_group_association_get_by_peer_group_and_system_peer_id( + context, + peer_group_id, + system_peer_id) + except exception.PeerGroupAssociationCombinationNotFound: + LOG.warning("Peer Group Association Combination Not Found, " + "peer_group_id: %s, system_peer_id: %s" + % (peer_group_id, system_peer_id)) + except Exception as e: + LOG.warning("Peer Group Association get failed: %s;" + "peer_group_id: %s, system_peer_id: %s" + % (e, peer_group_id, system_peer_id)) + pecan.abort(httpclient.INTERNAL_SERVER_ERROR, + _('peer_group_association_get_by_peer_group_and_' + 'system_peer_id failed: %s' % e)) + if association: + LOG.info("Failed to create Peer group association, association \ + with peer_group_id:[%s],system_peer_id:[%s] \ + already exists" % (peer_group_id, system_peer_id)) + pecan.abort(httpclient.BAD_REQUEST, + _('A Peer group association with same peer_group_id, ' + 'system_peer_id already exists')) + + # Create the peer group association + try: + sync_status = consts.ASSOCIATION_SYNC_STATUS_SYNCING if \ + sync_enabled else consts.ASSOCIATION_SYNC_STATUS_DISABLED + association = db_api.peer_group_association_create( + context, peer_group_id, system_peer_id, peer_group_priority, + sync_status) + + if sync_enabled: + # Sync the subcloud peer group to peer site + self.rpc_client.sync_subcloud_peer_group(context, association.id) + return db_api.peer_group_association_db_model_to_dict(association) + except RemoteError as e: + pecan.abort(httpclient.UNPROCESSABLE_ENTITY, e.value) + except Exception as e: + LOG.exception(e) + pecan.abort(httpclient.INTERNAL_SERVER_ERROR, + _('Unable to create peer group association')) + + @index.when(method='PATCH', template='json') + def patch(self, association_id, sync=False): + """Update a peer group association. + + :param association_id: ID of peer group association to update + :param sync: sync action that sync the peer group + """ + + policy.authorize(peer_group_association_policy.POLICY_ROOT % "modify", + {}, restcomm.extract_credentials_for_policy()) + context = restcomm.extract_context_from_environ() + if association_id is None: + pecan.abort(httpclient.BAD_REQUEST, + _('Peer Group Association ID required')) + elif not association_id.isdigit(): + pecan.abort(httpclient.BAD_REQUEST, + _('Peer Group Association ID must be an integer')) + + try: + association = db_api.peer_group_association_get(context, + association_id) + except exception.PeerGroupAssociationNotFound: + pecan.abort(httpclient.NOT_FOUND, + _('Peer Group Association not found')) + + sync_disabled = association.sync_status == consts.\ + ASSOCIATION_SYNC_STATUS_DISABLED + if sync_disabled: + pecan.abort(httpclient.BAD_REQUEST, + _('Peer Group Association sync or update is not allowed' + ' when the sync_status is disabled.')) + + if sync: + peer_group = db_api.subcloud_peer_group_get( + context, association.peer_group_id) + if not self._validate_peer_group_leader_id(peer_group. + system_leader_id): + pecan.abort(httpclient.BAD_REQUEST, + _('Peer Group Association sync is not allowed when ' + 'the subcloud peer group system_leader_id is not ' + 'the current system controller UUID.')) + try: + # Sync the subcloud peer group to peer site + self.rpc_client.sync_subcloud_peer_group(context, + association.id) + association = db_api.peer_group_association_update( + context, id=association_id, + sync_status=consts.ASSOCIATION_SYNC_STATUS_SYNCING, + sync_message='None') + return db_api.peer_group_association_db_model_to_dict( + association) + except RemoteError as e: + pecan.abort(httpclient.UNPROCESSABLE_ENTITY, e.value) + except Exception as e: + # additional exceptions. + LOG.exception(e) + pecan.abort(httpclient.INTERNAL_SERVER_ERROR, + _('Unable to sync peer group association')) + + payload = self._get_payload(request) + if not payload: + pecan.abort(httpclient.BAD_REQUEST, _('Body required')) + + peer_group_priority = payload.get('peer_group_priority') + # Check value is not None or empty before calling validate + if not peer_group_priority: + pecan.abort(httpclient.BAD_REQUEST, _('nothing to update')) + if not self._validate_peer_group_priority(peer_group_priority): + pecan.abort(httpclient.BAD_REQUEST, + _('Invalid peer_group_priority')) + + try: + # Ask dcmanager-manager to update the subcloud peer group priority + # to peer site. It will do the real work... + return self.rpc_client.update_subcloud_peer_group( + context, association.id, peer_group_priority) + except RemoteError as e: + pecan.abort(httpclient.UNPROCESSABLE_ENTITY, e.value) + except Exception as e: + # additional exceptions. + LOG.exception(e) + pecan.abort(httpclient.INTERNAL_SERVER_ERROR, + _('Unable to update peer group association')) + + @index.when(method='delete', template='json') + def delete(self, association_id): + """Delete the peer group association. + + :param association_id: ID of peer group association to delete + """ + policy.authorize(peer_group_association_policy.POLICY_ROOT % "delete", + {}, restcomm.extract_credentials_for_policy()) + context = restcomm.extract_context_from_environ() + + if association_id is None: + pecan.abort(httpclient.BAD_REQUEST, + _('Peer Group Association ID required')) + # Validate the ID + if not association_id.isdigit(): + pecan.abort(httpclient.BAD_REQUEST, + _('Peer Group Association ID must be an integer')) + + try: + association = db_api.peer_group_association_get(context, + association_id) + sync_disabled = association.sync_status == consts.\ + ASSOCIATION_SYNC_STATUS_DISABLED + if sync_disabled: + return db_api.peer_group_association_destroy(context, + association_id) + # Ask system-peer-manager to delete the association. + # It will do all the real work... + return self.rpc_client.delete_peer_group_association( + context, association_id) + except exception.PeerGroupAssociationNotFound: + pecan.abort(httpclient.NOT_FOUND, + _('Peer Group Association not found')) + except RemoteError as e: + pecan.abort(httpclient.UNPROCESSABLE_ENTITY, e.value) + except Exception as e: + LOG.exception(e) + pecan.abort(httpclient.INTERNAL_SERVER_ERROR, + _('Unable to delete peer group association')) diff --git a/distributedcloud/dcmanager/api/controllers/v1/root.py b/distributedcloud/dcmanager/api/controllers/v1/root.py index d0df7b33e..7031fdde7 100644 --- a/distributedcloud/dcmanager/api/controllers/v1/root.py +++ b/distributedcloud/dcmanager/api/controllers/v1/root.py @@ -18,6 +18,7 @@ import pecan from dcmanager.api.controllers.v1 import alarm_manager from dcmanager.api.controllers.v1 import notifications +from dcmanager.api.controllers.v1 import peer_group_association from dcmanager.api.controllers.v1 import phased_subcloud_deploy from dcmanager.api.controllers.v1 import subcloud_backup from dcmanager.api.controllers.v1 import subcloud_deploy @@ -58,6 +59,8 @@ class Controller(object): PhasedSubcloudDeployController sub_controllers["subcloud-peer-groups"] = \ subcloud_peer_group.SubcloudPeerGroupsController + sub_controllers["peer-group-associations"] = \ + peer_group_association.PeerGroupAssociationsController sub_controllers["system-peers"] = system_peers.\ SystemPeersController diff --git a/distributedcloud/dcmanager/api/controllers/v1/subcloud_peer_group.py b/distributedcloud/dcmanager/api/controllers/v1/subcloud_peer_group.py index 9f262c6d1..588de2538 100644 --- a/distributedcloud/dcmanager/api/controllers/v1/subcloud_peer_group.py +++ b/distributedcloud/dcmanager/api/controllers/v1/subcloud_peer_group.py @@ -485,16 +485,13 @@ class SubcloudPeerGroupsController(restcomm.GenericPathController): pecan.abort(httpclient.NOT_FOUND, _('Subcloud Peer Group not found')) LOG.info("Handling delete subcloud peer group request for: %s" % group) - # TODO(Jon): uncomment in Association of System and Peer Group management commit - ''' - # a peer group may not be deleted if it is used by any associations + # A peer group cannot be deleted if it is used by any associations association = db_api.peer_group_association_get_by_peer_group_id(context, group.id) if len(association) > 0: pecan.abort(httpclient.BAD_REQUEST, _("Cannot delete a peer group " "which is associated with a system peer.")) - ''' try: db_api.subcloud_peer_group_destroy(context, group.id) # Disassociate the subcloud. diff --git a/distributedcloud/dcmanager/api/controllers/v1/subclouds.py b/distributedcloud/dcmanager/api/controllers/v1/subclouds.py index 8c38a81ff..95ceb36bd 100644 --- a/distributedcloud/dcmanager/api/controllers/v1/subclouds.py +++ b/distributedcloud/dcmanager/api/controllers/v1/subclouds.py @@ -514,7 +514,12 @@ class SubcloudsController(object): if 'secondary' not in payload: psd_common.validate_sysadmin_password(payload) - psd_common.subcloud_region_create(payload, context) + # Use the region_name if it has been provided in the payload. + # The typical scenario is adding a secondary subcloud from + # peer site where subcloud region_name is known and can be + # put into the payload of the subcloud add request. + if 'region_name' not in payload: + psd_common.subcloud_region_create(payload, context) psd_common.pre_deploy_create(payload, context, request) @@ -524,8 +529,14 @@ class SubcloudsController(object): # Ask dcmanager-manager to add the subcloud. # It will do all the real work... - self.dcmanager_rpc_client.add_subcloud( - context, subcloud.id, payload) + # If the subcloud is secondary, it will be synchronous operation. + # A normal subcloud add will be a synchronous operation. + if 'secondary' in payload: + self.dcmanager_rpc_client.add_secondary_subcloud( + context, subcloud.id, payload) + else: + self.dcmanager_rpc_client.add_subcloud( + context, subcloud.id, payload) return db_api.subcloud_db_model_to_dict(subcloud) except RemoteError as e: diff --git a/distributedcloud/dcmanager/api/controllers/v1/system_peers.py b/distributedcloud/dcmanager/api/controllers/v1/system_peers.py index 6f78e8604..f54c0901c 100755 --- a/distributedcloud/dcmanager/api/controllers/v1/system_peers.py +++ b/distributedcloud/dcmanager/api/controllers/v1/system_peers.py @@ -79,6 +79,10 @@ class SystemPeersController(restcomm.GenericPathController): pecan.abort(400, _('Invalid request body format')) return payload + def _get_peer_group_list_for_system_peer(self, context, peer_id): + peer_groups = db_api.peer_group_get_for_system_peer(context, peer_id) + return utils.subcloud_peer_group_db_list_to_dict(peer_groups) + def _get_system_peer_list(self, context): peers = db_api.system_peer_get_all(context) @@ -92,10 +96,16 @@ class SystemPeersController(restcomm.GenericPathController): return result @index.when(method='GET', template='json') - def get(self, peer_ref=None): - """Get details about system peer. + def get(self, peer_ref=None, subcloud_peer_groups=False): + """Retrieve information about a system peer. + + This function allows you to retrieve details about a specific + system peer or obtain a list of subcloud peer groups associated with + a specific system peer. :param peer_ref: ID or UUID or Name of system peer + :param subcloud_peer_groups: If this request should return subcloud + peer groups """ policy.authorize(system_peer_policy.POLICY_ROOT % "get", {}, restcomm.extract_credentials_for_policy()) @@ -108,6 +118,8 @@ class SystemPeersController(restcomm.GenericPathController): peer = utils.system_peer_get_by_ref(context, peer_ref) if peer is None: pecan.abort(httpclient.NOT_FOUND, _('System Peer not found')) + if subcloud_peer_groups: + return self._get_peer_group_list_for_system_peer(context, peer.id) system_peer_dict = db_api.system_peer_db_model_to_dict(peer) return system_peer_dict @@ -467,13 +479,14 @@ class SystemPeersController(restcomm.GenericPathController): if peer is None: pecan.abort(httpclient.NOT_FOUND, _('System Peer not found')) - # TODO(jon): Add this back in when we have peer group associations - # a system peer may not be deleted if it is use by any associations - # association = db_api.peer_group_association_get_by_system_peer_id(context, - # str(peer.id)) - # if len(association) > 0: - # pecan.abort(httpclient.BAD_REQUEST, - # _('System peer associated with peer group')) + # A system peer cannot be deleted if it is used by any associations + association = db_api.\ + peer_group_association_get_by_system_peer_id(context, + str(peer.id)) + if len(association) > 0: + pecan.abort(httpclient.BAD_REQUEST, + _('Cannot delete a system peer which is ' + 'associated with peer group.')) try: db_api.system_peer_destroy(context, peer.id) diff --git a/distributedcloud/dcmanager/api/policies/__init__.py b/distributedcloud/dcmanager/api/policies/__init__.py index ea8db37cf..80df4200a 100644 --- a/distributedcloud/dcmanager/api/policies/__init__.py +++ b/distributedcloud/dcmanager/api/policies/__init__.py @@ -8,6 +8,7 @@ import itertools from dcmanager.api.policies import alarm_manager from dcmanager.api.policies import base +from dcmanager.api.policies import peer_group_association from dcmanager.api.policies import phased_subcloud_deploy from dcmanager.api.policies import subcloud_backup from dcmanager.api.policies import subcloud_deploy @@ -31,5 +32,6 @@ def list_rules(): subcloud_backup.list_rules(), phased_subcloud_deploy.list_rules(), subcloud_peer_group.list_rules(), + peer_group_association.list_rules(), system_peers.list_rules() ) diff --git a/distributedcloud/dcmanager/api/policies/peer_group_association.py b/distributedcloud/dcmanager/api/policies/peer_group_association.py new file mode 100644 index 000000000..bf610fd88 --- /dev/null +++ b/distributedcloud/dcmanager/api/policies/peer_group_association.py @@ -0,0 +1,72 @@ +# +# Copyright (c) 2023 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +from dcmanager.api.policies import base +from oslo_policy import policy + +POLICY_ROOT = 'dc_api:peer_group_associations:%s' + + +peer_group_associations_rules = [ + + # CRUD of peer_group_associations entity + policy.DocumentedRuleDefault( + name=POLICY_ROOT % 'create', + check_str='rule:' + base.ADMIN_IN_SYSTEM_PROJECTS, + description="Create peer group association.", + operations=[ + { + 'method': 'POST', + 'path': '/v1.0/peer-group-associations' + } + ] + ), + policy.DocumentedRuleDefault( + name=POLICY_ROOT % 'delete', + check_str='rule:' + base.ADMIN_IN_SYSTEM_PROJECTS, + description="Delete peer group association.", + operations=[ + { + 'method': 'DELETE', + 'path': '/v1.0/peer-group-associations/{associate_id}' + } + ] + ), + policy.DocumentedRuleDefault( + name=POLICY_ROOT % 'get', + check_str='rule:' + base.READER_IN_SYSTEM_PROJECTS, + description="Get peer group associations.", + operations=[ + { + 'method': 'GET', + 'path': '/v1.0/peer-group-associations' + }, + { + 'method': 'GET', + 'path': '/v1.0/peer-group-associations/{associate_id}' + } + ] + ), + policy.DocumentedRuleDefault( + name=POLICY_ROOT % 'modify', + check_str='rule:' + base.ADMIN_IN_SYSTEM_PROJECTS, + description="Modify peer group association.", + operations=[ + { + 'method': 'PATCH', + 'path': '/v1.0/peer-group-associations/{associate_id}' + }, + { + 'method': 'PATCH', + 'path': '/v1.0/peer-group-associations/{associate_id}/sync' + } + ] + ) +] + + +def list_rules(): + return peer_group_associations_rules diff --git a/distributedcloud/dcmanager/api/policies/system_peers.py b/distributedcloud/dcmanager/api/policies/system_peers.py index d587590f1..78ef06130 100755 --- a/distributedcloud/dcmanager/api/policies/system_peers.py +++ b/distributedcloud/dcmanager/api/policies/system_peers.py @@ -41,9 +41,15 @@ system_peers_rules = [ 'method': 'GET', 'path': '/v1.0/system-peers' }, + # Show details of a specified System Peer { 'method': 'GET', 'path': '/v1.0/system-peers/{system_peer}' + }, + # List Subcloud Peer Groups associated with the given System Peer + { + 'method': 'GET', + 'path': '/v1.0/system-peers/{system_peer}/subcloud-peer-groups' } ] ), diff --git a/distributedcloud/dcmanager/common/consts.py b/distributedcloud/dcmanager/common/consts.py index 223bd24da..50c3c3e38 100644 --- a/distributedcloud/dcmanager/common/consts.py +++ b/distributedcloud/dcmanager/common/consts.py @@ -445,3 +445,13 @@ STATES_FOR_SUBCLOUD_RENAME = [DEPLOY_STATE_DONE, # batch rehome manage state wait timeout BATCH_REHOME_MGMT_STATES_TIMEOUT = 900 + +# System peer heartbeat status +SYSTEM_PEER_HEARTBEAT_STATUS_ALIVE = 'alive' +SYSTEM_PEER_HEARTBEAT_STATUS_FAILURE = 'failure' + +# Peer group association sync status +ASSOCIATION_SYNC_STATUS_SYNCING = 'syncing' +ASSOCIATION_SYNC_STATUS_SYNCED = 'synced' +ASSOCIATION_SYNC_STATUS_FAILED = 'failed' +ASSOCIATION_SYNC_STATUS_DISABLED = 'disabled' diff --git a/distributedcloud/dcmanager/common/exceptions.py b/distributedcloud/dcmanager/common/exceptions.py index c7d8fa0de..5ff852cd7 100644 --- a/distributedcloud/dcmanager/common/exceptions.py +++ b/distributedcloud/dcmanager/common/exceptions.py @@ -177,6 +177,26 @@ class SubcloudPeerGroupNotFound(NotFound): message = _("Subcloud Peer Group with id %(group_id)s doesn't exist.") +class PeerGroupAssociationCombinationNotFound(NotFound): + message = _("Peer Group Association between peer group: %(peer_group_id)s " + "and system peer: %(system_peer_id)s doesn't exist.") + + +class PeerGroupAssociationTargetNotMatch(NotFound): + message = _("Peer Group Association with peer site controller " + "UUID %(uuid)s doesn't match.") + + +class SubcloudPeerGroupHasWrongPriority(DCManagerException): + message = _("Subcloud Peer group of peer site has wrong " + "priority %(priority)s.") + + +class PeerGroupAssociationNotFound(NotFound): + message = _("Peer Group Association with id %(association_id)s " + "doesn't exist.") + + class SubcloudGroupNameViolation(DCManagerException): message = _("Default Subcloud Group name cannot be changed or reused.") diff --git a/distributedcloud/dcmanager/db/api.py b/distributedcloud/dcmanager/db/api.py index 1ca1182d7..e782e8f7c 100644 --- a/distributedcloud/dcmanager/db/api.py +++ b/distributedcloud/dcmanager/db/api.py @@ -381,6 +381,7 @@ def system_peer_db_model_to_dict(system_peer): "heartbeat-failure-policy": system_peer.heartbeat_failure_policy, "heartbeat-maintenance-timeout": system_peer. heartbeat_maintenance_timeout, + "heartbeat-status": system_peer.heartbeat_status, "created-at": system_peer.created_at, "updated-at": system_peer.updated_at} return result @@ -427,6 +428,11 @@ def system_peer_get_all(context): return IMPL.system_peer_get_all(context) +def peer_group_get_for_system_peer(context, peer_id): + """Get subcloud peer groups associated with a system peer.""" + return IMPL.peer_group_get_for_system_peer(context, peer_id) + + def system_peer_update(context, peer_id, peer_uuid, peer_name, endpoint, username, password, @@ -435,7 +441,8 @@ def system_peer_update(context, peer_id, heartbeat_interval, heartbeat_failure_threshold, heartbeat_failure_policy, - heartbeat_maintenance_timeout): + heartbeat_maintenance_timeout, + heartbeat_status=None): """Update the system peer or raise if it does not exist.""" return IMPL.system_peer_update(context, peer_id, peer_uuid, peer_name, @@ -445,7 +452,8 @@ def system_peer_update(context, peer_id, heartbeat_interval, heartbeat_failure_threshold, heartbeat_failure_policy, - heartbeat_maintenance_timeout) + heartbeat_maintenance_timeout, + heartbeat_status) def system_peer_destroy(context, peer_id): @@ -530,6 +538,76 @@ def subcloud_peer_group_update(context, group_id, peer_group_name, group_priorit ################### +################### +# peer_group_association +def peer_group_association_db_model_to_dict(peer_group_association): + """Convert peer_group_association db model to dictionary.""" + result = {"id": peer_group_association.id, + "peer-group-id": peer_group_association.peer_group_id, + "system-peer-id": peer_group_association.system_peer_id, + "peer-group-priority": peer_group_association.peer_group_priority, + "sync-status": peer_group_association.sync_status, + "sync-message": peer_group_association.sync_message, + "created-at": peer_group_association.created_at, + "updated-at": peer_group_association.updated_at} + return result + + +def peer_group_association_create(context, peer_group_id, system_peer_id, + peer_group_priority, sync_status=None, + sync_message=None): + """Create a peer_group_association.""" + return IMPL.peer_group_association_create(context, + peer_group_id, + system_peer_id, + peer_group_priority, + sync_status, + sync_message) + + +def peer_group_association_update(context, id, peer_group_priority=None, + sync_status=None, sync_message=None): + """Update the system peer or raise if it does not exist.""" + return IMPL.peer_group_association_update(context, id, peer_group_priority, + sync_status, sync_message) + + +def peer_group_association_destroy(context, id): + """Destroy the peer_group_association or raise if it does not exist.""" + return IMPL.peer_group_association_destroy(context, id) + + +def peer_group_association_get(context, id): + """Retrieve a peer_group_association or raise if it does not exist.""" + return IMPL.peer_group_association_get(context, id) + + +def peer_group_association_get_all(context): + """Retrieve all peer_group_associations.""" + return IMPL.peer_group_association_get_all(context) + + +def peer_group_association_get_by_peer_group_and_system_peer_id(context, + peer_group_id, + system_peer_id): + """Get peer group associations by peer_group_id and system_peer_id.""" + return IMPL.peer_group_association_get_by_peer_group_and_system_peer_id( + context, peer_group_id, system_peer_id) + + +def peer_group_association_get_by_peer_group_id(context, peer_group_id): + """Get the peer_group_association list by peer_group_id""" + return IMPL.peer_group_association_get_by_peer_group_id(context, + peer_group_id) + + +def peer_group_association_get_by_system_peer_id(context, system_peer_id): + """Get the peer_group_association list by system_peer_id""" + return IMPL.peer_group_association_get_by_system_peer_id(context, + system_peer_id) +################### + + def sw_update_strategy_db_model_to_dict(sw_update_strategy): """Convert sw update db model to dictionary.""" result = {"id": sw_update_strategy.id, diff --git a/distributedcloud/dcmanager/db/sqlalchemy/api.py b/distributedcloud/dcmanager/db/sqlalchemy/api.py index 6d08d5c47..3de3c10de 100644 --- a/distributedcloud/dcmanager/db/sqlalchemy/api.py +++ b/distributedcloud/dcmanager/db/sqlalchemy/api.py @@ -863,6 +863,18 @@ def system_peer_get_all(context): return result +# This method returns all subcloud peer groups for a particular system peer +@require_context +def peer_group_get_for_system_peer(context, peer_id): + return model_query(context, models.SubcloudPeerGroup). \ + join(models.PeerGroupAssociation, models.SubcloudPeerGroup.id == + models.PeerGroupAssociation.peer_group_id). \ + filter(models.SubcloudPeerGroup.deleted == 0). \ + filter(models.PeerGroupAssociation.system_peer_id == peer_id). \ + order_by(models.SubcloudPeerGroup.id). \ + all() + + @require_admin_context def system_peer_create(context, peer_uuid, peer_name, @@ -872,7 +884,8 @@ def system_peer_create(context, heartbeat_interval=60, heartbeat_failure_threshold=3, heartbeat_failure_policy="alarm", - heartbeat_maintenance_timeout=600): + heartbeat_maintenance_timeout=600, + heartbeat_status="created"): with write_session() as session: system_peer_ref = models.SystemPeer() system_peer_ref.peer_uuid = peer_uuid @@ -888,6 +901,7 @@ def system_peer_create(context, system_peer_ref.heartbeat_failure_policy = heartbeat_failure_policy system_peer_ref.heartbeat_maintenance_timeout = \ heartbeat_maintenance_timeout + system_peer_ref.heartbeat_status = heartbeat_status session.add(system_peer_ref) return system_peer_ref @@ -901,7 +915,8 @@ def system_peer_update(context, peer_id, heartbeat_interval=None, heartbeat_failure_threshold=None, heartbeat_failure_policy=None, - heartbeat_maintenance_timeout=None): + heartbeat_maintenance_timeout=None, + heartbeat_status=None): with write_session() as session: system_peer_ref = system_peer_get(context, peer_id) if peer_uuid is not None: @@ -928,6 +943,8 @@ def system_peer_update(context, peer_id, if heartbeat_maintenance_timeout is not None: system_peer_ref.heartbeat_maintenance_timeout = \ heartbeat_maintenance_timeout + if heartbeat_status is not None: + system_peer_ref.heartbeat_status = heartbeat_status system_peer_ref.save(session) return system_peer_ref @@ -1201,6 +1218,126 @@ def subcloud_peer_group_update(context, ########################## +########################## +# peer group association +########################## +@require_admin_context +def peer_group_association_create(context, + peer_group_id, + system_peer_id, + peer_group_priority, + sync_status, + sync_message): + with write_session() as session: + peer_group_association_ref = models.PeerGroupAssociation() + peer_group_association_ref.peer_group_id = peer_group_id + peer_group_association_ref.system_peer_id = system_peer_id + peer_group_association_ref.peer_group_priority = peer_group_priority + peer_group_association_ref.sync_status = sync_status + peer_group_association_ref.sync_message = sync_message + session.add(peer_group_association_ref) + return peer_group_association_ref + + +@require_admin_context +def peer_group_association_update(context, + associate_id, + peer_group_priority=None, + sync_status=None, + sync_message=None): + with write_session() as session: + association_ref = peer_group_association_get(context, associate_id) + if peer_group_priority is not None: + association_ref.peer_group_priority = peer_group_priority + if sync_status is not None: + association_ref.sync_status = sync_status + if sync_message is not None: + association_ref.sync_message = sync_message + association_ref.save(session) + return association_ref + + +@require_admin_context +def peer_group_association_destroy(context, association_id): + with write_session() as session: + association_ref = peer_group_association_get(context, association_id) + session.delete(association_ref) + + +@require_context +def peer_group_association_get(context, association_id): + try: + result = model_query(context, models.PeerGroupAssociation). \ + filter_by(deleted=0). \ + filter_by(id=association_id). \ + one() + except NoResultFound: + raise exception.PeerGroupAssociationNotFound( + association_id=association_id) + except MultipleResultsFound: + raise exception.InvalidParameterValue( + err="Multiple entries found for peer group association %s" % + association_id) + + return result + + +@require_context +def peer_group_association_get_all(context): + result = model_query(context, models.PeerGroupAssociation). \ + filter_by(deleted=0). \ + order_by(models.PeerGroupAssociation.id). \ + all() + + return result + + +# Each combination of 'peer_group_id' and 'system_peer_id' is unique +# and appears only once in the entries. +@require_context +def peer_group_association_get_by_peer_group_and_system_peer_id(context, + peer_group_id, + system_peer_id): + try: + result = model_query(context, models.PeerGroupAssociation). \ + filter_by(deleted=0). \ + filter_by(peer_group_id=peer_group_id). \ + filter_by(system_peer_id=system_peer_id). \ + one() + except NoResultFound: + raise exception.PeerGroupAssociationCombinationNotFound( + peer_group_id=peer_group_id, system_peer_id=system_peer_id) + except MultipleResultsFound: + # This exception should never happen due to the UNIQUE setting for name + raise exception.InvalidParameterValue( + err="Multiple entries found for peer group association %s,%s" % + (peer_group_id, system_peer_id)) + return result + + +@require_context +def peer_group_association_get_by_peer_group_id(context, peer_group_id): + result = model_query(context, models.PeerGroupAssociation). \ + filter_by(deleted=0). \ + filter_by(peer_group_id=peer_group_id). \ + order_by(models.PeerGroupAssociation.id). \ + all() + + return result + + +@require_context +def peer_group_association_get_by_system_peer_id(context, system_peer_id): + result = model_query(context, models.PeerGroupAssociation). \ + filter_by(deleted=0). \ + filter_by(system_peer_id=system_peer_id). \ + order_by(models.PeerGroupAssociation.id). \ + all() + + return result +########################## + + @require_context def strategy_step_get(context, subcloud_id): result = model_query(context, models.StrategyStep). \ diff --git a/distributedcloud/dcmanager/db/sqlalchemy/migrate_repo/versions/014_add_subcloud_peer_group_and_association.py b/distributedcloud/dcmanager/db/sqlalchemy/migrate_repo/versions/014_add_subcloud_peer_group_and_association.py index af04d98f2..af2bedccd 100644 --- a/distributedcloud/dcmanager/db/sqlalchemy/migrate_repo/versions/014_add_subcloud_peer_group_and_association.py +++ b/distributedcloud/dcmanager/db/sqlalchemy/migrate_repo/versions/014_add_subcloud_peer_group_and_association.py @@ -72,6 +72,33 @@ def upgrade(migrate_engine): ) system_peer.create() + # Declare the new peer_group_association table + peer_group_association = sqlalchemy.Table( + 'peer_group_association', meta, + sqlalchemy.Column('id', sqlalchemy.Integer, + primary_key=True, + autoincrement=True, + nullable=False), + sqlalchemy.Column('peer_group_id', sqlalchemy.Integer, + sqlalchemy.ForeignKey('subcloud_peer_group.id', + ondelete='CASCADE')), + sqlalchemy.Column('system_peer_id', sqlalchemy.Integer, + sqlalchemy.ForeignKey('system_peer.id', + ondelete='CASCADE')), + sqlalchemy.Column('peer_group_priority', sqlalchemy.Integer), + sqlalchemy.Column('sync_status', sqlalchemy.String(255)), + sqlalchemy.Column('sync_message', sqlalchemy.Text), + sqlalchemy.Column('reserved_1', sqlalchemy.Text), + sqlalchemy.Column('reserved_2', sqlalchemy.Text), + sqlalchemy.Column('created_at', sqlalchemy.DateTime), + sqlalchemy.Column('updated_at', sqlalchemy.DateTime), + sqlalchemy.Column('deleted_at', sqlalchemy.DateTime), + sqlalchemy.Column('deleted', sqlalchemy.Integer, default=0), + mysql_engine=ENGINE, + mysql_charset=CHARSET + ) + peer_group_association.create() + def downgrade(migrate_engine): raise NotImplementedError('Database downgrade is unsupported.') diff --git a/distributedcloud/dcmanager/db/sqlalchemy/models.py b/distributedcloud/dcmanager/db/sqlalchemy/models.py index 2ec76318a..51815c4b7 100644 --- a/distributedcloud/dcmanager/db/sqlalchemy/models.py +++ b/distributedcloud/dcmanager/db/sqlalchemy/models.py @@ -117,6 +117,7 @@ class SystemPeer(BASE, DCManagerBase): heartbeat_failure_threshold = Column(Integer) heartbeat_failure_policy = Column(String(255)) heartbeat_maintenance_timeout = Column(Integer) + heartbeat_status = Column(String(255)) class SubcloudGroup(BASE, DCManagerBase): @@ -145,6 +146,19 @@ class SubcloudPeerGroup(BASE, DCManagerBase): system_leader_name = Column(String(255)) +class PeerGroupAssociation(BASE, DCManagerBase): + """Represents a Peer Group Association""" + + __tablename__ = 'peer_group_association' + + id = Column(Integer, primary_key=True, autoincrement=True, nullable=False) + peer_group_id = Column(Integer) + system_peer_id = Column(Integer) + peer_group_priority = Column(Integer) + sync_status = Column(String(255)) + sync_message = Column(Text()) + + class Subcloud(BASE, DCManagerBase): """Represents a subcloud""" diff --git a/distributedcloud/dcmanager/manager/service.py b/distributedcloud/dcmanager/manager/service.py index 6a479f150..a411dbc6a 100644 --- a/distributedcloud/dcmanager/manager/service.py +++ b/distributedcloud/dcmanager/manager/service.py @@ -33,6 +33,7 @@ from dcmanager.common.i18n import _ from dcmanager.common import messaging as rpc_messaging from dcmanager.common import utils from dcmanager.manager.subcloud_manager import SubcloudManager +from dcmanager.manager.system_peer_manager import SystemPeerManager CONF = cfg.CONF LOG = logging.getLogger(__name__) @@ -88,6 +89,7 @@ class DCManagerService(service.Service): def init_managers(self): self.subcloud_manager = SubcloudManager() + self.syspeer_manager = SystemPeerManager() def start(self): utils.set_open_file_limit(cfg.CONF.worker_rlimit_nofile) @@ -300,6 +302,21 @@ class DCManagerService(service.Service): payload['peer_group']) return self.subcloud_manager.batch_migrate_subcloud(context, payload) + @request_context + def sync_subcloud_peer_group(self, context, association_id, + sync_subclouds=True, priority=None): + LOG.info("Handling sync_subcloud_peer_group request for: %s", + association_id) + return self.syspeer_manager.sync_subcloud_peer_group( + context, association_id, sync_subclouds, priority) + + @request_context + def delete_peer_group_association(self, context, association_id): + LOG.info("Handling delete_peer_group_association request for: %s", + association_id) + return self.syspeer_manager.delete_peer_group_association( + context, association_id) + def _stop_rpc_server(self): # Stop RPC connection to prevent new requests LOG.debug(_("Attempting to stop RPC service...")) diff --git a/distributedcloud/dcmanager/manager/system_peer_manager.py b/distributedcloud/dcmanager/manager/system_peer_manager.py new file mode 100644 index 000000000..9493d92b4 --- /dev/null +++ b/distributedcloud/dcmanager/manager/system_peer_manager.py @@ -0,0 +1,508 @@ +# +# Copyright (c) 2023 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +import base64 +import functools +import json +import tempfile + +from eventlet import greenpool +from oslo_log import log as logging +import yaml + +from dccommon import consts as dccommon_consts +from dccommon.drivers.openstack.dcmanager_v1 import DcmanagerClient +from dccommon.drivers.openstack.peer_site import PeerSiteDriver +from dccommon.drivers.openstack.sysinv_v1 import SysinvClient +from dccommon import exceptions as dccommon_exceptions +from dcmanager.common import consts +from dcmanager.common import exceptions +from dcmanager.common.i18n import _ +from dcmanager.common import manager +from dcmanager.db import api as db_api + +LOG = logging.getLogger(__name__) + +TEMP_BOOTSTRAP_PREFIX = 'peer_subcloud_bootstrap_yaml' +MAX_PARALLEL_SUBCLOUD_SYNC = 10 +VERIFY_SUBCLOUD_SYNC_VALID = 'valid' +VERIFY_SUBCLOUD_SYNC_IGNORE = 'ignore' + + +class SystemPeerManager(manager.Manager): + """Manages tasks related to system peers.""" + + def __init__(self, *args, **kwargs): + LOG.debug(_('SystemPeerManager initialization...')) + + super(SystemPeerManager, self).__init__( + service_name="system_peer_manager", *args, **kwargs) + + @staticmethod + def get_peer_ks_client(peer): + """This will get a new peer keystone client (and new token)""" + try: + os_client = PeerSiteDriver( + auth_url=peer.manager_endpoint, + username=peer.manager_username, + password=base64.b64decode( + peer.manager_password.encode("utf-8")).decode("utf-8"), + site_uuid=peer.peer_uuid) + return os_client.keystone_client + except Exception: + LOG.warn('Failure initializing KeystoneClient ' + f'for system peer {peer.peer_name}') + raise + + @staticmethod + def get_peer_sysinv_client(peer): + p_ks_client = SystemPeerManager.get_peer_ks_client(peer) + sysinv_endpoint = p_ks_client.session.get_endpoint( + service_type='platform', + region_name=dccommon_consts.DEFAULT_REGION_NAME, + interface=dccommon_consts.KS_ENDPOINT_PUBLIC) + return SysinvClient(dccommon_consts.DEFAULT_REGION_NAME, + p_ks_client.session, + endpoint_type=dccommon_consts. + KS_ENDPOINT_PUBLIC, + endpoint=sysinv_endpoint) + + @staticmethod + def get_peer_dc_client(peer): + p_ks_client = SystemPeerManager.get_peer_ks_client(peer) + dc_endpoint = p_ks_client.session.get_endpoint( + service_type='dcmanager', + region_name=dccommon_consts.SYSTEM_CONTROLLER_NAME, + interface=dccommon_consts.KS_ENDPOINT_PUBLIC) + return DcmanagerClient(dccommon_consts.SYSTEM_CONTROLLER_NAME, + p_ks_client.session, + endpoint=dc_endpoint) + + @staticmethod + def get_peer_subcloud(dc_client, subcloud_name): + """Get subcloud on peer site if exist. + + :param dc_client: the dcmanager client object + :param subcloud_ref: subcloud name needs to check + """ + try: + peer_subcloud = dc_client.get_subcloud(subcloud_name) + return peer_subcloud + except dccommon_exceptions.SubcloudNotFound: + LOG.warn(f"Subcloud {subcloud_name} does not exist on peer site.") + + @staticmethod + def is_subcloud_secondary(subcloud): + """Check if subcloud on peer site is secondary. + + :param subcloud: peer subcloud dictionary + """ + deploy_status = 'deploy-status' if 'deploy-status' in subcloud else \ + 'deploy_status' + if subcloud.get(deploy_status) not in ( + consts.DEPLOY_STATE_SECONDARY_FAILED, + consts.DEPLOY_STATE_SECONDARY): + return False + return True + + @staticmethod + def delete_peer_secondary_subcloud(dc_client, subcloud_ref): + """Delete secondary subcloud on peer site. + + :param dc_client: the dcmanager client object + :param subcloud_ref: subcloud name to delete + """ + peer_subcloud = SystemPeerManager.get_peer_subcloud(dc_client, + subcloud_ref) + if not peer_subcloud: + LOG.info(f"Skip delete Peer Site Subcloud {subcloud_ref} cause " + f"it doesn't exist.") + return + + is_secondary = SystemPeerManager.is_subcloud_secondary(peer_subcloud) + if not is_secondary: + LOG.info(f"Ignoring delete Peer Site Subcloud {subcloud_ref} " + f"as is not in secondary state.") + return + + dc_client.delete_subcloud(subcloud_ref) + LOG.info(f"Deleted Subcloud {subcloud_ref} on peer site.") + + @staticmethod + def _run_parallel_group_operation(op_type, op_function, thread_pool, + subclouds): + """Run parallel group operation on subclouds.""" + failed_subclouds = [] + processed = 0 + error_msg = {} # Dictinary to store error message for each subcloud + + for subcloud, success in thread_pool.imap(op_function, subclouds): + processed += 1 + + if not success: + failed_subclouds.append(subcloud) + if hasattr(subcloud, 'msg'): + error_msg[subcloud.name] = subcloud.msg + + completion = float(processed) / float(len(subclouds)) * 100 + remaining = len(subclouds) - processed + LOG.info("Processed subcloud %s for %s (operation %.0f%% " + "complete, %d subcloud(s) remaining)" % + (subcloud.name, op_type, completion, remaining)) + + return failed_subclouds, error_msg + + def _add_or_update_subcloud(self, dc_client, peer_controller_gateway_ip, + dc_peer_pg_id, subcloud): + """Add or update subcloud on peer site in parallel.""" + with tempfile.NamedTemporaryFile(prefix=TEMP_BOOTSTRAP_PREFIX, + suffix=".yaml", + mode='w') as temp_file: + subcloud_name = subcloud.get('name') + region_name = subcloud.get('region_name') + rehome_data = json.loads(subcloud.rehome_data) + subcloud_payload = rehome_data['saved_payload'] + + subcloud_payload['systemcontroller_gateway_address'] = \ + peer_controller_gateway_ip + + yaml.dump(subcloud_payload, temp_file) + + files = {"bootstrap_values": temp_file.name} + data = { + "bootstrap-address": subcloud_payload['bootstrap-address'], + "region_name": subcloud.region_name, + "location": subcloud.location, + "description": subcloud.description + } + + try: + # Sync subcloud information to peer site + peer_subcloud = self.get_peer_subcloud(dc_client, subcloud_name) + if peer_subcloud: + dc_peer_subcloud = dc_client.update_subcloud(subcloud_name, + files, data) + LOG.info(f"Updated Subcloud {dc_peer_subcloud.get('name')} " + "(region_name: " + f"{dc_peer_subcloud.get('region_name')}) on peer " + "site.") + else: + # Create subcloud on peer site if not exist + dc_peer_subcloud = dc_client. \ + add_subcloud_with_secondary_status(files, data) + LOG.info(f"Created Subcloud {dc_peer_subcloud.get('name')} " + "(region_name: " + f"{dc_peer_subcloud.get('region_name')}) on peer " + "site.") + LOG.debug(f"Updating subcloud {subcloud_name} (region_name: " + f"{region_name}) with subcloud peer group id " + f"{dc_peer_pg_id} on peer site.") + # Update subcloud associated peer group on peer site + dc_client.update_subcloud(subcloud_name, files=None, + data={"peer_group": str(dc_peer_pg_id)}) + return subcloud, True + except Exception as e: + subcloud.msg = str(e) # Store error message for subcloud + LOG.error(f"Failed to add/update Subcloud {subcloud_name} " + f"(region_name: {region_name}) " + f"on peer site: {str(e)}") + return subcloud, False + + def _is_valid_for_subcloud_sync(self, subcloud): + """Verify subcloud data for sync.""" + subcloud_name = subcloud.get('name') + region_name = subcloud.get('region_name') + + # Ignore the secondary subclouds to sync with peer site + if self.is_subcloud_secondary(subcloud): + LOG.info(f"Ignoring the Subcloud {subcloud_name} (region_name: " + f"{region_name}) in secondary status to sync with " + "peer site.") + return VERIFY_SUBCLOUD_SYNC_IGNORE + + # Verify subcloud payload data + rehome_json = subcloud.rehome_data + if not rehome_json: + msg = f"Subcloud {subcloud_name} (region_name: " + \ + f"{region_name}) does not have rehome_data." + return msg + + rehome_data = json.loads(rehome_json) + if 'saved_payload' not in rehome_data: + msg = f"Subcloud {subcloud_name} (region_name: " + \ + f"{region_name}) does not have saved_payload." + return msg + + subcloud_payload = rehome_data['saved_payload'] + if not subcloud_payload: + msg = f"Subcloud {subcloud_name} (region_name: " + \ + f"{region_name}) saved_payload is empty." + return msg + + if 'bootstrap-address' not in subcloud_payload: + msg = f"Subcloud {subcloud_name} (region_name: " + \ + f"{region_name}) does not have bootstrap-address in " + \ + "saved_payload." + return msg + + if 'systemcontroller_gateway_address' not in subcloud_payload: + msg = f"Subcloud {subcloud_name} (region_name: " + \ + f"{region_name}) does not have systemcontroller_" + \ + "gateway_address in saved_payload." + return msg + + return VERIFY_SUBCLOUD_SYNC_VALID + + def _validate_subclouds_for_sync(self, subclouds, dc_client): + """Validate subclouds for sync.""" + valid_subclouds = [] + error_msg = {} # Dictinary to store error message for each subcloud + + for subcloud in subclouds: + subcloud_name = subcloud.get('name') + region_name = subcloud.get('region_name') + + validation = self._is_valid_for_subcloud_sync(subcloud) + if validation != VERIFY_SUBCLOUD_SYNC_IGNORE and \ + validation != VERIFY_SUBCLOUD_SYNC_VALID: + LOG.error(validation) + error_msg[subcloud_name] = validation + continue + + try: + peer_subcloud = self.get_peer_subcloud(dc_client, subcloud_name) + if not peer_subcloud: + LOG.info(f"Subcloud {subcloud_name} (region_name: " + f"{region_name}) does not exist on peer site.") + valid_subclouds.append(subcloud) + continue + + if not self.is_subcloud_secondary(peer_subcloud): + msg = "Ignoring update Peer Site Subcloud " + \ + f"{subcloud_name} (region_name: {region_name})" + \ + " as is not in secondary state." + LOG.info(msg) + error_msg[subcloud_name] = msg + + valid_subclouds.append(subcloud) + + except Exception as e: + subcloud.msg = str(e) # Store error message for subcloud + LOG.error(f"Failed to validate Subcloud {subcloud_name} " + f"(region_name: {region_name}): {str(e)}") + error_msg[subcloud_name] = str(e) + + return valid_subclouds, error_msg + + def _sync_subclouds(self, context, peer, dc_local_pg_id, dc_peer_pg_id): + """Sync subclouds of local peer group to peer site. + + :param context: request context object + :param peer: system peer object of the peer site + :param dc_local_pg_id: peer group id on local site for sync + :param dc_peer_pg_id: peer group id on peer site + """ + dc_client = self.get_peer_dc_client(peer) + subclouds = db_api.subcloud_get_for_peer_group(context, dc_local_pg_id) + + subclouds_to_sync, error_msg = self._validate_subclouds_for_sync( + subclouds, dc_client) + + # Use thread pool to limit number of operations in parallel + sync_pool = greenpool.GreenPool(size=MAX_PARALLEL_SUBCLOUD_SYNC) + + # Spawn threads to sync each applicable subcloud + sync_function = functools.partial(self._add_or_update_subcloud, + dc_client, + peer.peer_controller_gateway_ip, + dc_peer_pg_id) + + failed_subclouds, sync_error_msg = self._run_parallel_group_operation( + 'peer sync', sync_function, sync_pool, subclouds_to_sync) + + error_msg.update(sync_error_msg) + LOG.info("Subcloud peer sync operation finished") + + dc_local_region_names = set() + for subcloud in subclouds: + # Ignore the secondary subclouds to sync with peer site + if not self.is_subcloud_secondary(subcloud): + # Count all subcloud need to be sync + dc_local_region_names.add(subcloud.get('name')) + + dc_peer_subclouds = dc_client.get_subcloud_list_by_peer_group( + str(dc_peer_pg_id)) + dc_peer_region_names = set(subcloud.get('name') for subcloud in + dc_peer_subclouds) + + dc_peer_subcloud_diff_names = dc_peer_region_names - \ + dc_local_region_names + for subcloud_to_delete in dc_peer_subcloud_diff_names: + try: + LOG.debug(f"Deleting Subcloud name {subcloud_to_delete} " + "on peer site.") + self.delete_peer_secondary_subcloud(dc_client, + subcloud_to_delete) + except Exception as e: + msg = f"Subcloud delete failed: {str(e)}" + LOG.error(msg) + error_msg[subcloud_to_delete] = msg + + return error_msg + + def sync_subcloud_peer_group(self, context, association_id, + sync_subclouds=True, priority=None): + """Sync subcloud peer group to peer site. + + This function synchronizes subcloud peer groups from current site + to peer site, supporting two scenarios: + + 1. When creating an association between the system peer and a subcloud + peer group. This function creates the subcloud peer group on the + peer site and synchronizes the subclouds to it. + + 2. When synchronizing a subcloud peer group with the peer site. This + function syncs both the subcloud peer group and the subclouds + under it to the peer site. + + :param context: request context object + :param association_id: id of association to sync + :param sync_subclouds: Enabled to sync subclouds to peer site + """ + LOG.info(f"Synchronize the association {association_id} of the " + "Subcloud Peer Group with the System Peer pointing to the " + "peer site.") + + association = db_api.peer_group_association_get(context, + association_id) + peer = db_api.system_peer_get(context, association.system_peer_id) + dc_local_pg = db_api.subcloud_peer_group_get(context, + association.peer_group_id) + peer_group_name = dc_local_pg.peer_group_name + + try: + sysinv_client = self.get_peer_sysinv_client(peer) + + # Check if the system_uuid of the peer site matches with the + # peer_uuid + system = sysinv_client.get_system() + if system.uuid != peer.peer_uuid: + LOG.error(f"Peer site system uuid {system.uuid} does not match " + f"with the peer_uuid {peer.peer_uuid}") + raise exceptions.PeerGroupAssociationTargetNotMatch( + uuid=system.uuid) + + dc_client = self.get_peer_dc_client(peer) + + peer_group_kwargs = { + 'group-priority': association.peer_group_priority, + 'group-state': dc_local_pg.group_state, + 'system-leader-id': dc_local_pg.system_leader_id, + 'system-leader-name': dc_local_pg.system_leader_name, + 'max-subcloud-rehoming': dc_local_pg.max_subcloud_rehoming + } + if priority: + peer_group_kwargs['group-priority'] = priority + try: + dc_peer_pg = dc_client.get_subcloud_peer_group( + peer_group_name) + dc_peer_pg_id = dc_peer_pg.get('id') + dc_peer_pg_priority = dc_peer_pg.get('group_priority') + if dc_peer_pg_priority == 0: + LOG.error(f"Skip update. Peer Site {peer_group_name} " + f"has priority 0.") + raise exceptions.SubcloudPeerGroupHasWrongPriority( + priority=dc_peer_pg_priority) + + dc_peer_pg = dc_client.update_subcloud_peer_group( + peer_group_name, **peer_group_kwargs) + LOG.info(f"Updated Subcloud Peer Group {peer_group_name} on " + f"peer site, ID is {dc_peer_pg.get('id')}.") + except dccommon_exceptions.SubcloudPeerGroupNotFound: + peer_group_kwargs['peer-group-name'] = peer_group_name + dc_peer_pg = dc_client.add_subcloud_peer_group( + **peer_group_kwargs) + dc_peer_pg_id = dc_peer_pg.get('id') + LOG.info(f"Created Subcloud Peer Group {peer_group_name} on " + f"peer site, ID is {dc_peer_pg_id}.") + + association_update = { + 'sync_status': consts.ASSOCIATION_SYNC_STATUS_SYNCED, + 'sync_message': None + } + if sync_subclouds: + error_msg = self._sync_subclouds(context, peer, dc_local_pg.id, + dc_peer_pg_id) + if len(error_msg) > 0: + association_update['sync_status'] = \ + consts.ASSOCIATION_SYNC_STATUS_FAILED + association_update['sync_message'] = json.dumps(error_msg) + if priority: + association_update['peer_group_priority'] = priority + association = db_api.peer_group_association_update( + context, association_id, **association_update) + + return db_api.peer_group_association_db_model_to_dict(association) + + except Exception as exception: + LOG.exception(f"Failed to sync peer group {peer_group_name} to " + f"peer site {peer.peer_name}") + db_api.peer_group_association_update( + context, association_id, + sync_status=consts.ASSOCIATION_SYNC_STATUS_FAILED, + sync_message=str(exception)) + raise exception + + def delete_peer_group_association(self, context, association_id): + """Delete association and remove related association from peer site. + + :param context: request context object. + :param association_id: id of association to delete + """ + LOG.info(f"Deleting association peer group {association_id}.") + + # Retrieve the peer group association details from the database + association = db_api.peer_group_association_get(context, + association_id) + peer = db_api.system_peer_get(context, association.system_peer_id) + peer_group = db_api.subcloud_peer_group_get(context, + association.peer_group_id) + + try: + dc_client = self.get_peer_dc_client(peer) + dc_peer_pg = dc_client.get_subcloud_peer_group( + peer_group.peer_group_name) + dc_peer_pg_priority = dc_peer_pg.get('group_priority') + if dc_peer_pg_priority == 0: + LOG.error(f"Failed to delete peer_group_association. " + f"Peer Group {peer_group.peer_group_name} " + f"has priority 0 on peer site.") + raise exceptions.SubcloudPeerGroupHasWrongPriority( + priority=dc_peer_pg_priority) + + subclouds = db_api.subcloud_get_for_peer_group(context, + peer_group.id) + for subcloud in subclouds: + self.delete_peer_secondary_subcloud(dc_client, + subcloud.name) + try: + dc_client.delete_subcloud_peer_group(peer_group.peer_group_name) + LOG.info("Deleted Subcloud Peer Group " + f"{peer_group.peer_group_name} on peer site.") + except dccommon_exceptions.SubcloudPeerGroupNotFound: + LOG.debug(f"Subcloud Peer Group {peer_group.peer_group_name} " + "does not exist on peer site.") + except dccommon_exceptions.SubcloudPeerGroupDeleteFailedAssociated: + LOG.debug(f"Subcloud Peer Group {peer_group.peer_group_name} " + "delete failed as it is associated with system peer " + "on peer site.") + + db_api.peer_group_association_destroy(context, association_id) + + except Exception as exception: + LOG.exception("Failed to delete peer_group_association " + f"{association.id}") + raise exception diff --git a/distributedcloud/dcmanager/rpc/client.py b/distributedcloud/dcmanager/rpc/client.py index 985bc49d1..a309129d7 100644 --- a/distributedcloud/dcmanager/rpc/client.py +++ b/distributedcloud/dcmanager/rpc/client.py @@ -135,6 +135,11 @@ class ManagerClient(RPCClient): subcloud_id=subcloud_id, payload=payload)) + def add_secondary_subcloud(self, ctxt, subcloud_id, payload): + return self.call(ctxt, self.make_msg('add_subcloud', + subcloud_id=subcloud_id, + payload=payload)) + def delete_subcloud(self, ctxt, subcloud_id): return self.call(ctxt, self.make_msg('delete_subcloud', subcloud_id=subcloud_id)) @@ -256,6 +261,19 @@ class ManagerClient(RPCClient): return self.cast(ctxt, self.make_msg('batch_migrate_subcloud', payload=payload)) + def sync_subcloud_peer_group(self, ctxt, association_id): + return self.cast(ctxt, self.make_msg( + 'sync_subcloud_peer_group', association_id=association_id)) + + def update_subcloud_peer_group(self, ctxt, association_id, priority): + return self.call(ctxt, self.make_msg( + 'sync_subcloud_peer_group', association_id=association_id, + sync_subclouds=False, priority=priority)) + + def delete_peer_group_association(self, ctxt, association_id): + return self.call(ctxt, self.make_msg('delete_peer_group_association', + association_id=association_id)) + class DCManagerNotifications(RPCClient): """DC Manager Notification interface to broadcast subcloud state changed diff --git a/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_peer_group_association.py b/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_peer_group_association.py new file mode 100644 index 000000000..e0fc25f1b --- /dev/null +++ b/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_peer_group_association.py @@ -0,0 +1,358 @@ +# +# Copyright (c) 2023 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +import mock +from six.moves import http_client +import uuid + +from dcmanager.db.sqlalchemy import api as db_api +from dcmanager.rpc import client as rpc_client + +from dcmanager.api.controllers.v1 import peer_group_association +from dcmanager.common import phased_subcloud_deploy as psd_common +from dcmanager.tests.unit.api import test_root_controller as testroot +from dcmanager.tests.unit.api.v1.controllers.mixins import APIMixin +from dcmanager.tests.unit.api.v1.controllers.mixins import GetMixin +from dcmanager.tests.unit.api.v1.controllers.mixins import UpdateMixin +from dcmanager.tests import utils + +# SAMPLE SYSTEM PEER DATA +SAMPLE_SYSTEM_PEER_UUID = str(uuid.uuid4()) +SAMPLE_SYSTEM_PEER_NAME = 'SystemPeer1' +SAMPLE_MANAGER_ENDPOINT = 'http://127.0.0.1:5000' +SAMPLE_MANAGER_USERNAME = 'admin' +SAMPLE_MANAGER_PASSWORD = 'password' +SAMPLE_PEER_CONTROLLER_GATEWAY_IP = '128.128.128.1' +SAMPLE_ADMINISTRATIVE_STATE = 'enabled' +SAMPLE_HEARTBEAT_INTERVAL = 10 +SAMPLE_HEARTBEAT_FAILURE_THRESHOLD = 3 +SAMPLE_HEARTBEAT_FAILURES_POLICY = 'alarm' +SAMPLE_HEARTBEAT_MAINTENANCE_TIMEOUT = 600 +SAMPLE_HEARTBEAT_STATUS_ALIVE = 'alive' + +# SAMPLE SUBCLOUD PEER GROUP DATA +SAMPLE_SUBCLOUD_PEER_GROUP_NAME = 'GroupX' +SAMPLE_SUBCLOUD_PEER_GROUP_SYSTEM_LEADER_ID = str(uuid.uuid4()) +SAMPLE_SUBCLOUD_PEER_GROUP_SYSTEM_LEADER_NAME = 'dc-local' +SAMPLE_SUBCLOUD_PEER_GROUP_MAX_SUBCLOUDS_REHOMING = 50 +SAMPLE_SUBCLOUD_PEER_GROUP_PRIORITY = 0 +SAMPLE_SUBCLOUD_PEER_GROUP_STATE = 'enabled' + +# SAMPLE PEER GROUP ASSOCIATION DATA +SAMPLE_SUBCLOUD_PEER_GROUP_ID = 1 +SAMPLE_SYSTEM_PEER_ID = 1 +SAMPLE_PEER_GROUP_PRIORITY = 1 +SAMPLE_PEER_GROUP_PRIORITY_UPDATED = 99 +SAMPLE_SYNC_STATUS = 'synced' +SAMPLE_SYNC_MESSAGE = 'None' + + +class FakeSystem(object): + def __init__(self, uuid): + self.uuid = uuid + + +class FakeKeystoneClient(object): + def __init__(self): + self.keystone_client = mock.MagicMock() + self.session = mock.MagicMock() + self.endpoint_cache = mock.MagicMock() + + +class FakeSysinvClient(object): + def __init__(self): + self.system = FakeSystem(SAMPLE_SUBCLOUD_PEER_GROUP_SYSTEM_LEADER_ID) + + def get_system(self): + return self.system + + +class PeerGroupAssociationAPIMixin(APIMixin): + + API_PREFIX = '/v1.0/peer-group-associations' + RESULT_KEY = 'peer_group_associations' + EXPECTED_FIELDS = ['id', + 'peer-group-id', + 'system-peer-id', + 'peer-group-priority', + 'created-at', + 'updated-at'] + + def setUp(self): + super(PeerGroupAssociationAPIMixin, self).setUp() + self.fake_rpc_client.some_method = mock.MagicMock() + + def _get_test_system_peer_dict(self, **kw): + # id should not be part of the structure + system_peer = { + 'peer_uuid': kw.get('peer_uuid', SAMPLE_SYSTEM_PEER_UUID), + 'peer_name': kw.get('peer_name', SAMPLE_SYSTEM_PEER_NAME), + 'endpoint': kw.get('manager_endpoint', SAMPLE_MANAGER_ENDPOINT), + 'username': kw.get('manager_username', SAMPLE_MANAGER_USERNAME), + 'password': kw.get('manager_password', SAMPLE_MANAGER_PASSWORD), + 'gateway_ip': kw.get( + 'peer_controller_gateway_ip', SAMPLE_PEER_CONTROLLER_GATEWAY_IP), + 'administrative_state': kw.get('administrative_state', + SAMPLE_ADMINISTRATIVE_STATE), + 'heartbeat_interval': kw.get('heartbeat_interval', + SAMPLE_HEARTBEAT_INTERVAL), + 'heartbeat_failure_threshold': kw.get( + 'heartbeat_failure_threshold', SAMPLE_HEARTBEAT_FAILURE_THRESHOLD), + 'heartbeat_failure_policy': kw.get( + 'heartbeat_failure_policy', SAMPLE_HEARTBEAT_FAILURES_POLICY), + 'heartbeat_maintenance_timeout': kw.get( + 'heartbeat_maintenance_timeout', + SAMPLE_HEARTBEAT_MAINTENANCE_TIMEOUT) + } + return system_peer + + def _get_test_subcloud_peer_group_dict(self, **kw): + # id should not be part of the structure + group = { + 'peer_group_name': kw.get('peer_group_name', + SAMPLE_SUBCLOUD_PEER_GROUP_NAME), + 'system_leader_id': kw.get( + 'system_leader_id', + SAMPLE_SUBCLOUD_PEER_GROUP_SYSTEM_LEADER_ID), + 'system_leader_name': kw.get( + 'system_leader_name', + SAMPLE_SUBCLOUD_PEER_GROUP_SYSTEM_LEADER_NAME), + 'group_priority': kw.get( + 'group_priority', + SAMPLE_SUBCLOUD_PEER_GROUP_PRIORITY), + 'group_state': kw.get( + 'group_state', + SAMPLE_SUBCLOUD_PEER_GROUP_STATE), + 'max_subcloud_rehoming': kw.get( + 'max_subcloud_rehoming', + SAMPLE_SUBCLOUD_PEER_GROUP_MAX_SUBCLOUDS_REHOMING) + } + return group + + def _get_test_peer_group_association_dict(self, **kw): + # id should not be part of the structure + association = { + 'peer_group_id': kw.get('peer_group_id', + SAMPLE_SUBCLOUD_PEER_GROUP_ID), + 'system_peer_id': kw.get('system_peer_id', SAMPLE_SYSTEM_PEER_ID), + 'peer_group_priority': kw.get('peer_group_priority', + SAMPLE_PEER_GROUP_PRIORITY), + 'sync_status': kw.get('sync_status', SAMPLE_SYNC_STATUS), + 'sync_message': kw.get('sync_message', SAMPLE_SYNC_MESSAGE) + } + return association + + def _post_get_test_peer_group_association(self, **kw): + post_body = self._get_test_peer_group_association_dict(**kw) + return post_body + + # The following methods are required for subclasses of APIMixin + def get_api_prefix(self): + return self.API_PREFIX + + def get_result_key(self): + return self.RESULT_KEY + + def get_expected_api_fields(self): + return self.EXPECTED_FIELDS + + def get_omitted_api_fields(self): + return [] + + def _create_db_related_objects(self, context): + system_peer_fields = self._get_test_system_peer_dict() + peer = db_api.system_peer_create(context, **system_peer_fields) + + peer_group_fields = self._get_test_subcloud_peer_group_dict() + peer_group = db_api.subcloud_peer_group_create(context, + **peer_group_fields) + + return peer.id, peer_group.id + + def _create_db_object(self, context, **kw): + + peer_id, peer_group_id = self._create_db_related_objects(context) + + kw['peer_group_id'] = peer_group_id if kw.get('peer_group_id') is None \ + else kw.get('peer_group_id') + kw['system_peer_id'] = peer_id if kw.get('system_peer_id') is None \ + else kw.get('system_peer_id') + creation_fields = self._get_test_peer_group_association_dict(**kw) + return db_api.peer_group_association_create(context, **creation_fields) + + def get_post_object(self): + return self._post_get_test_peer_group_association() + + def get_update_object(self): + update_object = { + 'peer_group_priority': SAMPLE_PEER_GROUP_PRIORITY_UPDATED + } + return update_object + + +# Combine Peer Group Association API with mixins to test post, get, update and delete +class TestPeerGroupAssociationPost(testroot.DCManagerApiTest, + PeerGroupAssociationAPIMixin): + def setUp(self): + super(TestPeerGroupAssociationPost, self).setUp() + + p = mock.patch.object(rpc_client, 'ManagerClient') + self.mock_rpc_client = p.start() + self.addCleanup(p.stop) + + context = utils.dummy_context() + self.context = context + peer_id, _ = self._create_db_related_objects(context) + db_api.system_peer_update(context, peer_id=peer_id, + heartbeat_status=SAMPLE_HEARTBEAT_STATUS_ALIVE) + + def verify_post_failure(self, response): + # Failures will return text rather than JSON + self.assertEqual(response.content_type, 'text/plain') + self.assertEqual(response.status_code, http_client.BAD_REQUEST) + + def test_create_success(self): + self.mock_rpc_client().sync_subcloud_peer_group.return_value = True + + ndict = self.get_post_object() + response = self.app.post_json(self.get_api_prefix(), + ndict, + headers=self.get_api_headers()) + self.assertEqual(response.content_type, 'application/json') + + def test_create_with_string_id_fails(self): + # A string system peer id is not permitted. + ndict = self.get_post_object() + ndict['system_peer_id'] = 'test-system-peer-id' + response = self.app.post_json(self.get_api_prefix(), + ndict, + headers=self.get_api_headers(), + expect_errors=True) + self.verify_post_failure(response) + + def test_create_with_blank_id_fails(self): + # An empty system_peer_id is not permitted + ndict = self.get_post_object() + ndict['system_peer_id'] = '' + response = self.app.post_json(self.get_api_prefix(), + ndict, + headers=self.get_api_headers(), + expect_errors=True) + self.verify_post_failure(response) + + def test_create_with_wrong_peer_group_priority_fails(self): + # A string peer group priority is not permitted. + ndict = self.get_post_object() + ndict['peer_group_id'] = 'peer-group-id' + response = self.app.post_json(self.get_api_prefix(), + ndict, + headers=self.get_api_headers(), + expect_errors=True) + self.verify_post_failure(response) + + def test_create_with_bad_peer_group_priority(self): + # peer_group_priority must be an integer between 1 and 65536 + ndict = self.get_post_object() + # All the entries in bad_values should be considered invalid + bad_values = [0, 65537, -2, 'abc'] + for bad_value in bad_values: + ndict['peer_group_priority'] = bad_value + response = self.app.post_json(self.get_api_prefix(), + ndict, + headers=self.get_api_headers(), + expect_errors=True) + self.verify_post_failure(response) + + +class TestPeerGroupAssociationGet(testroot.DCManagerApiTest, + PeerGroupAssociationAPIMixin, + GetMixin): + def setUp(self): + super(TestPeerGroupAssociationGet, self).setUp() + + +class TestPeerGroupAssociationUpdate(testroot.DCManagerApiTest, + PeerGroupAssociationAPIMixin, + UpdateMixin): + def setUp(self): + super(TestPeerGroupAssociationUpdate, self).setUp() + + def validate_updated_fields(self, sub_dict, full_obj): + for key, value in sub_dict.items(): + key = key.replace('_', '-') + self.assertEqual(value, full_obj.get(key)) + + @mock.patch.object(rpc_client, 'ManagerClient') + def test_update_success(self, mock_client): + mock_client().update_subcloud_peer_group.return_value = { + 'peer-group-priority': SAMPLE_PEER_GROUP_PRIORITY_UPDATED + } + context = utils.dummy_context() + single_obj = self._create_db_object(context) + update_data = self.get_update_object() + response = self.app.patch_json(self.get_single_url(single_obj.id), + headers=self.get_api_headers(), + params=update_data) + self.assertEqual(response.content_type, 'application/json') + self.assertEqual(response.status_code, http_client.OK) + self.validate_updated_fields(update_data, response.json) + + @mock.patch.object(psd_common, 'OpenStackDriver') + @mock.patch.object(peer_group_association, 'SysinvClient') + @mock.patch.object(rpc_client, 'ManagerClient') + def test_sync_association(self, mock_client, mock_sysinv_client, mock_keystone_client): + mock_client().sync_subcloud_peer_group.return_value = True + mock_keystone_client().keystone_client = FakeKeystoneClient() + mock_sysinv_client.return_value = FakeSysinvClient() + + context = utils.dummy_context() + single_obj = self._create_db_object(context) + response = self.app.patch_json( + self.get_single_url(single_obj.id) + '/sync', + headers=self.get_api_headers()) + self.assertEqual(response.content_type, 'application/json') + self.assertEqual(response.status_code, http_client.OK) + mock_client().sync_subcloud_peer_group.assert_called_once() + + +class TestPeerGroupAssociationDelete(testroot.DCManagerApiTest, + PeerGroupAssociationAPIMixin): + def setUp(self): + super(TestPeerGroupAssociationDelete, self).setUp() + + p = mock.patch.object(rpc_client, 'ManagerClient') + self.mock_rpc_client = p.start() + self.addCleanup(p.stop) + + self.mock_rpc_client().delete_peer_group_association.return_value = True + + def test_delete_success(self): + context = utils.dummy_context() + single_obj = self._create_db_object(context) + response = self.app.delete(self.get_single_url(single_obj.id), + headers=self.get_api_headers()) + self.mock_rpc_client().delete_peer_group_association. \ + assert_called_once() + self.assertEqual(response.content_type, 'application/json') + self.assertEqual(response.status_code, http_client.OK) + + def test_double_delete(self): + context = utils.dummy_context() + single_obj = self._create_db_object(context) + response = self.app.delete(self.get_single_url(single_obj.id), + headers=self.get_api_headers()) + self.mock_rpc_client().delete_peer_group_association. \ + assert_called_once() + self.assertEqual(response.content_type, 'application/json') + self.assertEqual(response.status_code, http_client.OK) + + db_api.peer_group_association_destroy(context, single_obj.id) + # delete the same object a second time. this should fail (NOT_FOUND) + response = self.app.delete(self.get_single_url(single_obj.id), + headers=self.get_api_headers(), + expect_errors=True) + self.assertEqual(response.content_type, 'text/plain') + self.assertEqual(response.status_code, http_client.NOT_FOUND) diff --git a/distributedcloud/dcmanager/tests/unit/manager/test_system_peer_manager.py b/distributedcloud/dcmanager/tests/unit/manager/test_system_peer_manager.py new file mode 100644 index 000000000..5d2a53d90 --- /dev/null +++ b/distributedcloud/dcmanager/tests/unit/manager/test_system_peer_manager.py @@ -0,0 +1,389 @@ +# +# Copyright (c) 2023 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +import json +import mock +import uuid + +from dccommon import exceptions as dccommon_exceptions +from dcmanager.db.sqlalchemy import api as db_api +from dcmanager.manager import system_peer_manager +from dcmanager.tests import base +from dcmanager.tests.unit.common import fake_subcloud + +# FAKE SYSINV DATA +FAKE_SITE0_SYSTEM_UUID = str(uuid.uuid4()) +FAKE_SITE1_SYSTEM_UUID = str(uuid.uuid4()) + +# FAKE SYSTEM PEER DATA +FAKE_SYSTEM_PEER_ID = 1 +FAKE_SYSTEM_PEER_UUID = FAKE_SITE1_SYSTEM_UUID +FAKE_SYSTEM_PEER_NAME = 'PeerSite1' +FAKE_MANAGER_ENDPOINT = 'http://128.128.128.128:5000/v3' +FAKE_MANAGER_USERNAME = 'admin' +FAKE_MANAGER_PASSWORD = 'cGFzc3dvcmQ=' +FAKE_PEER_CONTROLLER_GATEWAY_IP = '128.128.1.1' + +# FAKE SUBCLOUD PEER GROUP DATA (SITE0) +FAKE_SITE0_PEER_GROUP_ID = 1 +FAKE_SITE0_PEER_GROUP_NAME = 'PeerGroup1' +FAKE_SITE0_PEER_GROUP_SYSTEM_LEADER_ID = FAKE_SITE0_SYSTEM_UUID +FAKE_SITE0_PEER_GROUP_SYSTEM_LEADER_NAME = 'site0' +FAKE_SITE0_PEER_GROUP_MAX_SUBCLOUDS_REHOMING = 50 +FAKE_SITE0_PEER_GROUP_PRIORITY = 0 +FAKE_SITE0_PEER_GROUP_STATE = 'enabled' + +# FAKE SUBCLOUD PEER GROUP DATA (SITE1) +FAKE_SITE1_PEER_GROUP_ID = 9 + +# FAKE SUBCLOUD DATA (SITE1) +FAKE_SITE1_SUBCLOUD1_ID = 11 +FAKE_SITE1_SUBCLOUD1_DEPLOY_STATUS = 'secondary' +FAKE_SITE1_SUBCLOUD1_DATA = {"id": FAKE_SITE1_SUBCLOUD1_ID, + "name": "subcloud1", + "region-name": "subcloud1", + "deploy-status": + FAKE_SITE1_SUBCLOUD1_DEPLOY_STATUS} +FAKE_SITE1_SUBCLOUD2_ID = 12 +FAKE_SITE1_SUBCLOUD2_DEPLOY_STATUS = 'secondary-failed' +FAKE_SITE1_SUBCLOUD2_DATA = {"id": FAKE_SITE1_SUBCLOUD2_ID, + "name": "subcloud2", + "region-name": "subcloud2", + "deploy-status": + FAKE_SITE1_SUBCLOUD2_DEPLOY_STATUS} +FAKE_SITE1_SUBCLOUD3_ID = 13 +FAKE_SITE1_SUBCLOUD3_DEPLOY_STATUS = 'secondary' +FAKE_SITE1_SUBCLOUD3_DATA = {"id": FAKE_SITE1_SUBCLOUD3_ID, + "name": "subcloud3", + "region-name": "subcloud3", + "deploy-status": + FAKE_SITE1_SUBCLOUD3_DEPLOY_STATUS} + +# FAKE PEER GROUP ASSOCIATION DATA +FAKE_ASSOCIATION_PEER_GROUP_ID = \ + FAKE_SITE0_PEER_GROUP_ID +FAKE_ASSOCIATION_SYSTEM_PEER_ID = \ + FAKE_SYSTEM_PEER_ID +FAKE_ASSOCIATION_PEER_GROUP_PRIORITY = 1 +FAKE_ASSOCIATION_SYNC_STATUS = 'synced' +FAKE_ASSOCIATION_SYNC_MESSAGE = 'None' + + +class FakeDCManagerAuditAPI(object): + def __init__(self): + pass + + +class FakeSystem(object): + def __init__(self, uuid): + self.uuid = uuid + + +class FakePeerGroup(object): + def __init__(self): + self.id = FAKE_SITE1_PEER_GROUP_ID + + +class FakeKeystoneClient(object): + def __init__(self): + self.keystone_client = mock.MagicMock() + self.session = mock.MagicMock() + self.endpoint_cache = mock.MagicMock() + + +class FakeSysinvClient(object): + def __init__(self): + self.system = FakeSystem(FAKE_SITE1_SYSTEM_UUID) + + def get_system(self): + return self.system + + +class FakeDcmanagerClient(object): + def __init__(self): + self.peer_groups = [FakePeerGroup()] + + def add_subcloud_peer_group(self, **kwargs): + return self.peer_groups + + def get_subcloud_peer_group(self, peer_group_name): + return self.peer_groups + + +class FakeException(Exception): + pass + + +class TestSystemPeerManager(base.DCManagerTestCase): + def setUp(self): + super(TestSystemPeerManager, self).setUp() + + # Mock the DCManager Audit API + self.fake_dcmanager_audit_api = FakeDCManagerAuditAPI() + p = mock.patch('dcmanager.audit.rpcapi.ManagerAuditClient') + self.mock_dcmanager_audit_api = p.start() + self.mock_dcmanager_audit_api.return_value = \ + self.fake_dcmanager_audit_api + self.addCleanup(p.stop) + + @staticmethod + def create_subcloud_with_pg_static(ctxt, peer_group_id, + rehome_data=None, **kwargs): + subcloud = fake_subcloud.create_fake_subcloud(ctxt, **kwargs) + return db_api.subcloud_update(ctxt, subcloud.id, + peer_group_id=peer_group_id, + rehome_data=rehome_data) + + @staticmethod + def create_system_peer_static(ctxt, **kwargs): + values = { + 'peer_uuid': FAKE_SYSTEM_PEER_UUID, + 'peer_name': FAKE_SYSTEM_PEER_NAME, + 'endpoint': FAKE_MANAGER_ENDPOINT, + 'username': FAKE_MANAGER_USERNAME, + 'password': FAKE_MANAGER_PASSWORD, + 'gateway_ip': FAKE_PEER_CONTROLLER_GATEWAY_IP + } + values.update(kwargs) + return db_api.system_peer_create(ctxt, **values) + + @staticmethod + def create_subcloud_peer_group_static(ctxt, **kwargs): + values = { + "peer_group_name": FAKE_SITE0_PEER_GROUP_NAME, + "system_leader_id": FAKE_SITE0_PEER_GROUP_SYSTEM_LEADER_ID, + "system_leader_name": FAKE_SITE0_PEER_GROUP_SYSTEM_LEADER_NAME, + "group_priority": FAKE_SITE0_PEER_GROUP_PRIORITY, + "group_state": FAKE_SITE0_PEER_GROUP_STATE, + "max_subcloud_rehoming": + FAKE_SITE0_PEER_GROUP_MAX_SUBCLOUDS_REHOMING + } + values.update(kwargs) + return db_api.subcloud_peer_group_create(ctxt, **values) + + @staticmethod + def create_peer_group_association_static(ctxt, **kwargs): + values = { + "system_peer_id": FAKE_ASSOCIATION_SYSTEM_PEER_ID, + "peer_group_id": FAKE_ASSOCIATION_PEER_GROUP_ID, + "peer_group_priority": FAKE_ASSOCIATION_PEER_GROUP_PRIORITY, + "sync_status": FAKE_ASSOCIATION_SYNC_STATUS, + "sync_message": FAKE_ASSOCIATION_SYNC_MESSAGE + } + values.update(kwargs) + return db_api.peer_group_association_create(ctxt, **values) + + def test_init(self): + spm = system_peer_manager.SystemPeerManager() + self.assertIsNotNone(spm) + self.assertEqual('system_peer_manager', spm.service_name) + self.assertEqual('localhost', spm.host) + + @mock.patch.object(system_peer_manager, 'PeerSiteDriver') + @mock.patch.object(system_peer_manager, 'SysinvClient') + @mock.patch.object(system_peer_manager, 'DcmanagerClient') + def test_sync_subclouds(self, mock_dc_client, + mock_sysinv_client, + mock_keystone_client): + mock_keystone_client().keystone_client = FakeKeystoneClient() + mock_sysinv_client.return_value = FakeSysinvClient() + mock_dc_client.return_value = FakeDcmanagerClient() + mock_dc_client().add_subcloud_with_secondary_status = mock.MagicMock() + mock_dc_client().delete_subcloud = mock.MagicMock() + + peer = self.create_system_peer_static( + self.ctx, + peer_name='SystemPeer1') + peer_group = self.create_subcloud_peer_group_static( + self.ctx, + peer_group_name='SubcloudPeerGroup1') + rehome_data = { + "saved_payload": { + "bootstrap-address": "192.168.10.10", + "systemcontroller_gateway_address": "192.168.204.101" + } + } + # Create local dc subcloud1 mock data in database + self.create_subcloud_with_pg_static( + self.ctx, + peer_group_id=peer_group.id, + rehome_data=json.dumps(rehome_data), + name='subcloud1', + region_name='subcloud1') + # Create local dc subcloud2 mock data in database + self.create_subcloud_with_pg_static( + self.ctx, + peer_group_id=peer_group.id, + rehome_data=json.dumps(rehome_data), + name='subcloud2', + region_name='subcloud2') + peer_subcloud1 = FAKE_SITE1_SUBCLOUD1_DATA + peer_subcloud2 = FAKE_SITE1_SUBCLOUD2_DATA + peer_subcloud3 = FAKE_SITE1_SUBCLOUD3_DATA + mock_dc_client().get_subcloud = mock.MagicMock() + mock_dc_client().get_subcloud.side_effect = [ + peer_subcloud1, dccommon_exceptions.SubcloudNotFound, + peer_subcloud1, dccommon_exceptions.SubcloudNotFound, + peer_subcloud3] + mock_dc_client().get_subcloud_list_by_peer_group = mock.MagicMock() + mock_dc_client().get_subcloud_list_by_peer_group.return_value = [ + peer_subcloud1, peer_subcloud2, peer_subcloud3] + mock_dc_client().update_subcloud = mock.MagicMock() + mock_dc_client().update_subcloud.side_effect = [ + peer_subcloud1, peer_subcloud1, peer_subcloud2] + + spm = system_peer_manager.SystemPeerManager() + spm._sync_subclouds(self.ctx, peer, peer_group.id, + FAKE_SITE1_PEER_GROUP_ID) + + mock_dc_client().get_subcloud.assert_has_calls([ + mock.call(peer_subcloud1.get('name')), + mock.call(peer_subcloud2.get('name')), + mock.call(peer_subcloud3.get('name')) + ]) + mock_dc_client().update_subcloud.assert_has_calls([ + mock.call('subcloud1', mock.ANY, mock.ANY), + mock.call('subcloud1', files=None, + data={'peer_group': str(FAKE_SITE1_PEER_GROUP_ID)}) + ]) + mock_dc_client().add_subcloud_with_secondary_status. \ + assert_called_once() + mock_dc_client().delete_subcloud.assert_called_once_with('subcloud3') + + @mock.patch.object( + system_peer_manager.SystemPeerManager, '_sync_subclouds') + @mock.patch.object(system_peer_manager, 'PeerSiteDriver') + @mock.patch.object(system_peer_manager, 'SysinvClient') + @mock.patch.object(system_peer_manager, 'DcmanagerClient') + def test_sync_subcloud_peer_group(self, + mock_dc_client, + mock_sysinv_client, + mock_keystone_client, + mock_sync_subclouds): + mock_sync_subclouds.return_value = True + mock_keystone_client().keystone_client = FakeKeystoneClient() + mock_sysinv_client.return_value = FakeSysinvClient() + mock_dc_client.return_value = FakeDcmanagerClient() + mock_dc_client().get_subcloud_peer_group = mock.MagicMock() + mock_dc_client().update_subcloud_peer_group = mock.MagicMock() + + peer = self.create_system_peer_static( + self.ctx, + peer_name='SystemPeer1') + peer_group = self.create_subcloud_peer_group_static( + self.ctx, + peer_group_name='SubcloudPeerGroup1') + association = self.create_peer_group_association_static( + self.ctx, + system_peer_id=peer.id, + peer_group_id=peer_group.id) + + spm = system_peer_manager.SystemPeerManager() + spm.sync_subcloud_peer_group(self.ctx, association.id, False) + + mock_dc_client().get_subcloud_peer_group.assert_called_once_with( + peer_group.peer_group_name) + mock_dc_client().update_subcloud_peer_group.assert_called_once() + + @mock.patch.object( + system_peer_manager.SystemPeerManager, '_sync_subclouds') + @mock.patch.object(system_peer_manager, 'PeerSiteDriver') + @mock.patch.object(system_peer_manager, 'SysinvClient') + @mock.patch.object(system_peer_manager, 'DcmanagerClient') + def test_sync_subcloud_peer_group_not_exist(self, mock_dc_client, + mock_sysinv_client, + mock_keystone_client, + mock_sync_subclouds): + mock_sync_subclouds.return_value = True + mock_keystone_client().keystone_client = FakeKeystoneClient() + mock_sysinv_client.return_value = FakeSysinvClient() + mock_dc_client.return_value = FakeDcmanagerClient() + mock_dc_client().get_subcloud_peer_group = mock.MagicMock() + mock_dc_client().add_subcloud_peer_group = mock.MagicMock() + mock_dc_client().update_subcloud_peer_group = mock.MagicMock() + + peer = self.create_system_peer_static( + self.ctx, + peer_name='SystemPeer1') + peer_group = self.create_subcloud_peer_group_static( + self.ctx, + peer_group_name='SubcloudPeerGroup1') + association = self.create_peer_group_association_static( + self.ctx, + system_peer_id=peer.id, + peer_group_id=peer_group.id) + + mock_dc_client().get_subcloud_peer_group.side_effect = \ + dccommon_exceptions.SubcloudPeerGroupNotFound + + spm = system_peer_manager.SystemPeerManager() + spm.sync_subcloud_peer_group(self.ctx, association.id, False) + + mock_dc_client().get_subcloud_peer_group.assert_called_once_with( + peer_group.peer_group_name) + mock_dc_client().add_subcloud_peer_group.assert_called_once_with(**{ + 'peer-group-name': peer_group.peer_group_name, + 'group-priority': association.peer_group_priority, + 'group-state': peer_group.group_state, + 'system-leader-id': peer_group.system_leader_id, + 'system-leader-name': peer_group.system_leader_name, + 'max-subcloud-rehoming': peer_group.max_subcloud_rehoming + }) + mock_dc_client().update_subcloud_peer_group.assert_not_called() + + @mock.patch.object(system_peer_manager, 'PeerSiteDriver') + @mock.patch.object(system_peer_manager, 'DcmanagerClient') + def test_delete_peer_group_association(self, + mock_dc_client, + mock_keystone_client): + mock_keystone_client().keystone_client = FakeKeystoneClient() + mock_dc_client.return_value = FakeDcmanagerClient() + mock_dc_client().delete_subcloud_peer_group = mock.MagicMock() + mock_dc_client().delete_subcloud = mock.MagicMock() + + peer = self.create_system_peer_static( + self.ctx, + peer_name='SystemPeer1') + peer_group = self.create_subcloud_peer_group_static( + self.ctx, + peer_group_name='SubcloudPeerGroup1') + # Create local dc subcloud1 mock data in database + subcloud1 = self.create_subcloud_with_pg_static( + self.ctx, + peer_group_id=peer_group.id, + name='subcloud1') + # Create local dc subcloud2 mock data in database + subcloud2 = self.create_subcloud_with_pg_static( + self.ctx, + peer_group_id=peer_group.id, + name='subcloud2') + association = self.create_peer_group_association_static( + self.ctx, + system_peer_id=peer.id, + peer_group_id=peer_group.id) + peer_subcloud1 = FAKE_SITE1_SUBCLOUD1_DATA + peer_subcloud2 = FAKE_SITE1_SUBCLOUD2_DATA + mock_dc_client().get_subcloud = mock.MagicMock() + mock_dc_client().get_subcloud.side_effect = [ + peer_subcloud1, peer_subcloud2] + + mock_dc_client().get_subcloud_peer_group = mock.MagicMock() + mock_dc_client().get_subcloud_peer_group.return_value = { + 'group_priority': 1 + } + + spm = system_peer_manager.SystemPeerManager() + spm.delete_peer_group_association(self.ctx, association.id) + + mock_dc_client().delete_subcloud.assert_has_calls([ + mock.call(subcloud1.name), + mock.call(subcloud2.name) + ]) + mock_dc_client().delete_subcloud_peer_group.assert_called_once_with( + peer_group.peer_group_name) + + associations = db_api.peer_group_association_get_all(self.ctx) + self.assertEqual(0, len(associations))