From 5e884374d2e9d7a26a5b03c230fd59bca428af01 Mon Sep 17 00:00:00 2001 From: albailey Date: Tue, 3 Mar 2020 13:55:18 -0600 Subject: [PATCH] Adding DistributedCloud subcloud group feature A new subcloud group feature has been added as a way to logically group several subclouds. This would allow a group of subclouds to be updated in parallel or in serial, or to limit how many are updated in parallel. A 'Default' subcloud group is created in all installations, and all subclouds are automatically members of that subcloud group unless explicitly updated to reference another. The Default subcloud group cannot have its name changed, and other groups cannot use the same name. A subcloud group that has subclouds in it, cannot be deleted. Change-Id: Id0f232966c0128ecf6d87f941abcd42ff5a1a5c1 Story: 2007518 Task: 39301 Signed-off-by: albailey --- api-ref/source/api-ref-dcmanager-v1.rst | 350 ++++++++++++ .../dcmanager/api/controllers/v1/root.py | 3 + .../api/controllers/v1/subcloud_group.py | 313 ++++++++++ .../dcmanager/api/controllers/v1/subclouds.py | 71 ++- distributedcloud/dcmanager/common/consts.py | 7 + .../dcmanager/common/exceptions.py | 22 + distributedcloud/dcmanager/common/utils.py | 54 ++ distributedcloud/dcmanager/db/api.py | 73 ++- .../dcmanager/db/sqlalchemy/api.py | 153 ++++- .../versions/006_add_subcloud_group_table.py | 101 ++++ .../dcmanager/db/sqlalchemy/models.py | 17 + distributedcloud/dcmanager/manager/service.py | 5 +- .../dcmanager/manager/subcloud_manager.py | 22 +- distributedcloud/dcmanager/rpc/client.py | 5 +- distributedcloud/dcmanager/tests/base.py | 31 +- .../api/v1/controllers/test_subcloud_group.py | 539 ++++++++++++++++++ .../unit/api/v1/controllers/test_subclouds.py | 3 +- .../tests/unit/db/test_subcloud_db_api.py | 2 + .../tests/unit/manager/test_service.py | 5 +- .../manager/test_subcloud_audit_manager.py | 1 + .../unit/manager/test_subcloud_manager.py | 54 +- distributedcloud/dcmanager/tests/utils.py | 3 +- distributedcloud/tox.ini | 3 - 23 files changed, 1757 insertions(+), 80 deletions(-) create mode 100755 distributedcloud/dcmanager/api/controllers/v1/subcloud_group.py create mode 100644 distributedcloud/dcmanager/db/sqlalchemy/migrate_repo/versions/006_add_subcloud_group_table.py create mode 100644 distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_subcloud_group.py diff --git a/api-ref/source/api-ref-dcmanager-v1.rst b/api-ref/source/api-ref-dcmanager-v1.rst index aa208a8f8..4eec71ce8 100644 --- a/api-ref/source/api-ref-dcmanager-v1.rst +++ b/api-ref/source/api-ref-dcmanager-v1.rst @@ -93,6 +93,7 @@ internalServerError (500), serviceUnavailable (503) "compute_sync_status (Optional)", "plain", "xsd:string", "The compute sync status of the subcloud." "network_sync_status (Optional)", "plain", "xsd:string", "The network sync status of the subcloud." "patching_sync_status (Optional)", "plain", "xsd:string", "The patching sync status of the subcloud." + "group_id (Optional)", "plain", "xsd:int", "The unique identifier for the subcloud group for this subcloud." :: @@ -132,6 +133,7 @@ internalServerError (500), serviceUnavailable (503) "endpoint_type": "patching" }, "created-at": u"2018-02-25 19:06:35.208505", + "group_id": 1, "management-gateway-ip": u"192.168.204.1", "management-end-ip": u"192.168.204.100", "id": 1, @@ -171,6 +173,7 @@ internalServerError (500), serviceUnavailable (503) "endpoint_type": "patching" }, "created-at": "2018-02-25 19:06:35.208505", + "group_id": 1, "management-gateway-ip": "192.168.205.1", "management-end-ip": "192.168.205.100", "id": 2, @@ -210,6 +213,7 @@ serviceUnavailable (503) "management-start-ip", "plain", "xsd:string", "Start of management IP address range for subcloud." "management-end-ip", "plain", "xsd:string", "End of management IP address range for subcloud." "systemcontroller-gateway-ip", "plain", "xsd:string", "Systemcontroller gateway IP Address." + "group_id", "plain", "xsd:int", "Id of the subcloud group. Defaults to 1" **Response parameters** @@ -227,6 +231,7 @@ serviceUnavailable (503) "management-start-ip (Optional)", "plain", "xsd:string", "Start of management IP address range for subcloud." "management-end-ip (Optional)", "plain", "xsd:string", "End of management IP address range for subcloud." "systemcontroller-gateway-ip (Optional)", "plain", "xsd:string", "Systemcontroller gateway IP Address." + "group_id (Optional)", "plain", "xsd:int", "Id of the subcloud group." :: @@ -238,6 +243,7 @@ serviceUnavailable (503) "management-subnet": "192.168.205.0/24", "management-gateway-ip": "192.168.205.1", "management-end-ip": "192.168.205.160", + "group_id": 1, "description": "new subcloud" } @@ -253,6 +259,7 @@ serviceUnavailable (503) "availability-status": "offline", "systemcontroller-gateway-ip": "192.168.204.102", "location": None, + "group_id": 1, "management-subnet": "192.168.205.0/24", "management-gateway-ip": "192.168.205.1", "management-end-ip": "192.168.205.160", @@ -306,6 +313,7 @@ internalServerError (500), serviceUnavailable (503) "compute_sync_status (Optional)", "plain", "xsd:string", "The compute sync status of the subcloud." "network_sync_status (Optional)", "plain", "xsd:string", "The network sync status of the subcloud." "patching_sync_status (Optional)", "plain", "xsd:string", "The patching sync status of the subcloud." + "group_id (Optional)", "plain", "xsd:int", "Id of the subcloud group." :: @@ -344,6 +352,7 @@ internalServerError (500), serviceUnavailable (503) ], "management-gateway-ip": "192.168.204.1", "management-end-ip": "192.168.204.100", + "group_id": 1, "id": 1, "name": "subcloud6" } @@ -397,6 +406,7 @@ internalServerError (500), serviceUnavailable (503) "network_sync_status (Optional)", "plain", "xsd:string", "The network sync status of the subcloud." "patching_sync_status (Optional)", "plain", "xsd:string", "The patching sync status of the subcloud." "oam_floating_ip (Optional)", "plain", "xsd:string", "OAM Floating IP of the subcloud." + "group_id (Optional)", "plain", "xsd:int", "Id of the subcloud group." :: @@ -435,6 +445,7 @@ internalServerError (500), serviceUnavailable (503) ], "management-gateway-ip": "192.168.204.1", "management-end-ip": "192.168.204.100", + "group_id": 1, "id": 1, "name": "subcloud6", "oam_floating_ip" "10.10.10.12" @@ -476,6 +487,7 @@ serviceUnavailable (503) "description (Optional)", "plain", "xsd:string", "The description of the subcloud." "location (Optional)", "plain", "xsd:string", "The location of the subcloud." "management-state (Optional)", "plain", "xsd:string", "The management-state of the subcloud, ``managed`` or ``unmanaged``. The subcloud must be online before this can be modified to managed." + "group_id (Optional)", "plain", "xsd:int", "Id of the subcloud group. The group must exist." **Response parameters** @@ -493,6 +505,7 @@ serviceUnavailable (503) "management-start-ip (Optional)", "plain", "xsd:string", "Start of management IP address range for subcloud." "management-end-ip (Optional)", "plain", "xsd:string", "End of management IP address range for subcloud." "systemcontroller-gateway-ip (Optional)", "plain", "xsd:string", "Systemcontroller gateway IP Address." + "group_id (Optional)", "plain", "xsd:int", "Id of the subcloud group." :: @@ -500,6 +513,7 @@ serviceUnavailable (503) "description": "new description", "location": "new location", "management-state": "managed" + "group_id": 2, } :: @@ -517,6 +531,7 @@ serviceUnavailable (503) "management-subnet": "192.168.204.0/24", "management-gateway-ip": "192.168.204.1", "management-end-ip": "192.168.204.100", + "group_id": 2, "id": 1, "name": "subcloud6" } @@ -541,6 +556,341 @@ Deletes a specific subcloud This operation does not accept a request body. +---------------- +Subcloud Groups +---------------- + +Subcloud Groups are a logical grouping managed by a central System Controller. +Subclouds in a group can be updated in parallel when applying patches or +software upgrades. + +************************** +Lists all subcloud groups +************************** + +.. rest_method:: GET /v1.0/subcloud-groups + +**Normal response codes** + +200 + +**Error response codes** + +itemNotFound (404), badRequest (400), unauthorized (401), forbidden +(403), badMethod (405), HTTPUnprocessableEntity (422), +internalServerError (500), serviceUnavailable (503) + +**Response parameters** + +.. csv-table:: + :header: "Parameter", "Style", "Type", "Description" + :widths: 20, 20, 20, 60 + + "subcloud_groups (Optional)", "plain", "xsd:list", "The list of subcloud groups." + "id (Optional)", "plain", "xsd:int", "The unique identifier for this object." + "name (Optional)", "plain", "xsd:string", "The unique name for the subcloud group." + "description (Optional)", "plain", "xsd:string", "The description of the subcloud group." + "update_apply_type (Optional)", "plain", "xsd:string", "The method for applying an update. ```serial``` or ```parallel```." + "max_parallel_subclouds (Optional)", "plain", "xsd:int", "The maximum number of subclouds to update in parallel." + "created_at (Optional)", "plain", "xsd:dateTime", "The time when the object was created." + "updated_at (Optional)", "plain", "xsd:dateTime", "The time when the object was last updated." + +:: + + { + "subcloud_groups": [ + { + "update_apply_type": "parallel", + "description": "Default Subcloud Group", + "updated-at": null, + "created-at": null, + "max_parallel_subclouds": 2, + "id": 1, + "name": "Default" + }, + ] + } + +This operation does not accept a request body. + +************************* +Creates a subcloud group +************************* + +.. rest_method:: POST /v1.0/subcloud-groups + +**Normal response codes** + +200 + +**Error response codes** + +badRequest (400), unauthorized (401), forbidden (403), badMethod (405), +HTTPUnprocessableEntity (422), internalServerError (500), +serviceUnavailable (503) + +**Request parameters** + +.. csv-table:: + :header: "Parameter", "Style", "Type", "Description" + :widths: 20, 20, 20, 60 + + "name (Optional)", "plain", "xsd:string", "The name for the subcloud group. Must be unique." + "description (Optional)", "plain", "xsd:string", "The description of the subcloud group." + "update_apply_type (Optional)", "plain", "xsd:string", "The method for applying an update. Must be ```serial``` or ```parallel```." + "max_parallel_subclouds (Optional)", "plain", "xsd:int", "The maximum number of subclouds to update in parallel. Must be greater than 0." + +**Response parameters** + +.. csv-table:: + :header: "Parameter", "Style", "Type", "Description" + :widths: 20, 20, 20, 60 + + "id (Optional)", "plain", "xsd:int", "The unique identifier for this object." + "name (Optional)", "plain", "xsd:string", "The unique name for the subcloud group." + "description (Optional)", "plain", "xsd:string", "The description of the subcloud group." + "update_apply_type (Optional)", "plain", "xsd:string", "The method for applying an update. ```serial``` or ```parallel```." + "max_parallel_subclouds (Optional)", "plain", "xsd:int", "The maximum number of subclouds to update in parallel." + +:: + + { + "name": "GroupX", + "description": "A new group", + "update_apply_type": "parallel", + "max_parallel_subclouds": 3 + } + +:: + + { + "id": 2, + "name": "GroupX", + "description": "A new group", + "update_apply_type": "parallel", + "max_parallel_subclouds": "3", + "updated-at": null, + "created-at": "2020-04-08 15:15:10.750592", + } + +****************************************************** +Shows information about a specific subcloud group +****************************************************** + +.. rest_method:: GET /v1.0/subcloud-groups/​{subcloud-group}​ + +**Normal response codes** + +200 + +**Error response codes** + +itemNotFound (404), badRequest (400), unauthorized (401), forbidden +(403), badMethod (405), HTTPUnprocessableEntity (422), +internalServerError (500), serviceUnavailable (503) + +**Request parameters** + +.. csv-table:: + :header: "Parameter", "Style", "Type", "Description" + :widths: 20, 20, 20, 60 + + "subcloud-group", "URI", "xsd:string", "The subcloud group reference, name or id." + +**Response parameters** + +.. csv-table:: + :header: "Parameter", "Style", "Type", "Description" + :widths: 20, 20, 20, 60 + + "id (Optional)", "plain", "xsd:int", "The unique identifier for this object." + "name (Optional)", "plain", "xsd:string", "The name provisioned for the subcloud group." + "description (Optional)", "plain", "xsd:string", "The description for the subcloud group." + "max_parallel_subclouds (Optional)", "plain", "xsd:int", "The maximum number of subclouds to update in parallel." + "update_apply_type (Optional)", "plain", "xsd:string", "The update apply type for the subcloud group: ```serial``` or ```parallel```." + "created_at (Optional)", "plain", "xsd:dateTime", "The time when the object was created." + "updated_at (Optional)", "plain", "xsd:dateTime", "The time when the object was last updated." + +:: + + { + "id": 2, + "name": "GroupX", + "description": "A new group", + "max_parallel_subclouds": 3, + "update_apply_type": "parallel", + "created-at": "2020-04-08 15:15:10.750592", + "updated-at": null + } + +This operation does not accept a request body. + +****************************************************** +Shows subclouds that are part of a subcloud group +****************************************************** + +.. rest_method:: GET /v1.0/subcloud-groups/​{subcloud-group}​/subclouds + +**Normal response codes** + +200 + +**Error response codes** + +itemNotFound (404), badRequest (400), unauthorized (401), forbidden +(403), badMethod (405), HTTPUnprocessableEntity (422), +internalServerError (500), serviceUnavailable (503) + +**Request parameters** + +.. csv-table:: + :header: "Parameter", "Style", "Type", "Description" + :widths: 20, 20, 20, 60 + + "subcloud-group", "URI", "xsd:string", "The subcloud group reference, name or id." + +**Response parameters** + +.. csv-table:: + :header: "Parameter", "Style", "Type", "Description" + :widths: 20, 20, 20, 60 + + "subclouds (Optional)", "plain", "xsd:list", "The list of subclouds." + "id (Optional)", "plain", "xsd:int", "The unique identifier for a subcloud." + "group_id (Optional)", "plain", "xsd:int", "The unique identifier for the subcloud group." + "created_at (Optional)", "plain", "xsd:dateTime", "The time when the object was created." + "updated_at (Optional)", "plain", "xsd:dateTime", "The time when the object was last updated." + "name (Optional)", "plain", "xsd:string", "The name provisioned for the subcloud." + "management-state (Optional)", "plain", "xsd:string", "Management state of the subcloud." + "management-start-ip (Optional)", "plain", "xsd:string", "Start of management IP address range for subcloud." + "software-version (Optional)", "plain", "xsd:string", "Software version for subcloud." + "availability-status (Optional)", "plain", "xsd:string", "Availability status of the subcloud." + "systemcontroller-gateway-ip (Optional)", "plain", "xsd:string", "Systemcontroller gateway IP Address." + "location (Optional)", "plain", "xsd:string", "The location provisioned for the subcloud." + "openstack-installed (Optional)", "plain", "xsd:boolean", "Whether openstack is installed on the subcloud." + "management-subnet (Optional)", "plain", "xsd:string", "Management subnet for subcloud in CIDR format." + "management-gateway-ip (Optional)", "plain", "xsd:string", "Management gateway IP for subcloud." + "management-end-ip (Optional)", "plain", "xsd:string", "End of management IP address range for subcloud." + "description (Optional)", "plain", "xsd:string", "The description provisioned for the subcloud." + +:: + + { + "subclouds": [ + { + "deploy-status": "complete", + "id": 1, + "group_id": 2, + "created-at": "2020-04-13 13:16:21.903294", + "updated-at": "2020-04-13 13:36:27.494056", + "name": "subcloud1", + "management-state": "unmanaged", + "management-start-ip": "192.168.101.2", + "software-version": "20.01", + "availability-status": "offline", + "systemcontroller-gateway-ip": "192.168.204.101", + "location": "YOW", + "openstack-installed": false, + "management-subnet": "192.168.101.0/24", + "management-gateway-ip": "192.168.101.1", + "management-end-ip": "192.168.101.50", + "description": "Ottawa Site" + } + ] + } + +This operation does not accept a request body. + +*********************************** +Modifies a specific subcloud group +*********************************** + +.. rest_method:: PATCH /v1.0/subcloud-groups/​{subcloud-group}​ + +The attributes of a subcloud group which are modifiable: + +- name + +- description + +- update_apply_type + +- max_parallel_subclouds + +**Normal response codes** + +200 + +**Error response codes** + +badRequest (400), unauthorized (401), forbidden (403), badMethod (405), +HTTPUnprocessableEntity (422), internalServerError (500), +serviceUnavailable (503) + +**Request parameters** + +.. csv-table:: + :header: "Parameter", "Style", "Type", "Description" + :widths: 20, 20, 20, 60 + + "subcloud-group", "URI", "xsd:string", "The subcloud group reference, name or id." + "name (Optional)", "plain", "xsd:string", "The name of the subcloud group. Must be unique." + "description (Optional)", "plain", "xsd:string", "The description of the subcloud group." + "update_apply_type (Optional)", "plain", "xsd:string", "The update apply type for the subcloud group. Either ```serial``` or ```parallel```." + "max_parallel_subclouds (Optional)", "plain", "xsd:int", "The number of subclouds to update in parallel. Must be greater than 0." + +**Response parameters** + +.. csv-table:: + :header: "Parameter", "Style", "Type", "Description" + :widths: 20, 20, 20, 60 + + "id (Optional)", "plain", "xsd:int", "The unique identifier for this object." + "name (Optional)", "plain", "xsd:string", "The name provisioned for the subcloud group." + "description (Optional)", "plain", "xsd:string", "The description for the subcloud group." + "created_at (Optional)", "plain", "xsd:dateTime", "The time when the object was created." + "updated_at (Optional)", "plain", "xsd:dateTime", "The time when the object was last updated." + +:: + + { + "description": "new description", + "update_apply_type": "serial", + "max_parallel_subclouds": 5 + } + +:: + + { + "id": 2, + "name": "GroupX", + "description": "new description", + "update_apply_type": "serial", + "max_parallel_subclouds": 5, + "created-at": "2020-04-08 15:15:10.750592", + "updated-at": "2020-04-08 15:21:01.527101" + } + +********************************** +Deletes a specific subcloud group +********************************** + +.. rest_method:: DELETE /v1.0/subcloud-groups/​{subcloud-group}​ + +**Normal response codes** + +204 + +**Request parameters** + +.. csv-table:: + :header: "Parameter", "Style", "Type", "Description" + :widths: 20, 20, 20, 60 + + "subcloud-group", "URI", "xsd:string", "The subcloud group reference, name or id." + +This operation does not accept a request body. + ---------------- Subcloud Alarms ---------------- diff --git a/distributedcloud/dcmanager/api/controllers/v1/root.py b/distributedcloud/dcmanager/api/controllers/v1/root.py index 9dc4c25b0..4f9682a45 100644 --- a/distributedcloud/dcmanager/api/controllers/v1/root.py +++ b/distributedcloud/dcmanager/api/controllers/v1/root.py @@ -22,6 +22,7 @@ from dcmanager.api.controllers.v1 import alarm_manager +from dcmanager.api.controllers.v1 import subcloud_group from dcmanager.api.controllers.v1 import subclouds from dcmanager.api.controllers.v1 import sw_update_options from dcmanager.api.controllers.v1 import sw_update_strategy @@ -46,6 +47,8 @@ class Controller(object): sw_update_strategy.SwUpdateStrategyController sub_controllers["sw-update-options"] = \ sw_update_options.SwUpdateOptionsController + sub_controllers["subcloud-groups"] = \ + subcloud_group.SubcloudGroupsController for name, ctrl in sub_controllers.items(): setattr(self, name, ctrl) diff --git a/distributedcloud/dcmanager/api/controllers/v1/subcloud_group.py b/distributedcloud/dcmanager/api/controllers/v1/subcloud_group.py new file mode 100755 index 000000000..082621f6c --- /dev/null +++ b/distributedcloud/dcmanager/api/controllers/v1/subcloud_group.py @@ -0,0 +1,313 @@ +# Copyright (c) 2017 Ericsson AB. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Copyright (c) 2020 Wind River Systems, Inc. +# +# The right to copy, distribute, modify, or otherwise make use +# of this software may be licensed only pursuant to the terms +# of an applicable Wind River license agreement. +# + +from oslo_config import cfg +from oslo_db import exception as db_exc +from oslo_log import log as logging +from oslo_messaging import RemoteError + +import http.client as httpclient +import pecan +from pecan import expose +from pecan import request + +from dcmanager.api.controllers import restcomm +from dcmanager.common import consts +from dcmanager.common import exceptions +from dcmanager.common.i18n import _ +from dcmanager.db import api as db_api +from dcmanager.rpc import client as rpc_client + +CONF = cfg.CONF +LOG = logging.getLogger(__name__) + +SUPPORTED_GROUP_APPLY_TYPES = [ + consts.SUBCLOUD_APPLY_TYPE_PARALLEL, + consts.SUBCLOUD_APPLY_TYPE_SERIAL +] + +# validation constants for Subcloud Group +MAX_SUBCLOUD_GROUP_NAME_LEN = 255 +MAX_SUBCLOUD_GROUP_DESCRIPTION_LEN = 255 +MIN_SUBCLOUD_GROUP_MAX_PARALLEL_SUBCLOUDS = 1 +MAX_SUBCLOUD_GROUP_MAX_PARALLEL_SUBCLOUDS = 100 + + +class SubcloudGroupsController(object): + + def __init__(self): + super(SubcloudGroupsController, self).__init__() + self.rpc_client = rpc_client.ManagerClient() + + @expose(generic=True, template='json') + def index(self): + # Route the request to specific methods with parameters + pass + + def _get_subcloud_list_for_group(self, context, group_id): + subcloud_list = [] + subclouds = db_api.subcloud_get_for_group(context, group_id) + + for subcloud in subclouds: + subcloud_dict = db_api.subcloud_db_model_to_dict(subcloud) + subcloud_list.append(subcloud_dict) + result = dict() + result['subclouds'] = subcloud_list + return result + + def _get_subcloud_group_list(self, context): + groups = db_api.subcloud_group_get_all(context) + subcloud_group_list = [] + + for group in groups: + group_dict = db_api.subcloud_group_db_model_to_dict(group) + subcloud_group_list.append(group_dict) + + result = dict() + result['subcloud_groups'] = subcloud_group_list + return result + + def _get_by_ref(self, context, group_ref): + # Handle getting a group by either name, or ID + group = None + if group_ref.isdigit(): + # Lookup subcloud group as an ID + try: + group = db_api.subcloud_group_get(context, group_ref) + except exceptions.SubcloudGroupNotFound: + return None + else: + # Lookup subcloud group as a name + try: + group = db_api.subcloud_group_get_by_name(context, group_ref) + except exceptions.SubcloudGroupNameNotFound: + return None + return group + + @index.when(method='GET', template='json') + def get(self, group_ref=None, subclouds=False): + """Get details about subcloud group. + + :param group_ref: ID or name of subcloud group + """ + context = restcomm.extract_context_from_environ() + + if group_ref is None: + # List of subcloud groups requested + return self._get_subcloud_group_list(context) + + group = self._get_by_ref(context, group_ref) + if group is None: + pecan.abort(httpclient.NOT_FOUND, _('Subcloud Group not found')) + if subclouds: + # Return only the subclouds for this subcloud group + return self._get_subcloud_list_for_group(context, group.id) + subcloud_group_dict = db_api.subcloud_group_db_model_to_dict(group) + return subcloud_group_dict + + def _validate_name(self, name): + # Reject post and update operations for name that: + # - attempt to set to None + # - attempt to set to a number + # - attempt to set to the Default subcloud group + # - exceed the max length + if not name: + return False + if name.isdigit(): + return False + if name == consts.DEFAULT_SUBCLOUD_GROUP_NAME: + return False + if len(name) >= MAX_SUBCLOUD_GROUP_NAME_LEN: + return False + return True + + def _validate_description(self, description): + if not description: + return False + if len(description) >= MAX_SUBCLOUD_GROUP_DESCRIPTION_LEN: + return False + return True + + def _validate_update_apply_type(self, update_apply_type): + if not update_apply_type: + return False + if update_apply_type not in SUPPORTED_GROUP_APPLY_TYPES: + return False + return True + + def _validate_max_parallel_subclouds(self, max_parallel_str): + if not max_parallel_str: + return False + try: + # Check the value is an integer + val = int(max_parallel_str) + except ValueError: + return False + + # We do not support less than min or greater than max + if val < MIN_SUBCLOUD_GROUP_MAX_PARALLEL_SUBCLOUDS: + return False + if val > MAX_SUBCLOUD_GROUP_MAX_PARALLEL_SUBCLOUDS: + return False + return True + + @index.when(method='POST', template='json') + def post(self): + """Create a new subcloud group.""" + context = restcomm.extract_context_from_environ() + + payload = eval(request.body) + if not payload: + pecan.abort(httpclient.BAD_REQUEST, _('Body required')) + + name = payload.get('name') + description = payload.get('description') + update_apply_type = payload.get('update_apply_type') + max_parallel_subclouds = payload.get('max_parallel_subclouds') + + # Validate payload + if not self._validate_name(name): + pecan.abort(httpclient.BAD_REQUEST, _('Invalid group name')) + if not self._validate_description(description): + pecan.abort(httpclient.BAD_REQUEST, _('Invalid group description')) + if not self._validate_update_apply_type(update_apply_type): + pecan.abort(httpclient.BAD_REQUEST, + _('Invalid group update_apply_type')) + if not self._validate_max_parallel_subclouds(max_parallel_subclouds): + pecan.abort(httpclient.BAD_REQUEST, + _('Invalid group max_parallel_subclouds')) + try: + group_ref = db_api.subcloud_group_create(context, + name, + description, + update_apply_type, + max_parallel_subclouds) + return db_api.subcloud_group_db_model_to_dict(group_ref) + except db_exc.DBDuplicateEntry: + LOG.info("Group create failed. Group %s already exists" % name) + pecan.abort(httpclient.BAD_REQUEST, + _('A subcloud group with this name already exists')) + except RemoteError as e: + pecan.abort(httpclient.UNPROCESSABLE_ENTITY, e.value) + except Exception as e: + # TODO(abailey) add support for GROUP already exists (409) + LOG.exception(e) + pecan.abort(httpclient.INTERNAL_SERVER_ERROR, + _('Unable to create subcloud group')) + + @index.when(method='PATCH', template='json') + def patch(self, group_ref): + """Update a subcloud group. + + :param group_ref: ID or name of subcloud group to update + """ + + context = restcomm.extract_context_from_environ() + if group_ref is None: + pecan.abort(httpclient.BAD_REQUEST, + _('Subcloud Group Name or ID required')) + + payload = eval(request.body) + if not payload: + pecan.abort(httpclient.BAD_REQUEST, _('Body required')) + + group = self._get_by_ref(context, group_ref) + if group is None: + pecan.abort(httpclient.NOT_FOUND, _('Subcloud Group not found')) + + name = payload.get('name') + description = payload.get('description') + update_apply_type = payload.get('update_apply_type') + max_parallel_str = payload.get('max_parallel_subclouds') + + if not (name or description or update_apply_type or max_parallel_str): + pecan.abort(httpclient.BAD_REQUEST, _('nothing to update')) + + # Check value is not None or empty before calling validate + if name: + if not self._validate_name(name): + pecan.abort(httpclient.BAD_REQUEST, + _('Invalid group name')) + # Special case. Default group name cannot be changed + if group.id == consts.DEFAULT_SUBCLOUD_GROUP_ID: + pecan.abort(httpclient.BAD_REQUEST, + _('Default group name cannot be changed')) + + if description: + if not self._validate_description(description): + pecan.abort(httpclient.BAD_REQUEST, + _('Invalid group description')) + if update_apply_type: + if not self._validate_update_apply_type(update_apply_type): + pecan.abort(httpclient.BAD_REQUEST, + _('Invalid group update_apply_type')) + if max_parallel_str: + if not self._validate_max_parallel_subclouds(max_parallel_str): + pecan.abort(httpclient.BAD_REQUEST, + _('Invalid group max_parallel_subclouds')) + + try: + updated_group = db_api.subcloud_group_update( + context, + group.id, + name=name, + description=description, + update_apply_type=update_apply_type, + max_parallel_subclouds=max_parallel_str) + return db_api.subcloud_group_db_model_to_dict(updated_group) + except RemoteError as e: + pecan.abort(httpclient.UNPROCESSABLE_ENTITY, e.value) + except Exception as e: + # additional exceptions. + LOG.exception(e) + pecan.abort(httpclient.INTERNAL_SERVER_ERROR, + _('Unable to update subcloud group')) + + @index.when(method='delete', template='json') + def delete(self, group_ref): + """Delete the subcloud group.""" + context = restcomm.extract_context_from_environ() + + if group_ref is None: + pecan.abort(httpclient.BAD_REQUEST, + _('Subcloud Group Name or ID required')) + group = self._get_by_ref(context, group_ref) + if group is None: + pecan.abort(httpclient.NOT_FOUND, _('Subcloud Group not found')) + if group.name == consts.DEFAULT_SUBCLOUD_GROUP_NAME: + pecan.abort(httpclient.BAD_REQUEST, + _('Default Subcloud Group may not be deleted')) + try: + # a subcloud group may not be deleted if it is use by any subclouds + subclouds = db_api.subcloud_get_for_group(context, group.id) + if len(subclouds) > 0: + pecan.abort(httpclient.BAD_REQUEST, + _('Subcloud Group not empty')) + db_api.subcloud_group_destroy(context, group.id) + except RemoteError as e: + pecan.abort(httpclient.UNPROCESSABLE_ENTITY, e.value) + except Exception as e: + LOG.exception(e) + pecan.abort(httpclient.INTERNAL_SERVER_ERROR, + _('Unable to delete subcloud group')) + # This should return nothing + return None diff --git a/distributedcloud/dcmanager/api/controllers/v1/subclouds.py b/distributedcloud/dcmanager/api/controllers/v1/subclouds.py index 0bebabb15..8eff4de9c 100644 --- a/distributedcloud/dcmanager/api/controllers/v1/subclouds.py +++ b/distributedcloud/dcmanager/api/controllers/v1/subclouds.py @@ -32,10 +32,6 @@ import pecan from pecan import expose from pecan import request -from controllerconfig.common.exceptions import ValidateFail -from controllerconfig.utils import validate_address_str -from controllerconfig.utils import validate_network_str - from dccommon.drivers.openstack.keystone_v3 import KeystoneClient from dccommon.drivers.openstack.sysinv_v1 import SysinvClient from dccommon import exceptions as dccommon_exceptions @@ -81,6 +77,14 @@ class SubcloudsController(object): # Route the request to specific methods with parameters pass + def _validate_group_id(self, context, group_id): + try: + # The DB API will raise an exception if the group_id is invalid + db_api.subcloud_group_get(context, group_id) + except Exception as e: + LOG.exception(e) + pecan.abort(400, _("Invalid group_id")) + def _validate_subcloud_config(self, context, name, @@ -91,7 +95,8 @@ class SubcloudsController(object): external_oam_subnet_str, external_oam_gateway_address_str, external_oam_floating_address_str, - systemcontroller_gateway_ip_str): + systemcontroller_gateway_ip_str, + group_id): """Check whether subcloud config is valid.""" # Validate the name @@ -116,28 +121,28 @@ class SubcloudsController(object): management_subnet = None try: - management_subnet = validate_network_str( + management_subnet = utils.validate_network_str( management_subnet_str, minimum_size=MIN_MANAGEMENT_SUBNET_SIZE, existing_networks=subcloud_subnets) - except ValidateFail as e: + except exceptions.ValidateFail as e: LOG.exception(e) pecan.abort(400, _("management_subnet invalid: %s") % e) # Parse/validate the start/end addresses management_start_ip = None try: - management_start_ip = validate_address_str( + management_start_ip = utils.validate_address_str( management_start_ip_str, management_subnet) - except ValidateFail as e: + except exceptions.ValidateFail as e: LOG.exception(e) pecan.abort(400, _("management_start_address invalid: %s") % e) management_end_ip = None try: - management_end_ip = validate_address_str( + management_end_ip = utils.validate_address_str( management_end_ip_str, management_subnet) - except ValidateFail as e: + except exceptions.ValidateFail as e: LOG.exception(e) pecan.abort(400, _("management_end_address invalid: %s") % e) @@ -156,9 +161,9 @@ class SubcloudsController(object): # Parse/validate the gateway try: - validate_address_str( + utils.validate_address_str( management_gateway_ip_str, management_subnet) - except ValidateFail as e: + except exceptions.ValidateFail as e: LOG.exception(e) pecan.abort(400, _("management_gateway_address invalid: %s") % e) @@ -185,9 +190,9 @@ class SubcloudsController(object): management_address_pool.prefix) systemcontroller_subnet = IPNetwork(systemcontroller_subnet_str) try: - validate_address_str( + utils.validate_address_str( systemcontroller_gateway_ip_str, systemcontroller_subnet) - except ValidateFail as e: + except exceptions.ValidateFail as e: LOG.exception(e) pecan.abort(400, _("systemcontroller_gateway_address invalid: %s") % e) @@ -207,28 +212,29 @@ class SubcloudsController(object): MIN_OAM_SUBNET_SIZE = 3 oam_subnet = None try: - oam_subnet = validate_network_str( + oam_subnet = utils.validate_network_str( external_oam_subnet_str, minimum_size=MIN_OAM_SUBNET_SIZE, existing_networks=subcloud_subnets) - except ValidateFail as e: + except exceptions.ValidateFail as e: LOG.exception(e) pecan.abort(400, _("external_oam_subnet invalid: %s") % e) # Parse/validate the addresses try: - validate_address_str( + utils.validate_address_str( external_oam_gateway_address_str, oam_subnet) - except ValidateFail as e: + except exceptions.ValidateFail as e: LOG.exception(e) pecan.abort(400, _("oam_gateway_address invalid: %s") % e) try: - validate_address_str( + utils.validate_address_str( external_oam_floating_address_str, oam_subnet) - except ValidateFail as e: + except exceptions.ValidateFail as e: LOG.exception(e) pecan.abort(400, _("oam_floating_address invalid: %s") % e) + self._validate_group_id(context, group_id) @staticmethod def _validate_install_values(payload): @@ -288,8 +294,8 @@ class SubcloudsController(object): network_str = (install_values['network_address'] + '/' + str(install_values['network_mask'])) try: - network = validate_network_str(network_str, 1) - except ValidateFail as e: + network = utils.validate_network_str(network_str, 1) + except exceptions.ValidateFail as e: LOG.exception(e) pecan.abort(400, _("network address invalid: %s") % e) @@ -546,6 +552,10 @@ class SubcloudsController(object): if not sysadmin_password: pecan.abort(400, _('subcloud sysadmin_password required')) + # If a subcloud group is not passed, use the default + group_id = payload.get('group_id', + consts.DEFAULT_SUBCLOUD_GROUP_ID) + self._validate_subcloud_config(context, name, management_subnet, @@ -555,7 +565,8 @@ class SubcloudsController(object): external_oam_subnet, external_oam_gateway_ip, external_oam_floating_ip, - systemcontroller_gateway_ip) + systemcontroller_gateway_ip, + group_id) if 'install_values' in payload: self._validate_install_values(payload) @@ -609,8 +620,9 @@ class SubcloudsController(object): management_state = payload.get('management-state') description = payload.get('description') location = payload.get('location') + group_id = payload.get('group_id') - if not (management_state or description or location): + if not (management_state or description or location or group_id): pecan.abort(400, _('nothing to update')) # Syntax checking @@ -619,12 +631,19 @@ class SubcloudsController(object): consts.MANAGEMENT_MANAGED]: pecan.abort(400, _('Invalid management-state')) + # Verify the group_id is valid + if group_id: + try: + db_api.subcloud_group_get(context, group_id) + except exceptions.SubcloudGroupNotFound: + pecan.abort(400, _('Invalid group-id')) + try: # Inform dcmanager-manager that subcloud has been updated. # It will do all the real work... subcloud = self.rpc_client.update_subcloud( context, subcloud_id, management_state=management_state, - description=description, location=location) + description=description, location=location, group_id=group_id) return subcloud except RemoteError as e: pecan.abort(422, e.value) diff --git a/distributedcloud/dcmanager/common/consts.py b/distributedcloud/dcmanager/common/consts.py index 3f1f15d74..c0f4f4c14 100644 --- a/distributedcloud/dcmanager/common/consts.py +++ b/distributedcloud/dcmanager/common/consts.py @@ -77,6 +77,13 @@ SW_UPDATE_ACTION_ABORT = "abort" SUBCLOUD_APPLY_TYPE_PARALLEL = "parallel" SUBCLOUD_APPLY_TYPE_SERIAL = "serial" +# Values for the Default Subcloud Group +DEFAULT_SUBCLOUD_GROUP_ID = 1 +DEFAULT_SUBCLOUD_GROUP_NAME = 'Default' +DEFAULT_SUBCLOUD_GROUP_DESCRIPTION = 'Default Subcloud Group' +DEFAULT_SUBCLOUD_GROUP_UPDATE_APPLY_TYPE = SUBCLOUD_APPLY_TYPE_PARALLEL +DEFAULT_SUBCLOUD_GROUP_MAX_PARALLEL_SUBCLOUDS = 2 + # Strategy step states STRATEGY_STATE_INITIAL = "initial" STRATEGY_STATE_UPDATING_PATCHES = "updating patches" diff --git a/distributedcloud/dcmanager/common/exceptions.py b/distributedcloud/dcmanager/common/exceptions.py index fb603943d..335963c4d 100644 --- a/distributedcloud/dcmanager/common/exceptions.py +++ b/distributedcloud/dcmanager/common/exceptions.py @@ -65,6 +65,12 @@ class BadRequest(DCManagerException): message = _('Bad %(resource)s request: %(msg)s') +class ValidateFail(DCManagerException): + def __init__(self, message): + self.message = message + super(ValidateFail, self).__init__() + + class NotFound(DCManagerException): message = _("Not found") @@ -124,6 +130,22 @@ class SubcloudPatchOptsNotFound(NotFound): "defaults will be used.") +class SubcloudGroupNotFound(NotFound): + message = _("Subcloud Group with id %(group_id)s doesn't exist.") + + +class SubcloudGroupNameNotFound(NotFound): + message = _("Subcloud Group with name %(name)s doesn't exist.") + + +class SubcloudGroupNameViolation(DCManagerException): + message = _("Default Subcloud Group name cannot be changed or reused.") + + +class SubcloudGroupDefaultNotDeletable(DCManagerException): + message = _("Default Subcloud Group %(group_id)s may not be deleted.") + + class ConnectionRefused(DCManagerException): message = _("Connection to the service endpoint is refused") diff --git a/distributedcloud/dcmanager/common/utils.py b/distributedcloud/dcmanager/common/utils.py index 1ac04c25c..37c31bed0 100644 --- a/distributedcloud/dcmanager/common/utils.py +++ b/distributedcloud/dcmanager/common/utils.py @@ -22,6 +22,7 @@ import grp import itertools +import netaddr import os import pwd import six.moves @@ -53,6 +54,59 @@ def get_batch_projects(batch_size, project_list, fillvalue=None): return six.moves.zip_longest(fillvalue=fillvalue, *args) +def validate_address_str(ip_address_str, network): + """Determine whether an address is valid.""" + try: + ip_address = netaddr.IPAddress(ip_address_str) + if ip_address.version != network.version: + msg = ("Invalid IP version - must match network version " + + ip_version_to_string(network.version)) + raise exceptions.ValidateFail(msg) + elif ip_address == network: + raise exceptions.ValidateFail("Cannot use network address") + elif ip_address == network.broadcast: + raise exceptions.ValidateFail("Cannot use broadcast address") + elif ip_address not in network: + raise exceptions.ValidateFail( + "Address must be in subnet %s" % str(network)) + return ip_address + except netaddr.AddrFormatError: + raise exceptions.ValidateFail( + "Invalid address - not a valid IP address") + + +def ip_version_to_string(ip_version): + """Returns a string representation of ip_version.""" + if ip_version == 4: + return "IPv4" + elif ip_version == 6: + return "IPv6" + else: + return "IP" + + +def validate_network_str(network_str, minimum_size, + existing_networks=None, multicast=False): + """Determine whether a network is valid.""" + try: + network = netaddr.IPNetwork(network_str) + if network.size < minimum_size: + raise exceptions.ValidateFail("Subnet too small - must have at " + "least %d addresses" % minimum_size) + elif network.version == 6 and network.prefixlen < 64: + raise exceptions.ValidateFail("IPv6 minimum prefix length is 64") + elif existing_networks: + if any(network.ip in subnet for subnet in existing_networks): + raise exceptions.ValidateFail("Subnet overlaps with another " + "configured subnet") + elif multicast and not network.is_multicast(): + raise exceptions.ValidateFail("Invalid subnet - must be multicast") + return network + except netaddr.AddrFormatError: + raise exceptions.ValidateFail( + "Invalid subnet - not a valid IP subnet") + + # to do validate the quota limits def validate_quota_limits(payload): for resource in payload: diff --git a/distributedcloud/dcmanager/db/api.py b/distributedcloud/dcmanager/db/api.py index c6b288765..1e204defa 100644 --- a/distributedcloud/dcmanager/db/api.py +++ b/distributedcloud/dcmanager/db/api.py @@ -67,7 +67,8 @@ def subcloud_db_model_to_dict(subcloud): "systemcontroller-gateway-ip": subcloud.systemcontroller_gateway_ip, "created-at": subcloud.created_at, - "updated-at": subcloud.updated_at} + "updated-at": subcloud.updated_at, + "group_id": subcloud.group_id} return result @@ -75,14 +76,14 @@ def subcloud_create(context, name, description, location, software_version, management_subnet, management_gateway_ip, management_start_ip, management_end_ip, systemcontroller_gateway_ip, deploy_status, - openstack_installed): + openstack_installed, group_id): """Create a subcloud.""" return IMPL.subcloud_create(context, name, description, location, software_version, management_subnet, management_gateway_ip, management_start_ip, management_end_ip, systemcontroller_gateway_ip, deploy_status, - openstack_installed) + openstack_installed, group_id) def subcloud_get(context, subcloud_id): @@ -113,12 +114,13 @@ def subcloud_get_all_with_status(context): def subcloud_update(context, subcloud_id, management_state=None, availability_status=None, software_version=None, description=None, location=None, audit_fail_count=None, - deploy_status=None, openstack_installed=None): + deploy_status=None, openstack_installed=None, + group_id=None): """Update a subcloud or raise if it does not exist.""" return IMPL.subcloud_update(context, subcloud_id, management_state, availability_status, software_version, description, location, audit_fail_count, - deploy_status, openstack_installed) + deploy_status, openstack_installed, group_id) def subcloud_destroy(context, subcloud_id): @@ -196,6 +198,67 @@ def subcloud_status_destroy_all(context, subcloud_id): return IMPL.subcloud_status_destroy_all(context, subcloud_id) +################### +# subcloud_group + +def subcloud_group_db_model_to_dict(subcloud_group): + """Convert subcloud_group db model to dictionary.""" + result = {"id": subcloud_group.id, + "name": subcloud_group.name, + "description": subcloud_group.description, + "update_apply_type": subcloud_group.update_apply_type, + "max_parallel_subclouds": subcloud_group.max_parallel_subclouds, + "created-at": subcloud_group.created_at, + "updated-at": subcloud_group.updated_at} + return result + + +def subcloud_group_create(context, name, description, update_apply_type, + max_parallel_subclouds): + """Create a subcloud_group.""" + return IMPL.subcloud_group_create(context, + name, + description, + update_apply_type, + max_parallel_subclouds) + + +def subcloud_group_get(context, group_id): + """Retrieve a subcloud_group or raise if it does not exist.""" + return IMPL.subcloud_group_get(context, group_id) + + +def subcloud_group_get_by_name(context, name): + """Retrieve a subcloud_group b name or raise if it does not exist.""" + return IMPL.subcloud_group_get_by_name(context, name) + + +def subcloud_group_get_all(context): + """Retrieve all subcloud groups.""" + return IMPL.subcloud_group_get_all(context) + + +def subcloud_get_for_group(context, group_id): + """Retrieve a subcloud_group or raise if it does not exist.""" + return IMPL.subcloud_get_for_group(context, group_id) + + +def subcloud_group_update(context, group_id, name, description, + update_apply_type, max_parallel_subclouds): + """Update the subcloud group or raise if it does not exist.""" + return IMPL.subcloud_group_update(context, + group_id, + name, + description, + update_apply_type, + max_parallel_subclouds) + + +def subcloud_group_destroy(context, group_id): + """Destroy the subcloud group or raise if it does not exist.""" + return IMPL.subcloud_group_destroy(context, group_id) + + ################### def sw_update_strategy_db_model_to_dict(sw_update_strategy): diff --git a/distributedcloud/dcmanager/db/sqlalchemy/api.py b/distributedcloud/dcmanager/db/sqlalchemy/api.py index 88c98051d..7d604452d 100644 --- a/distributedcloud/dcmanager/db/sqlalchemy/api.py +++ b/distributedcloud/dcmanager/db/sqlalchemy/api.py @@ -24,12 +24,13 @@ Implementation of SQLAlchemy backend. """ +import sqlalchemy import sys import threading from oslo_db import exception as db_exc +from oslo_db.exception import DBDuplicateEntry from oslo_db.sqlalchemy import enginefacade - from oslo_log import log as logging from oslo_utils import strutils from oslo_utils import uuidutils @@ -212,7 +213,7 @@ def subcloud_create(context, name, description, location, software_version, management_subnet, management_gateway_ip, management_start_ip, management_end_ip, systemcontroller_gateway_ip, deploy_status, - openstack_installed): + openstack_installed, group_id): with write_session() as session: subcloud_ref = models.Subcloud() subcloud_ref.name = name @@ -229,6 +230,7 @@ def subcloud_create(context, name, description, location, software_version, subcloud_ref.deploy_status = deploy_status subcloud_ref.audit_fail_count = 0 subcloud_ref.openstack_installed = openstack_installed + subcloud_ref.group_id = group_id session.add(subcloud_ref) return subcloud_ref @@ -237,7 +239,8 @@ def subcloud_create(context, name, description, location, software_version, def subcloud_update(context, subcloud_id, management_state=None, availability_status=None, software_version=None, description=None, location=None, audit_fail_count=None, - deploy_status=None, openstack_installed=None): + deploy_status=None, openstack_installed=None, + group_id=None): with write_session() as session: subcloud_ref = subcloud_get(context, subcloud_id) if management_state is not None: @@ -256,6 +259,8 @@ def subcloud_update(context, subcloud_id, management_state=None, subcloud_ref.deploy_status = deploy_status if openstack_installed is not None: subcloud_ref.openstack_installed = openstack_installed + if group_id is not None: + subcloud_ref.group_id = group_id subcloud_ref.save(session) return subcloud_ref @@ -269,7 +274,6 @@ def subcloud_destroy(context, subcloud_id): ########################## - @require_context def subcloud_status_get(context, subcloud_id, endpoint_type): result = model_query(context, models.SubcloudStatus). \ @@ -531,6 +535,138 @@ def sw_update_opts_default_destroy(context): session.delete(sw_update_opts_default_ref) +########################## +# subcloud group +########################## +@require_context +def subcloud_group_get(context, group_id): + try: + result = model_query(context, models.SubcloudGroup). \ + filter_by(deleted=0). \ + filter_by(id=group_id). \ + one() + except NoResultFound: + raise exception.SubcloudGroupNotFound(group_id=group_id) + except MultipleResultsFound: + raise exception.InvalidParameterValue( + err="Multiple entries found for subcloud group %s" % group_id) + + return result + + +@require_context +def subcloud_group_get_by_name(context, name): + try: + result = model_query(context, models.SubcloudGroup). \ + filter_by(deleted=0). \ + filter_by(name=name). \ + one() + except NoResultFound: + raise exception.SubcloudGroupNameNotFound(name=name) + except MultipleResultsFound: + # This exception should never happen due to the UNIQUE setting for name + raise exception.InvalidParameterValue( + err="Multiple entries found for subcloud group %s" % name) + + return result + + +# This method returns all subclouds for a particular subcloud group +@require_context +def subcloud_get_for_group(context, group_id): + return model_query(context, models.Subcloud). \ + filter_by(deleted=0). \ + filter_by(group_id=group_id). \ + order_by(models.Subcloud.id). \ + all() + + +@require_context +def subcloud_group_get_all(context): + result = model_query(context, models.SubcloudGroup). \ + filter_by(deleted=0). \ + order_by(models.SubcloudGroup.id). \ + all() + + return result + + +@require_admin_context +def subcloud_group_create(context, + name, + description, + update_apply_type, + max_parallel_subclouds): + with write_session() as session: + subcloud_group_ref = models.SubcloudGroup() + subcloud_group_ref.name = name + subcloud_group_ref.description = description + subcloud_group_ref.update_apply_type = update_apply_type + subcloud_group_ref.max_parallel_subclouds = max_parallel_subclouds + session.add(subcloud_group_ref) + return subcloud_group_ref + + +@require_admin_context +def subcloud_group_update(context, + group_id, + name=None, + description=None, + update_apply_type=None, + max_parallel_subclouds=None): + with write_session() as session: + subcloud_group_ref = subcloud_group_get(context, group_id) + if name is not None: + # Do not allow the name of the default group to be edited + if subcloud_group_ref.id == consts.DEFAULT_SUBCLOUD_GROUP_ID: + raise exception.SubcloudGroupNameViolation() + # do not allow another group to use the default group name + if name == consts.DEFAULT_SUBCLOUD_GROUP_NAME: + raise exception.SubcloudGroupNameViolation() + subcloud_group_ref.name = name + if description is not None: + subcloud_group_ref.description = description + if update_apply_type is not None: + subcloud_group_ref.update_apply_type = update_apply_type + if max_parallel_subclouds is not None: + subcloud_group_ref.max_parallel_subclouds = max_parallel_subclouds + subcloud_group_ref.save(session) + return subcloud_group_ref + + +@require_admin_context +def subcloud_group_destroy(context, group_id): + with write_session() as session: + subcloud_group_ref = subcloud_group_get(context, group_id) + if subcloud_group_ref.id == consts.DEFAULT_SUBCLOUD_GROUP_ID: + raise exception.SubcloudGroupDefaultNotDeletable(group_id=group_id) + session.delete(subcloud_group_ref) + + +def initialize_subcloud_group_default(engine): + try: + default_group = { + "id": consts.DEFAULT_SUBCLOUD_GROUP_ID, + "name": consts.DEFAULT_SUBCLOUD_GROUP_NAME, + "description": consts.DEFAULT_SUBCLOUD_GROUP_DESCRIPTION, + "update_apply_type": + consts.DEFAULT_SUBCLOUD_GROUP_UPDATE_APPLY_TYPE, + "max_parallel_subclouds": + consts.DEFAULT_SUBCLOUD_GROUP_MAX_PARALLEL_SUBCLOUDS, + "deleted": 0 + } + meta = sqlalchemy.MetaData(bind=engine) + subcloud_group = sqlalchemy.Table('subcloud_group', meta, autoload=True) + try: + with engine.begin() as conn: + conn.execute(subcloud_group.insert(), default_group) + LOG.info("Default Subcloud Group created") + except DBDuplicateEntry: + # The default already exists. + pass + except Exception as ex: + LOG.error("Exception occurred setting up default subcloud group", ex) + ########################## @@ -612,11 +748,18 @@ def strategy_step_destroy_all(context): ########################## +def initialize_db_defaults(engine): + # a default value may already exist. If it does not, create it + initialize_subcloud_group_default(engine) def db_sync(engine, version=None): """Migrate the database to `version` or the most recent version.""" - return migration.db_sync(engine, version=version) + retVal = migration.db_sync(engine, version=version) + # returns None if migration has completed + if retVal is None: + initialize_db_defaults(engine) + return retVal def db_version(engine): diff --git a/distributedcloud/dcmanager/db/sqlalchemy/migrate_repo/versions/006_add_subcloud_group_table.py b/distributedcloud/dcmanager/db/sqlalchemy/migrate_repo/versions/006_add_subcloud_group_table.py new file mode 100644 index 000000000..a8b8b782a --- /dev/null +++ b/distributedcloud/dcmanager/db/sqlalchemy/migrate_repo/versions/006_add_subcloud_group_table.py @@ -0,0 +1,101 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Copyright (c) 2020 Wind River Systems, Inc. +# +# The right to copy, distribute, modify, or otherwise make use +# of this software may be licensed only pursuant to the terms +# of an applicable Wind River license agreement. +# + +from migrate.changeset import constraint +import sqlalchemy + +from dcmanager.common import consts + +ENGINE = 'InnoDB', +CHARSET = 'utf8' + + +def upgrade(migrate_engine): + meta = sqlalchemy.MetaData(bind=migrate_engine) + + # Declare the new subcloud_group table + subcloud_group = sqlalchemy.Table( + 'subcloud_group', meta, + sqlalchemy.Column('id', sqlalchemy.Integer, + primary_key=True, + autoincrement=True, + nullable=False), + sqlalchemy.Column('name', sqlalchemy.String(255), unique=True), + sqlalchemy.Column('description', sqlalchemy.String(255)), + sqlalchemy.Column('update_apply_type', sqlalchemy.String(255)), + sqlalchemy.Column('max_parallel_subclouds', sqlalchemy.Integer), + sqlalchemy.Column('reserved_1', sqlalchemy.Text), + sqlalchemy.Column('reserved_2', sqlalchemy.Text), + sqlalchemy.Column('created_at', sqlalchemy.DateTime), + sqlalchemy.Column('updated_at', sqlalchemy.DateTime), + sqlalchemy.Column('deleted_at', sqlalchemy.DateTime), + sqlalchemy.Column('deleted', sqlalchemy.Integer, default=0), + mysql_engine=ENGINE, + mysql_charset=CHARSET + ) + subcloud_group.create() + + subclouds = sqlalchemy.Table('subclouds', meta, autoload=True) + + # TODO(abailey) do we want to fix the missing constraint for strategy_steps + # strat_steps = sqlalchemy.Table('strategy_steps', meta, autoload=True) + # strat_fkey = constraint.ForeignKeyConstraint( + # columns=[strat_steps.c.subcloud_id], + # refcolumns=[subclouds.c.id], + # name='strat_subcloud_ref') + # strat_steps.append_constraint(strat_fkey) + + # Create a default subcloud group + default_group = { + "id": consts.DEFAULT_SUBCLOUD_GROUP_ID, + "name": consts.DEFAULT_SUBCLOUD_GROUP_NAME, + "description": consts.DEFAULT_SUBCLOUD_GROUP_DESCRIPTION, + "update_apply_type": consts.DEFAULT_SUBCLOUD_GROUP_UPDATE_APPLY_TYPE, + "max_parallel_subclouds": + consts.DEFAULT_SUBCLOUD_GROUP_MAX_PARALLEL_SUBCLOUDS, + "deleted": 0 + } + # Inserting the GROUP as ID 1, + # This should increment the pkey to 2 + with migrate_engine.begin() as conn: + conn.execute(subcloud_group.insert(), default_group) + + # postgres does not increment the subcloud group id sequence + # after the insert above as part of the migrate. + # Note: use different SQL syntax if using mysql or sqlite + if migrate_engine.name == 'postgresql': + with migrate_engine.begin() as conn: + conn.execute("ALTER SEQUENCE subcloud_group_id_seq RESTART WITH 2") + + # Add group_id column to subclouds table + group_id = \ + sqlalchemy.Column('group_id', + sqlalchemy.Integer, + server_default=str(consts.DEFAULT_SUBCLOUD_GROUP_ID)) + group_id.create(subclouds) + + subcloud_fkey = constraint.ForeignKeyConstraint( + columns=[subclouds.c.group_id], + refcolumns=[subcloud_group.c.id], + name='subclouds_group_ref') + subclouds.append_constraint(subcloud_fkey) + + +def downgrade(migrate_engine): + raise NotImplementedError('Database downgrade is unsupported.') diff --git a/distributedcloud/dcmanager/db/sqlalchemy/models.py b/distributedcloud/dcmanager/db/sqlalchemy/models.py index dc106ee6f..76e340839 100644 --- a/distributedcloud/dcmanager/db/sqlalchemy/models.py +++ b/distributedcloud/dcmanager/db/sqlalchemy/models.py @@ -75,6 +75,18 @@ class DCManagerBase(models.ModelBase, session.commit() +class SubcloudGroup(BASE, DCManagerBase): + """Represents a subcloud group""" + + __tablename__ = 'subcloud_group' + + id = Column(Integer, primary_key=True, autoincrement=True, nullable=False) + name = Column(String(255), unique=True) + description = Column(String(255)) + update_apply_type = Column(String(255)) + max_parallel_subclouds = Column(Integer) + + class Subcloud(BASE, DCManagerBase): """Represents a subcloud""" @@ -95,6 +107,11 @@ class Subcloud(BASE, DCManagerBase): openstack_installed = Column(Boolean, nullable=False, default=False) systemcontroller_gateway_ip = Column(String(255)) audit_fail_count = Column(Integer) + # multiple subclouds can be in a particular group + group_id = Column(Integer, + ForeignKey('subcloud_group.id')) + group = relationship(SubcloudGroup, + backref=backref('subcloud')) class SubcloudStatus(BASE, DCManagerBase): diff --git a/distributedcloud/dcmanager/manager/service.py b/distributedcloud/dcmanager/manager/service.py index 6968d401b..841248f9e 100644 --- a/distributedcloud/dcmanager/manager/service.py +++ b/distributedcloud/dcmanager/manager/service.py @@ -149,13 +149,14 @@ class DCManagerService(service.Service): @request_context def update_subcloud(self, context, subcloud_id, management_state=None, - description=None, location=None): + description=None, location=None, group_id=None): # Updates a subcloud LOG.info("Handling update_subcloud request for: %s" % subcloud_id) subcloud = self.subcloud_manager.update_subcloud(context, subcloud_id, management_state, description, - location) + location, + group_id) # If a subcloud has been set to the managed state, trigger the # patching audit so it can update the sync status ASAP. if management_state == consts.MANAGEMENT_MANAGED: diff --git a/distributedcloud/dcmanager/manager/subcloud_manager.py b/distributedcloud/dcmanager/manager/subcloud_manager.py index 07e8bad62..73131fef1 100644 --- a/distributedcloud/dcmanager/manager/subcloud_manager.py +++ b/distributedcloud/dcmanager/manager/subcloud_manager.py @@ -130,6 +130,9 @@ class SubcloudManager(manager.Manager): # controller. software_version = SW_VERSION try: + # if group_id has been omitted from payload, use 'Default'. + group_id = payload.get('group_id', + consts.DEFAULT_SUBCLOUD_GROUP_ID) subcloud = db_api.subcloud_create( context, payload['name'], @@ -142,7 +145,8 @@ class SubcloudManager(manager.Manager): payload['management_end_address'], payload['systemcontroller_gateway_address'], consts.DEPLOY_STATE_NONE, - False) + False, + group_id) except Exception as e: LOG.exception(e) raise e @@ -660,8 +664,13 @@ class SubcloudManager(manager.Manager): "subcloud %s" % subcloud.name) LOG.exception(e) - def update_subcloud(self, context, subcloud_id, management_state=None, - description=None, location=None): + def update_subcloud(self, + context, + subcloud_id, + management_state=None, + description=None, + location=None, + group_id=None): """Update subcloud and notify orchestrators. :param context: request context object @@ -669,6 +678,7 @@ class SubcloudManager(manager.Manager): :param management_state: new management state :param description: new description :param location: new location + :param group_id: new subcloud group id """ LOG.info("Updating subcloud %s." % subcloud_id) @@ -699,10 +709,12 @@ class SubcloudManager(manager.Manager): LOG.error("Invalid management_state %s" % management_state) raise exceptions.InternalError() - subcloud = db_api.subcloud_update(context, subcloud_id, + subcloud = db_api.subcloud_update(context, + subcloud_id, management_state=management_state, description=description, - location=location) + location=location, + group_id=group_id) # Inform orchestrators that subcloud has been updated if management_state: diff --git a/distributedcloud/dcmanager/rpc/client.py b/distributedcloud/dcmanager/rpc/client.py index beebe02a3..d8b1b52fc 100644 --- a/distributedcloud/dcmanager/rpc/client.py +++ b/distributedcloud/dcmanager/rpc/client.py @@ -72,12 +72,13 @@ class ManagerClient(object): subcloud_id=subcloud_id)) def update_subcloud(self, ctxt, subcloud_id, management_state=None, - description=None, location=None): + description=None, location=None, group_id=None): return self.call(ctxt, self.make_msg('update_subcloud', subcloud_id=subcloud_id, management_state=management_state, description=description, - location=location)) + location=location, + group_id=group_id)) def update_subcloud_endpoint_status(self, ctxt, subcloud_name=None, endpoint_type=None, diff --git a/distributedcloud/dcmanager/tests/base.py b/distributedcloud/dcmanager/tests/base.py index 72aa42a2d..dae3868d3 100644 --- a/distributedcloud/dcmanager/tests/base.py +++ b/distributedcloud/dcmanager/tests/base.py @@ -40,18 +40,37 @@ from sqlalchemy.engine import Engine from sqlalchemy import event SUBCLOUD_SAMPLE_DATA_0 = [ - 6, "subcloud-4", "demo subcloud", "Ottawa-Lab-Aisle_3-Rack_C", - "20.01", "managed", "online", "fd01:3::0/64", "fd01:3::1", - "fd01:3::2", "fd01:3::f", "fd01:1::1", 0, "NULL", "NULL", - "2018-05-15 14:45:12.508708", "2018-05-24 10:48:18.090931", - "NULL", 0, "10.10.10.0/24", "10.10.10.1", "10.10.10.12", "testpass" + 6, # id + "subcloud-4", # name + "demo subcloud", # description + "Ottawa-Lab-Aisle_3-Rack_C", # location + "20.01", # software-version + "managed", # management-state + "online", # availability-status + "fd01:3::0/64", # management_subnet + "fd01:3::1", # management_gateway_address + "fd01:3::2", # management_start_address + "fd01:3::f", # management_end_address + "fd01:1::1", # systemcontroller_gateway_address + 0, # audit-fail-count + "NULL", # reserved-1 + "NULL", # reserved-2 + "2018-05-15 14:45:12.508708", # created-at + "2018-05-24 10:48:18.090931", # updated-at + "NULL", # deleted-at + 0, # deleted + "10.10.10.0/24", # external_oam_subnet + "10.10.10.1", # external_oam_gateway_address + "10.10.10.12", # external_oam_floating_address + "testpass", # sysadmin_password + 1 # group_id ] @event.listens_for(Engine, "connect") def set_sqlite_pragma(dbapi_connection, connection_record): cursor = dbapi_connection.cursor() - cursor.execute("PRAGMA foreign_keys=ON") + cursor.execute("PRAGMA foreign_keys=ON;") cursor.close() diff --git a/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_subcloud_group.py b/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_subcloud_group.py new file mode 100644 index 000000000..7a1d24e39 --- /dev/null +++ b/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_subcloud_group.py @@ -0,0 +1,539 @@ +# Copyright (c) 2017 Ericsson AB +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Copyright (c) 2020 Wind River Systems, Inc. +# +# The right to copy, distribute, modify, or otherwise make use +# of this software may be licensed only pursuant to the terms +# of an applicable Wind River license agreement. +# + +import mock +from six.moves import http_client + +from dcmanager.common import consts +from dcmanager.db.sqlalchemy import api as db_api +from dcmanager.rpc import client as rpc_client + +from dcmanager.tests.unit.api import test_root_controller as testroot +from dcmanager.tests.unit.api.v1.controllers.test_subclouds \ + import FAKE_SUBCLOUD_DATA +from dcmanager.tests import utils + +SAMPLE_SUBCLOUD_GROUP_NAME = 'GroupX' +SAMPLE_SUBCLOUD_GROUP_DESCRIPTION = 'A Group of mystery' +SAMPLE_SUBCLOUD_GROUP_UPDATE_APPLY_TYPE = consts.SUBCLOUD_APPLY_TYPE_SERIAL +SAMPLE_SUBCLOUD_GROUP_MAX_PARALLEL_SUBCLOUDS = 3 + + +# APIMixin can be moved to its own file, once the other +# unit tests are refactored to utilize it +class APIMixin(object): + + FAKE_TENANT = utils.UUID1 + + api_headers = { + 'X-Tenant-Id': FAKE_TENANT, + 'X_ROLE': 'admin', + 'X-Identity-Status': 'Confirmed' + } + + # subclasses should provide methods + # get_api_prefix + # get_result_key + + def setUp(self): + super(APIMixin, self).setUp() + + def get_api_headers(self): + return self.api_headers + + def get_single_url(self, uuid): + return '%s/%s' % (self.get_api_prefix(), uuid) + + def get_api_prefix(self): + raise NotImplementedError + + def get_result_key(self): + raise NotImplementedError + + def get_expected_api_fields(self): + raise NotImplementedError + + def get_omitted_api_fields(self): + raise NotImplementedError + + # base mixin subclass MUST override these methods if the api supports them + def _create_db_object(self, context): + raise NotImplementedError + + # base mixin subclass should provide this method for testing of POST + def get_post_object(self): + raise NotImplementedError + + def get_update_object(self): + raise NotImplementedError + + def assert_fields(self, api_object): + # Verify that expected attributes are returned + for field in self.get_expected_api_fields(): + self.assertIn(field, api_object) + + # Verify that hidden attributes are not returned + for field in self.get_omitted_api_fields(): + self.assertNotIn(field, api_object) + + +# +# --------------------- POST ----------------------------------- +# +# An API test will mixin only one of: PostMixin or PostRejectedMixin +# depending on whether or not the API supports a post operation or not +class PostMixin(object): + + @mock.patch.object(rpc_client, 'ManagerClient') + def test_create_success(self, mock_client): + # Test that a POST operation is supported by the API + ndict = self.get_post_object() + response = self.app.post_json(self.get_api_prefix(), + ndict, + headers=self.get_api_headers()) + self.assertEqual(response.content_type, 'application/json') + self.assertEqual(response.status_code, http_client.OK) + self.assert_fields(response.json) + + +class PostRejectedMixin(object): + # Test that a POST operation is blocked by the API + # API should return 400 BAD_REQUEST or FORBIDDEN 403 + @mock.patch.object(rpc_client, 'ManagerClient') + def test_create_not_allowed(self, mock_client): + ndict = self.get_post_object() + response = self.app.post_json(self.API_PREFIX, + ndict, + headers=self.get_api_headers(), + expect_errors=True) + self.assertEqual(response.status_code, http_client.FORBIDDEN) + self.assertTrue(response.json['error_message']) + self.assertIn("Operation not permitted.", + response.json['error_message']) + + +# ------ API GET mixin +class GetMixin(object): + + # Mixins can override initial_list_size if a table is not empty during + # DB creation and migration sync + initial_list_size = 0 + + # Performing a GET on this ID should fail. subclass mixins can override + invalid_id = '123' + + def validate_entry(self, result_item): + self.assert_fields(result_item) + + def validate_list(self, expected_length, results): + self.assertIn(self.get_result_key(), results) + result_list = results.get(self.get_result_key()) + self.assertEqual(expected_length, len(result_list)) + for result_item in result_list: + self.validate_entry(result_item) + + def validate_list_response(self, expected_length, response): + self.assertEqual(response.content_type, 'application/json') + self.assertEqual(response.status_code, http_client.OK) + + # validate the list length + self.validate_list(expected_length, response.json) + + @mock.patch.object(rpc_client, 'ManagerClient') + def test_initial_list_size(self, mock_client): + # Test that a GET operation for a list is supported by the API + response = self.app.get(self.get_api_prefix(), + headers=self.get_api_headers()) + # Validate the initial length + self.validate_list_response(self.initial_list_size, response) + + # Add an entry + context = utils.dummy_context() + self._create_db_object(context) + + response = self.app.get(self.get_api_prefix(), + headers=self.get_api_headers()) + self.validate_list_response(self.initial_list_size + 1, response) + + @mock.patch.object(rpc_client, 'ManagerClient') + def test_fail_get_single(self, mock_client): + # Test that a GET operation for an invalid ID returns the + # appropriate error results + response = self.app.get(self.get_single_url(self.invalid_id), + headers=self.get_api_headers(), + expect_errors=True) + # Failures will return text rather than json + self.assertEqual(response.content_type, 'text/plain') + self.assertEqual(response.status_code, http_client.NOT_FOUND) + + @mock.patch.object(rpc_client, 'ManagerClient') + def test_get_single(self, mock_client): + # create a group + context = utils.dummy_context() + group_name = 'TestGroup' + db_group = self._create_db_object(context, name=group_name) + + # Test that a GET operation for a valid ID works + response = self.app.get(self.get_single_url(db_group.id), + headers=self.get_api_headers()) + self.assertEqual(response.content_type, 'application/json') + self.assertEqual(response.status_code, http_client.OK) + self.validate_entry(response.json) + + +# ------ API Update Mixin +class UpdateMixin(object): + + def validate_updated_fields(self, sub_dict, full_obj): + for key, value in sub_dict.items(): + self.assertEqual(value, full_obj.get(key)) + + @mock.patch.object(rpc_client, 'ManagerClient') + def test_update_success(self, mock_client): + context = utils.dummy_context() + single_obj = self._create_db_object(context) + update_data = self.get_update_object() + response = self.app.patch_json(self.get_single_url(single_obj.id), + headers=self.get_api_headers(), + params=update_data) + self.assertEqual(response.content_type, 'application/json') + self.assertEqual(response.status_code, http_client.OK) + self.validate_updated_fields(update_data, response.json) + + @mock.patch.object(rpc_client, 'ManagerClient') + def test_update_empty_changeset(self, mock_client): + context = utils.dummy_context() + single_obj = self._create_db_object(context) + update_data = {} + response = self.app.patch_json(self.get_single_url(single_obj.id), + headers=self.get_api_headers(), + params=update_data, + expect_errors=True) + # Failures will return text rather than json + self.assertEqual(response.content_type, 'text/plain') + self.assertEqual(response.status_code, http_client.BAD_REQUEST) + + +# ------ API Delete Mixin +class DeleteMixin(object): + + @mock.patch.object(rpc_client, 'ManagerClient') + def test_delete_success(self, mock_client): + context = utils.dummy_context() + single_obj = self._create_db_object(context) + response = self.app.delete(self.get_single_url(single_obj.id), + headers=self.get_api_headers()) + self.assertEqual(response.content_type, 'application/json') + self.assertEqual(response.status_code, http_client.OK) + + @mock.patch.object(rpc_client, 'ManagerClient') + def test_double_delete(self, mock_client): + context = utils.dummy_context() + single_obj = self._create_db_object(context) + response = self.app.delete(self.get_single_url(single_obj.id), + headers=self.get_api_headers()) + self.assertEqual(response.content_type, 'application/json') + self.assertEqual(response.status_code, http_client.OK) + # delete the same object a second time. this should fail (NOT_FOUND) + response = self.app.delete(self.get_single_url(single_obj.id), + headers=self.get_api_headers(), + expect_errors=True) + self.assertEqual(response.content_type, 'text/plain') + self.assertEqual(response.status_code, http_client.NOT_FOUND) + + +class SubcloudGroupAPIMixin(APIMixin): + + API_PREFIX = '/v1.0/subcloud-groups' + RESULT_KEY = 'subcloud_groups' + EXPECTED_FIELDS = ['id', + 'name', + 'description', + 'max_parallel_subclouds', + 'update_apply_type', + 'created-at', + 'updated-at'] + + def setUp(self): + super(SubcloudGroupAPIMixin, self).setUp() + self.fake_rpc_client.some_method = mock.MagicMock() + + def _get_test_subcloud_group_dict(self, **kw): + # id should not be part of the structure + group = { + 'name': kw.get('name', SAMPLE_SUBCLOUD_GROUP_NAME), + 'description': kw.get('description', + SAMPLE_SUBCLOUD_GROUP_DESCRIPTION), + 'update_apply_type': kw.get( + 'update_apply_type', + SAMPLE_SUBCLOUD_GROUP_UPDATE_APPLY_TYPE), + 'max_parallel_subclouds': kw.get( + 'max_parallel_subclouds', + SAMPLE_SUBCLOUD_GROUP_MAX_PARALLEL_SUBCLOUDS) + } + return group + + def _post_get_test_subcloud_group(self, **kw): + post_body = self._get_test_subcloud_group_dict(**kw) + return post_body + + # The following methods are required for subclasses of APIMixin + + def get_api_prefix(self): + return self.API_PREFIX + + def get_result_key(self): + return self.RESULT_KEY + + def get_expected_api_fields(self): + return self.EXPECTED_FIELDS + + def get_omitted_api_fields(self): + return [] + + def _create_db_object(self, context, **kw): + creation_fields = self._get_test_subcloud_group_dict(**kw) + return db_api.subcloud_group_create(context, **creation_fields) + + def get_post_object(self): + return self._post_get_test_subcloud_group() + + def get_update_object(self): + update_object = { + 'description': 'Updated description' + } + return update_object + + +# Combine Subcloud Group API with mixins to test post, get, update and delete +class TestSubcloudGroupPost(testroot.DCManagerApiTest, + SubcloudGroupAPIMixin, + PostMixin): + def setUp(self): + super(TestSubcloudGroupPost, self).setUp() + + def verify_post_failure(self, response): + # Failures will return text rather than json + self.assertEqual(response.content_type, 'text/plain') + self.assertEqual(response.status_code, http_client.BAD_REQUEST) + + @mock.patch.object(rpc_client, 'ManagerClient') + def test_create_with_numerical_name_fails(self, mock_client): + # A numerical name is not permitted. otherwise the 'get' operations + # which support getting by either name or ID could become confused + # if a name for one group was the same as an ID for another. + ndict = self.get_post_object() + ndict['name'] = '123' + response = self.app.post_json(self.get_api_prefix(), + ndict, + headers=self.get_api_headers(), + expect_errors=True) + self.verify_post_failure(response) + + @mock.patch.object(rpc_client, 'ManagerClient') + def test_create_with_blank_name_fails(self, mock_client): + # An empty name is not permitted + ndict = self.get_post_object() + ndict['name'] = '' + response = self.app.post_json(self.get_api_prefix(), + ndict, + headers=self.get_api_headers(), + expect_errors=True) + self.verify_post_failure(response) + + @mock.patch.object(rpc_client, 'ManagerClient') + def test_create_with_default_name_fails(self, mock_client): + # A name that is the same as the 'Default' group is not permitted. + # This would be a duplicate, and names must be unique. + ndict = self.get_post_object() + ndict['name'] = 'Default' + response = self.app.post_json(self.get_api_prefix(), + ndict, + headers=self.get_api_headers(), + expect_errors=True) + self.verify_post_failure(response) + + @mock.patch.object(rpc_client, 'ManagerClient') + def test_create_with_empty_description_fails(self, mock_client): + # An empty description is considered invalid + ndict = self.get_post_object() + ndict['description'] = '' + response = self.app.post_json(self.get_api_prefix(), + ndict, + headers=self.get_api_headers(), + expect_errors=True) + self.verify_post_failure(response) + + @mock.patch.object(rpc_client, 'ManagerClient') + def test_create_with_bad_apply_type(self, mock_client): + # update_apply_type must be either 'serial' or 'parallel' + ndict = self.get_post_object() + ndict['update_apply_type'] = 'something_invalid' + response = self.app.post_json(self.get_api_prefix(), + ndict, + headers=self.get_api_headers(), + expect_errors=True) + self.verify_post_failure(response) + + @mock.patch.object(rpc_client, 'ManagerClient') + def test_create_with_bad_max_parallel_subclouds(self, mock_client): + # max_parallel_subclouds must be an integer between 1 and 100 + ndict = self.get_post_object() + # All the entries in bad_values should be considered invalid + bad_values = [0, 101, -1, 'abc'] + for bad_value in bad_values: + ndict['max_parallel_subclouds'] = bad_value + response = self.app.post_json(self.get_api_prefix(), + ndict, + headers=self.get_api_headers(), + expect_errors=True) + self.verify_post_failure(response) + + +class TestSubcloudGroupGet(testroot.DCManagerApiTest, + SubcloudGroupAPIMixin, + GetMixin): + + def setUp(self): + super(TestSubcloudGroupGet, self).setUp() + # Override initial_list_size. Default group is setup during db sync + self.initial_list_size = 1 + + @mock.patch.object(rpc_client, 'ManagerClient') + def test_get_single_by_name(self, mock_client): + # create a group + context = utils.dummy_context() + # todo(abailey) make this a generic method + group_name = 'TestGroup' + self._create_db_object(context, name=group_name) + + # Test that a GET operation for a valid ID works + response = self.app.get(self.get_single_url(group_name), + headers=self.get_api_headers()) + self.assertEqual(response.content_type, 'application/json') + self.assertEqual(response.status_code, http_client.OK) + self.validate_entry(response.json) + + @mock.patch.object(rpc_client, 'ManagerClient') + def test_list_subclouds_empty(self, mock_client): + # API GET on: subcloud-groups//subclouds + uuid = 1 # The Default Subcloud Group is always ID=1 + url = '%s/%s/subclouds' % (self.get_api_prefix(), uuid) + response = self.app.get(url, + headers=self.get_api_headers()) + # This API returns 'subclouds' rather than 'subcloud-groups' + self.assertIn('subclouds', response.json) + # no subclouds exist yet, so this length should be zero + result_list = response.json.get('subclouds') + self.assertEqual(0, len(result_list)) + + def _create_subcloud_db_object(self, context): + creation_fields = { + 'name': FAKE_SUBCLOUD_DATA.get('name'), + 'description': FAKE_SUBCLOUD_DATA.get('description'), + 'location': FAKE_SUBCLOUD_DATA.get('location'), + 'software_version': FAKE_SUBCLOUD_DATA.get('software_version'), + 'management_subnet': FAKE_SUBCLOUD_DATA.get('management_subnet'), + 'management_gateway_ip': + FAKE_SUBCLOUD_DATA.get('management_gateway_ip'), + 'management_start_ip': + FAKE_SUBCLOUD_DATA.get('management_start_ip'), + 'management_end_ip': FAKE_SUBCLOUD_DATA.get('management_end_ip'), + 'systemcontroller_gateway_ip': + FAKE_SUBCLOUD_DATA.get('systemcontroller_gateway_ip'), + 'deploy_status': FAKE_SUBCLOUD_DATA.get('deploy_status'), + 'openstack_installed': + FAKE_SUBCLOUD_DATA.get('openstack_installed'), + 'group_id': FAKE_SUBCLOUD_DATA.get('group_id', 1) + } + return db_api.subcloud_create(context, **creation_fields) + + @mock.patch.object(rpc_client, 'ManagerClient') + def test_list_subclouds_populated(self, mock_client): + # subclouds are to Default group by default (unless specified) + context = utils.dummy_context() + self._create_subcloud_db_object(context) + + # API GET on: subcloud-groups//subclouds + uuid = 1 # The Default Subcloud Group is always ID=1 + url = '%s/%s/subclouds' % (self.get_api_prefix(), uuid) + response = self.app.get(url, + headers=self.get_api_headers()) + # This API returns 'subclouds' rather than 'subcloud-groups' + self.assertIn('subclouds', response.json) + # the subcloud created earlier will have been queried + result_list = response.json.get('subclouds') + self.assertEqual(1, len(result_list)) + + +class TestSubcloudGroupUpdate(testroot.DCManagerApiTest, + SubcloudGroupAPIMixin, + UpdateMixin): + def setUp(self): + super(TestSubcloudGroupUpdate, self).setUp() + + @mock.patch.object(rpc_client, 'ManagerClient') + def test_update_invalid_apply_type(self, mock_client): + context = utils.dummy_context() + single_obj = self._create_db_object(context) + update_data = { + 'update_apply_type': 'something_bad' + } + response = self.app.patch_json(self.get_single_url(single_obj.id), + headers=self.get_api_headers(), + params=update_data, + expect_errors=True) + # Failures will return text rather than json + self.assertEqual(response.content_type, 'text/plain') + self.assertEqual(response.status_code, http_client.BAD_REQUEST) + + @mock.patch.object(rpc_client, 'ManagerClient') + def test_update_invalid_max_parallel(self, mock_client): + context = utils.dummy_context() + single_obj = self._create_db_object(context) + update_data = { + 'max_parallel_subclouds': -1 + } + response = self.app.patch_json(self.get_single_url(single_obj.id), + headers=self.get_api_headers(), + params=update_data, + expect_errors=True) + # Failures will return text rather than json + self.assertEqual(response.content_type, 'text/plain') + self.assertEqual(response.status_code, http_client.BAD_REQUEST) + + +class TestSubcloudGroupDelete(testroot.DCManagerApiTest, + SubcloudGroupAPIMixin, + DeleteMixin): + def setUp(self): + super(TestSubcloudGroupDelete, self).setUp() + + @mock.patch.object(rpc_client, 'ManagerClient') + def test_delete_default_fails(self, mock_client): + default_zone_id = 1 + response = self.app.delete(self.get_single_url(default_zone_id), + headers=self.get_api_headers(), + expect_errors=True) + # Failures will return text rather than json + self.assertEqual(response.content_type, 'text/plain') + self.assertEqual(response.status_code, http_client.BAD_REQUEST) diff --git a/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_subclouds.py b/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_subclouds.py index 6bebedbd7..b138ac3a8 100644 --- a/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_subclouds.py +++ b/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_subclouds.py @@ -499,7 +499,8 @@ class TestSubclouds(testroot.DCManagerApiTest): mock.ANY, management_state=consts.MANAGEMENT_UNMANAGED, description=None, - location=None) + location=None, + group_id=None) self.assertEqual(response.status_int, 200) @mock.patch.object(rpc_client, 'ManagerClient') diff --git a/distributedcloud/dcmanager/tests/unit/db/test_subcloud_db_api.py b/distributedcloud/dcmanager/tests/unit/db/test_subcloud_db_api.py index 884cee5b7..cec930912 100644 --- a/distributedcloud/dcmanager/tests/unit/db/test_subcloud_db_api.py +++ b/distributedcloud/dcmanager/tests/unit/db/test_subcloud_db_api.py @@ -83,6 +83,7 @@ class DBAPISubcloudTest(base.DCManagerTestCase): 'systemcontroller_gateway_ip': "192.168.204.101", 'deploy_status': "not-deployed", 'openstack_installed': False, + 'group_id': 1, } values.update(kwargs) return db_api.subcloud_create(ctxt, **values) @@ -102,6 +103,7 @@ class DBAPISubcloudTest(base.DCManagerTestCase): 'systemcontroller_gateway_address'], 'deploy_status': "not-deployed", 'openstack_installed': False, + 'group_id': 1, } return db_api.subcloud_create(ctxt, **values) diff --git a/distributedcloud/dcmanager/tests/unit/manager/test_service.py b/distributedcloud/dcmanager/tests/unit/manager/test_service.py index 5213afdbb..8d247f91b 100644 --- a/distributedcloud/dcmanager/tests/unit/manager/test_service.py +++ b/distributedcloud/dcmanager/tests/unit/manager/test_service.py @@ -131,8 +131,9 @@ class TestDCManagerService(base.DCManagerTestCase): self.service_obj.update_subcloud( self.context, subcloud_id=1, management_state='testmgmtstatus') mock_subcloud_manager().update_subcloud.\ - assert_called_once_with(self.context, mock.ANY, mock.ANY, mock.ANY, - mock.ANY) + assert_called_once_with(self.context, mock.ANY, + mock.ANY, mock.ANY, + mock.ANY, mock.ANY) @mock.patch.object(service, 'SwUpdateManager') @mock.patch.object(service, 'SubcloudManager') diff --git a/distributedcloud/dcmanager/tests/unit/manager/test_subcloud_audit_manager.py b/distributedcloud/dcmanager/tests/unit/manager/test_subcloud_audit_manager.py index b0be0cc12..287e5ae2c 100644 --- a/distributedcloud/dcmanager/tests/unit/manager/test_subcloud_audit_manager.py +++ b/distributedcloud/dcmanager/tests/unit/manager/test_subcloud_audit_manager.py @@ -244,6 +244,7 @@ class TestAuditManager(base.DCManagerTestCase): 'systemcontroller_gateway_ip': "192.168.204.101", 'deploy_status': "not-deployed", 'openstack_installed': False, + 'group_id': 1, } values.update(kwargs) return db_api.subcloud_create(ctxt, **values) diff --git a/distributedcloud/dcmanager/tests/unit/manager/test_subcloud_manager.py b/distributedcloud/dcmanager/tests/unit/manager/test_subcloud_manager.py index 9c1869226..366803d37 100644 --- a/distributedcloud/dcmanager/tests/unit/manager/test_subcloud_manager.py +++ b/distributedcloud/dcmanager/tests/unit/manager/test_subcloud_manager.py @@ -34,13 +34,14 @@ from dcmanager.manager import subcloud_manager from dcmanager.tests import base from dcmanager.tests import utils from dcorch.common import consts as dcorch_consts -from dcorch.rpc import client as dcorch_rpc_client class FakeDCOrchAPI(object): def __init__(self): self.update_subcloud_states = mock.MagicMock() self.add_subcloud_sync_endpoint_type = mock.MagicMock() + self.del_subcloud = mock.MagicMock() + self.add_subcloud = mock.MagicMock() class FakeService(object): @@ -147,6 +148,7 @@ class TestSubcloudManager(base.DCManagerTestCase): "systemcontroller_gateway_ip": "192.168.204.101", 'deploy_status': "not-deployed", 'openstack_installed': False, + 'group_id': 1, } values.update(kwargs) return db_api.subcloud_create(ctxt, **values) @@ -160,8 +162,6 @@ class TestSubcloudManager(base.DCManagerTestCase): @mock.patch.object(subcloud_manager.SubcloudManager, '_delete_subcloud_inventory') - @mock.patch.object(dcorch_rpc_client, 'EngineClient') - @mock.patch.object(subcloud_manager, 'context') @mock.patch.object(subcloud_manager, 'KeystoneClient') @mock.patch.object(subcloud_manager, 'db_api') @mock.patch.object(subcloud_manager, 'SysinvClient') @@ -179,13 +179,11 @@ class TestSubcloudManager(base.DCManagerTestCase): mock_write_subcloud_ansible_config, mock_create_subcloud_inventory, mock_create_addn_hosts, mock_sysinv_client, - mock_db_api, mock_keystone_client, mock_context, - mock_dcorch_rpc_client, + mock_db_api, mock_keystone_client, mock_delete_subcloud_inventory): values = utils.create_subcloud_dict(base.SUBCLOUD_SAMPLE_DATA_0) controllers = FAKE_CONTROLLERS services = FAKE_SERVICES - mock_context.get_admin_context.return_value = self.ctx mock_db_api.subcloud_get_by_name.side_effect = \ exceptions.SubcloudNameNotFound() @@ -198,15 +196,13 @@ class TestSubcloudManager(base.DCManagerTestCase): mock_db_api.subcloud_create.assert_called_once() mock_db_api.subcloud_status_create.assert_called() mock_sysinv_client().create_route.assert_called() - mock_dcorch_rpc_client().add_subcloud.assert_called_once() + self.fake_dcorch_api.add_subcloud.assert_called_once() mock_create_addn_hosts.assert_called_once() mock_create_subcloud_inventory.assert_called_once() mock_write_subcloud_ansible_config.assert_called_once() mock_keyring.get_password.assert_called() mock_thread_start.assert_called_once() - @mock.patch.object(dcorch_rpc_client, 'EngineClient') - @mock.patch.object(subcloud_manager, 'context') @mock.patch.object(subcloud_manager, 'db_api') @mock.patch.object(subcloud_manager, 'SysinvClient') @mock.patch.object(subcloud_manager, 'KeystoneClient') @@ -215,11 +211,8 @@ class TestSubcloudManager(base.DCManagerTestCase): def test_delete_subcloud(self, mock_create_addn_hosts, mock_keystone_client, mock_sysinv_client, - mock_db_api, - mock_context, - mock_dcorch_rpc_client): + mock_db_api): controllers = FAKE_CONTROLLERS - mock_context.get_admin_context.return_value = self.ctx data = utils.create_subcloud_dict(base.SUBCLOUD_SAMPLE_DATA_0) fake_subcloud = Subcloud(data, False) mock_db_api.subcloud_get.return_value = fake_subcloud @@ -231,20 +224,15 @@ class TestSubcloudManager(base.DCManagerTestCase): mock_db_api.subcloud_destroy.assert_called_once() mock_create_addn_hosts.assert_called_once() - @mock.patch.object(dcorch_rpc_client, 'EngineClient') - @mock.patch.object(subcloud_manager, 'context') - @mock.patch.object(subcloud_manager, 'KeystoneClient') @mock.patch.object(subcloud_manager, 'db_api') - def test_update_subcloud(self, mock_db_api, - mock_endpoint, mock_context, - mock_dcorch_rpc_client): - mock_context.get_admin_context.return_value = self.ctx + def test_update_subcloud(self, mock_db_api): data = utils.create_subcloud_dict(base.SUBCLOUD_SAMPLE_DATA_0) subcloud_result = Subcloud(data, True) mock_db_api.subcloud_get.return_value = subcloud_result mock_db_api.subcloud_update.return_value = subcloud_result sm = subcloud_manager.SubcloudManager() - sm.update_subcloud(self.ctx, data['id'], + sm.update_subcloud(self.ctx, + data['id'], management_state=consts.MANAGEMENT_MANAGED, description="subcloud new description", location="subcloud new location") @@ -253,7 +241,29 @@ class TestSubcloudManager(base.DCManagerTestCase): data['id'], management_state=consts.MANAGEMENT_MANAGED, description="subcloud new description", - location="subcloud new location") + location="subcloud new location", + group_id=None) + + @mock.patch.object(subcloud_manager, 'db_api') + def test_update_subcloud_group_id(self, mock_db_api): + data = utils.create_subcloud_dict(base.SUBCLOUD_SAMPLE_DATA_0) + subcloud_result = Subcloud(data, True) + mock_db_api.subcloud_get.return_value = subcloud_result + mock_db_api.subcloud_update.return_value = subcloud_result + sm = subcloud_manager.SubcloudManager() + sm.update_subcloud(self.ctx, + data['id'], + management_state=consts.MANAGEMENT_MANAGED, + description="subcloud new description", + location="subcloud new location", + group_id=2) + mock_db_api.subcloud_update.assert_called_once_with( + mock.ANY, + data['id'], + management_state=consts.MANAGEMENT_MANAGED, + description="subcloud new description", + location="subcloud new location", + group_id=2) def test_update_subcloud_endpoint_status(self): # create a subcloud diff --git a/distributedcloud/dcmanager/tests/utils.py b/distributedcloud/dcmanager/tests/utils.py index 32f8e4f63..b1d127fc0 100644 --- a/distributedcloud/dcmanager/tests/utils.py +++ b/distributedcloud/dcmanager/tests/utils.py @@ -117,4 +117,5 @@ def create_subcloud_dict(data_list): 'external_oam_subnet': data_list[19], 'external_oam_gateway_address': data_list[20], 'external_oam_floating_address': data_list[21], - 'sysadmin_password': data_list[22]} + 'sysadmin_password': data_list[22], + 'group_id': data_list[23]} diff --git a/distributedcloud/tox.ini b/distributedcloud/tox.ini index 506421581..e63c93083 100644 --- a/distributedcloud/tox.ini +++ b/distributedcloud/tox.ini @@ -12,7 +12,6 @@ fmclient_src_dir = {[dc]stx_fault_dir}/python-fmclient/fmclient fm_api_src_dir = {[dc]stx_fault_dir}/fm-api sysinv_src_dir = ../../config/sysinv/sysinv/sysinv tsconfig_src_dir = ../../config/tsconfig/tsconfig -controllerconfig_src_dir = ../../config/controllerconfig/controllerconfig cgtsclient_src_dir = ../../config/sysinv/cgts-client/cgts-client cgcs_patch_src_dir = ../../update/cgcs-patch/cgcs-patch @@ -47,7 +46,6 @@ deps = -r{toxinidir}/test-requirements.txt -e{[dc]tsconfig_src_dir} -e{[dc]fmclient_src_dir} -e{[dc]fm_api_src_dir} - -e{[dc]controllerconfig_src_dir} -e{[dc]cgtsclient_src_dir} setenv = CURRENT_CFG_FILE={toxinidir}/.current.cfg @@ -87,7 +85,6 @@ deps = -r{toxinidir}/test-requirements.txt -e../{[dc]tsconfig_src_dir} -e../{[dc]fmclient_src_dir} -e../{[dc]fm_api_src_dir} - -e../{[dc]controllerconfig_src_dir} -e../{[dc]cgtsclient_src_dir} setenv = CURRENT_CFG_FILE={toxinidir}/.current.cfg