diff --git a/api-ref/source/api-ref-dcmanager-v1.rst b/api-ref/source/api-ref-dcmanager-v1.rst index aa208a8f8..4eec71ce8 100644 --- a/api-ref/source/api-ref-dcmanager-v1.rst +++ b/api-ref/source/api-ref-dcmanager-v1.rst @@ -93,6 +93,7 @@ internalServerError (500), serviceUnavailable (503) "compute_sync_status (Optional)", "plain", "xsd:string", "The compute sync status of the subcloud." "network_sync_status (Optional)", "plain", "xsd:string", "The network sync status of the subcloud." "patching_sync_status (Optional)", "plain", "xsd:string", "The patching sync status of the subcloud." + "group_id (Optional)", "plain", "xsd:int", "The unique identifier for the subcloud group for this subcloud." :: @@ -132,6 +133,7 @@ internalServerError (500), serviceUnavailable (503) "endpoint_type": "patching" }, "created-at": u"2018-02-25 19:06:35.208505", + "group_id": 1, "management-gateway-ip": u"192.168.204.1", "management-end-ip": u"192.168.204.100", "id": 1, @@ -171,6 +173,7 @@ internalServerError (500), serviceUnavailable (503) "endpoint_type": "patching" }, "created-at": "2018-02-25 19:06:35.208505", + "group_id": 1, "management-gateway-ip": "192.168.205.1", "management-end-ip": "192.168.205.100", "id": 2, @@ -210,6 +213,7 @@ serviceUnavailable (503) "management-start-ip", "plain", "xsd:string", "Start of management IP address range for subcloud." "management-end-ip", "plain", "xsd:string", "End of management IP address range for subcloud." "systemcontroller-gateway-ip", "plain", "xsd:string", "Systemcontroller gateway IP Address." + "group_id", "plain", "xsd:int", "Id of the subcloud group. Defaults to 1" **Response parameters** @@ -227,6 +231,7 @@ serviceUnavailable (503) "management-start-ip (Optional)", "plain", "xsd:string", "Start of management IP address range for subcloud." "management-end-ip (Optional)", "plain", "xsd:string", "End of management IP address range for subcloud." "systemcontroller-gateway-ip (Optional)", "plain", "xsd:string", "Systemcontroller gateway IP Address." + "group_id (Optional)", "plain", "xsd:int", "Id of the subcloud group." :: @@ -238,6 +243,7 @@ serviceUnavailable (503) "management-subnet": "192.168.205.0/24", "management-gateway-ip": "192.168.205.1", "management-end-ip": "192.168.205.160", + "group_id": 1, "description": "new subcloud" } @@ -253,6 +259,7 @@ serviceUnavailable (503) "availability-status": "offline", "systemcontroller-gateway-ip": "192.168.204.102", "location": None, + "group_id": 1, "management-subnet": "192.168.205.0/24", "management-gateway-ip": "192.168.205.1", "management-end-ip": "192.168.205.160", @@ -306,6 +313,7 @@ internalServerError (500), serviceUnavailable (503) "compute_sync_status (Optional)", "plain", "xsd:string", "The compute sync status of the subcloud." "network_sync_status (Optional)", "plain", "xsd:string", "The network sync status of the subcloud." "patching_sync_status (Optional)", "plain", "xsd:string", "The patching sync status of the subcloud." + "group_id (Optional)", "plain", "xsd:int", "Id of the subcloud group." :: @@ -344,6 +352,7 @@ internalServerError (500), serviceUnavailable (503) ], "management-gateway-ip": "192.168.204.1", "management-end-ip": "192.168.204.100", + "group_id": 1, "id": 1, "name": "subcloud6" } @@ -397,6 +406,7 @@ internalServerError (500), serviceUnavailable (503) "network_sync_status (Optional)", "plain", "xsd:string", "The network sync status of the subcloud." "patching_sync_status (Optional)", "plain", "xsd:string", "The patching sync status of the subcloud." "oam_floating_ip (Optional)", "plain", "xsd:string", "OAM Floating IP of the subcloud." + "group_id (Optional)", "plain", "xsd:int", "Id of the subcloud group." :: @@ -435,6 +445,7 @@ internalServerError (500), serviceUnavailable (503) ], "management-gateway-ip": "192.168.204.1", "management-end-ip": "192.168.204.100", + "group_id": 1, "id": 1, "name": "subcloud6", "oam_floating_ip" "10.10.10.12" @@ -476,6 +487,7 @@ serviceUnavailable (503) "description (Optional)", "plain", "xsd:string", "The description of the subcloud." "location (Optional)", "plain", "xsd:string", "The location of the subcloud." "management-state (Optional)", "plain", "xsd:string", "The management-state of the subcloud, ``managed`` or ``unmanaged``. The subcloud must be online before this can be modified to managed." + "group_id (Optional)", "plain", "xsd:int", "Id of the subcloud group. The group must exist." **Response parameters** @@ -493,6 +505,7 @@ serviceUnavailable (503) "management-start-ip (Optional)", "plain", "xsd:string", "Start of management IP address range for subcloud." "management-end-ip (Optional)", "plain", "xsd:string", "End of management IP address range for subcloud." "systemcontroller-gateway-ip (Optional)", "plain", "xsd:string", "Systemcontroller gateway IP Address." + "group_id (Optional)", "plain", "xsd:int", "Id of the subcloud group." :: @@ -500,6 +513,7 @@ serviceUnavailable (503) "description": "new description", "location": "new location", "management-state": "managed" + "group_id": 2, } :: @@ -517,6 +531,7 @@ serviceUnavailable (503) "management-subnet": "192.168.204.0/24", "management-gateway-ip": "192.168.204.1", "management-end-ip": "192.168.204.100", + "group_id": 2, "id": 1, "name": "subcloud6" } @@ -541,6 +556,341 @@ Deletes a specific subcloud This operation does not accept a request body. +---------------- +Subcloud Groups +---------------- + +Subcloud Groups are a logical grouping managed by a central System Controller. +Subclouds in a group can be updated in parallel when applying patches or +software upgrades. + +************************** +Lists all subcloud groups +************************** + +.. rest_method:: GET /v1.0/subcloud-groups + +**Normal response codes** + +200 + +**Error response codes** + +itemNotFound (404), badRequest (400), unauthorized (401), forbidden +(403), badMethod (405), HTTPUnprocessableEntity (422), +internalServerError (500), serviceUnavailable (503) + +**Response parameters** + +.. csv-table:: + :header: "Parameter", "Style", "Type", "Description" + :widths: 20, 20, 20, 60 + + "subcloud_groups (Optional)", "plain", "xsd:list", "The list of subcloud groups." + "id (Optional)", "plain", "xsd:int", "The unique identifier for this object." + "name (Optional)", "plain", "xsd:string", "The unique name for the subcloud group." + "description (Optional)", "plain", "xsd:string", "The description of the subcloud group." + "update_apply_type (Optional)", "plain", "xsd:string", "The method for applying an update. ```serial``` or ```parallel```." + "max_parallel_subclouds (Optional)", "plain", "xsd:int", "The maximum number of subclouds to update in parallel." + "created_at (Optional)", "plain", "xsd:dateTime", "The time when the object was created." + "updated_at (Optional)", "plain", "xsd:dateTime", "The time when the object was last updated." + +:: + + { + "subcloud_groups": [ + { + "update_apply_type": "parallel", + "description": "Default Subcloud Group", + "updated-at": null, + "created-at": null, + "max_parallel_subclouds": 2, + "id": 1, + "name": "Default" + }, + ] + } + +This operation does not accept a request body. + +************************* +Creates a subcloud group +************************* + +.. rest_method:: POST /v1.0/subcloud-groups + +**Normal response codes** + +200 + +**Error response codes** + +badRequest (400), unauthorized (401), forbidden (403), badMethod (405), +HTTPUnprocessableEntity (422), internalServerError (500), +serviceUnavailable (503) + +**Request parameters** + +.. csv-table:: + :header: "Parameter", "Style", "Type", "Description" + :widths: 20, 20, 20, 60 + + "name (Optional)", "plain", "xsd:string", "The name for the subcloud group. Must be unique." + "description (Optional)", "plain", "xsd:string", "The description of the subcloud group." + "update_apply_type (Optional)", "plain", "xsd:string", "The method for applying an update. Must be ```serial``` or ```parallel```." + "max_parallel_subclouds (Optional)", "plain", "xsd:int", "The maximum number of subclouds to update in parallel. Must be greater than 0." + +**Response parameters** + +.. csv-table:: + :header: "Parameter", "Style", "Type", "Description" + :widths: 20, 20, 20, 60 + + "id (Optional)", "plain", "xsd:int", "The unique identifier for this object." + "name (Optional)", "plain", "xsd:string", "The unique name for the subcloud group." + "description (Optional)", "plain", "xsd:string", "The description of the subcloud group." + "update_apply_type (Optional)", "plain", "xsd:string", "The method for applying an update. ```serial``` or ```parallel```." + "max_parallel_subclouds (Optional)", "plain", "xsd:int", "The maximum number of subclouds to update in parallel." + +:: + + { + "name": "GroupX", + "description": "A new group", + "update_apply_type": "parallel", + "max_parallel_subclouds": 3 + } + +:: + + { + "id": 2, + "name": "GroupX", + "description": "A new group", + "update_apply_type": "parallel", + "max_parallel_subclouds": "3", + "updated-at": null, + "created-at": "2020-04-08 15:15:10.750592", + } + +****************************************************** +Shows information about a specific subcloud group +****************************************************** + +.. rest_method:: GET /v1.0/subcloud-groups/​{subcloud-group}​ + +**Normal response codes** + +200 + +**Error response codes** + +itemNotFound (404), badRequest (400), unauthorized (401), forbidden +(403), badMethod (405), HTTPUnprocessableEntity (422), +internalServerError (500), serviceUnavailable (503) + +**Request parameters** + +.. csv-table:: + :header: "Parameter", "Style", "Type", "Description" + :widths: 20, 20, 20, 60 + + "subcloud-group", "URI", "xsd:string", "The subcloud group reference, name or id." + +**Response parameters** + +.. csv-table:: + :header: "Parameter", "Style", "Type", "Description" + :widths: 20, 20, 20, 60 + + "id (Optional)", "plain", "xsd:int", "The unique identifier for this object." + "name (Optional)", "plain", "xsd:string", "The name provisioned for the subcloud group." + "description (Optional)", "plain", "xsd:string", "The description for the subcloud group." + "max_parallel_subclouds (Optional)", "plain", "xsd:int", "The maximum number of subclouds to update in parallel." + "update_apply_type (Optional)", "plain", "xsd:string", "The update apply type for the subcloud group: ```serial``` or ```parallel```." + "created_at (Optional)", "plain", "xsd:dateTime", "The time when the object was created." + "updated_at (Optional)", "plain", "xsd:dateTime", "The time when the object was last updated." + +:: + + { + "id": 2, + "name": "GroupX", + "description": "A new group", + "max_parallel_subclouds": 3, + "update_apply_type": "parallel", + "created-at": "2020-04-08 15:15:10.750592", + "updated-at": null + } + +This operation does not accept a request body. + +****************************************************** +Shows subclouds that are part of a subcloud group +****************************************************** + +.. rest_method:: GET /v1.0/subcloud-groups/​{subcloud-group}​/subclouds + +**Normal response codes** + +200 + +**Error response codes** + +itemNotFound (404), badRequest (400), unauthorized (401), forbidden +(403), badMethod (405), HTTPUnprocessableEntity (422), +internalServerError (500), serviceUnavailable (503) + +**Request parameters** + +.. csv-table:: + :header: "Parameter", "Style", "Type", "Description" + :widths: 20, 20, 20, 60 + + "subcloud-group", "URI", "xsd:string", "The subcloud group reference, name or id." + +**Response parameters** + +.. csv-table:: + :header: "Parameter", "Style", "Type", "Description" + :widths: 20, 20, 20, 60 + + "subclouds (Optional)", "plain", "xsd:list", "The list of subclouds." + "id (Optional)", "plain", "xsd:int", "The unique identifier for a subcloud." + "group_id (Optional)", "plain", "xsd:int", "The unique identifier for the subcloud group." + "created_at (Optional)", "plain", "xsd:dateTime", "The time when the object was created." + "updated_at (Optional)", "plain", "xsd:dateTime", "The time when the object was last updated." + "name (Optional)", "plain", "xsd:string", "The name provisioned for the subcloud." + "management-state (Optional)", "plain", "xsd:string", "Management state of the subcloud." + "management-start-ip (Optional)", "plain", "xsd:string", "Start of management IP address range for subcloud." + "software-version (Optional)", "plain", "xsd:string", "Software version for subcloud." + "availability-status (Optional)", "plain", "xsd:string", "Availability status of the subcloud." + "systemcontroller-gateway-ip (Optional)", "plain", "xsd:string", "Systemcontroller gateway IP Address." + "location (Optional)", "plain", "xsd:string", "The location provisioned for the subcloud." + "openstack-installed (Optional)", "plain", "xsd:boolean", "Whether openstack is installed on the subcloud." + "management-subnet (Optional)", "plain", "xsd:string", "Management subnet for subcloud in CIDR format." + "management-gateway-ip (Optional)", "plain", "xsd:string", "Management gateway IP for subcloud." + "management-end-ip (Optional)", "plain", "xsd:string", "End of management IP address range for subcloud." + "description (Optional)", "plain", "xsd:string", "The description provisioned for the subcloud." + +:: + + { + "subclouds": [ + { + "deploy-status": "complete", + "id": 1, + "group_id": 2, + "created-at": "2020-04-13 13:16:21.903294", + "updated-at": "2020-04-13 13:36:27.494056", + "name": "subcloud1", + "management-state": "unmanaged", + "management-start-ip": "192.168.101.2", + "software-version": "20.01", + "availability-status": "offline", + "systemcontroller-gateway-ip": "192.168.204.101", + "location": "YOW", + "openstack-installed": false, + "management-subnet": "192.168.101.0/24", + "management-gateway-ip": "192.168.101.1", + "management-end-ip": "192.168.101.50", + "description": "Ottawa Site" + } + ] + } + +This operation does not accept a request body. + +*********************************** +Modifies a specific subcloud group +*********************************** + +.. rest_method:: PATCH /v1.0/subcloud-groups/​{subcloud-group}​ + +The attributes of a subcloud group which are modifiable: + +- name + +- description + +- update_apply_type + +- max_parallel_subclouds + +**Normal response codes** + +200 + +**Error response codes** + +badRequest (400), unauthorized (401), forbidden (403), badMethod (405), +HTTPUnprocessableEntity (422), internalServerError (500), +serviceUnavailable (503) + +**Request parameters** + +.. csv-table:: + :header: "Parameter", "Style", "Type", "Description" + :widths: 20, 20, 20, 60 + + "subcloud-group", "URI", "xsd:string", "The subcloud group reference, name or id." + "name (Optional)", "plain", "xsd:string", "The name of the subcloud group. Must be unique." + "description (Optional)", "plain", "xsd:string", "The description of the subcloud group." + "update_apply_type (Optional)", "plain", "xsd:string", "The update apply type for the subcloud group. Either ```serial``` or ```parallel```." + "max_parallel_subclouds (Optional)", "plain", "xsd:int", "The number of subclouds to update in parallel. Must be greater than 0." + +**Response parameters** + +.. csv-table:: + :header: "Parameter", "Style", "Type", "Description" + :widths: 20, 20, 20, 60 + + "id (Optional)", "plain", "xsd:int", "The unique identifier for this object." + "name (Optional)", "plain", "xsd:string", "The name provisioned for the subcloud group." + "description (Optional)", "plain", "xsd:string", "The description for the subcloud group." + "created_at (Optional)", "plain", "xsd:dateTime", "The time when the object was created." + "updated_at (Optional)", "plain", "xsd:dateTime", "The time when the object was last updated." + +:: + + { + "description": "new description", + "update_apply_type": "serial", + "max_parallel_subclouds": 5 + } + +:: + + { + "id": 2, + "name": "GroupX", + "description": "new description", + "update_apply_type": "serial", + "max_parallel_subclouds": 5, + "created-at": "2020-04-08 15:15:10.750592", + "updated-at": "2020-04-08 15:21:01.527101" + } + +********************************** +Deletes a specific subcloud group +********************************** + +.. rest_method:: DELETE /v1.0/subcloud-groups/​{subcloud-group}​ + +**Normal response codes** + +204 + +**Request parameters** + +.. csv-table:: + :header: "Parameter", "Style", "Type", "Description" + :widths: 20, 20, 20, 60 + + "subcloud-group", "URI", "xsd:string", "The subcloud group reference, name or id." + +This operation does not accept a request body. + ---------------- Subcloud Alarms ---------------- diff --git a/distributedcloud/dcmanager/api/controllers/v1/root.py b/distributedcloud/dcmanager/api/controllers/v1/root.py index 9dc4c25b0..4f9682a45 100644 --- a/distributedcloud/dcmanager/api/controllers/v1/root.py +++ b/distributedcloud/dcmanager/api/controllers/v1/root.py @@ -22,6 +22,7 @@ from dcmanager.api.controllers.v1 import alarm_manager +from dcmanager.api.controllers.v1 import subcloud_group from dcmanager.api.controllers.v1 import subclouds from dcmanager.api.controllers.v1 import sw_update_options from dcmanager.api.controllers.v1 import sw_update_strategy @@ -46,6 +47,8 @@ class Controller(object): sw_update_strategy.SwUpdateStrategyController sub_controllers["sw-update-options"] = \ sw_update_options.SwUpdateOptionsController + sub_controllers["subcloud-groups"] = \ + subcloud_group.SubcloudGroupsController for name, ctrl in sub_controllers.items(): setattr(self, name, ctrl) diff --git a/distributedcloud/dcmanager/api/controllers/v1/subcloud_group.py b/distributedcloud/dcmanager/api/controllers/v1/subcloud_group.py new file mode 100755 index 000000000..082621f6c --- /dev/null +++ b/distributedcloud/dcmanager/api/controllers/v1/subcloud_group.py @@ -0,0 +1,313 @@ +# Copyright (c) 2017 Ericsson AB. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Copyright (c) 2020 Wind River Systems, Inc. +# +# The right to copy, distribute, modify, or otherwise make use +# of this software may be licensed only pursuant to the terms +# of an applicable Wind River license agreement. +# + +from oslo_config import cfg +from oslo_db import exception as db_exc +from oslo_log import log as logging +from oslo_messaging import RemoteError + +import http.client as httpclient +import pecan +from pecan import expose +from pecan import request + +from dcmanager.api.controllers import restcomm +from dcmanager.common import consts +from dcmanager.common import exceptions +from dcmanager.common.i18n import _ +from dcmanager.db import api as db_api +from dcmanager.rpc import client as rpc_client + +CONF = cfg.CONF +LOG = logging.getLogger(__name__) + +SUPPORTED_GROUP_APPLY_TYPES = [ + consts.SUBCLOUD_APPLY_TYPE_PARALLEL, + consts.SUBCLOUD_APPLY_TYPE_SERIAL +] + +# validation constants for Subcloud Group +MAX_SUBCLOUD_GROUP_NAME_LEN = 255 +MAX_SUBCLOUD_GROUP_DESCRIPTION_LEN = 255 +MIN_SUBCLOUD_GROUP_MAX_PARALLEL_SUBCLOUDS = 1 +MAX_SUBCLOUD_GROUP_MAX_PARALLEL_SUBCLOUDS = 100 + + +class SubcloudGroupsController(object): + + def __init__(self): + super(SubcloudGroupsController, self).__init__() + self.rpc_client = rpc_client.ManagerClient() + + @expose(generic=True, template='json') + def index(self): + # Route the request to specific methods with parameters + pass + + def _get_subcloud_list_for_group(self, context, group_id): + subcloud_list = [] + subclouds = db_api.subcloud_get_for_group(context, group_id) + + for subcloud in subclouds: + subcloud_dict = db_api.subcloud_db_model_to_dict(subcloud) + subcloud_list.append(subcloud_dict) + result = dict() + result['subclouds'] = subcloud_list + return result + + def _get_subcloud_group_list(self, context): + groups = db_api.subcloud_group_get_all(context) + subcloud_group_list = [] + + for group in groups: + group_dict = db_api.subcloud_group_db_model_to_dict(group) + subcloud_group_list.append(group_dict) + + result = dict() + result['subcloud_groups'] = subcloud_group_list + return result + + def _get_by_ref(self, context, group_ref): + # Handle getting a group by either name, or ID + group = None + if group_ref.isdigit(): + # Lookup subcloud group as an ID + try: + group = db_api.subcloud_group_get(context, group_ref) + except exceptions.SubcloudGroupNotFound: + return None + else: + # Lookup subcloud group as a name + try: + group = db_api.subcloud_group_get_by_name(context, group_ref) + except exceptions.SubcloudGroupNameNotFound: + return None + return group + + @index.when(method='GET', template='json') + def get(self, group_ref=None, subclouds=False): + """Get details about subcloud group. + + :param group_ref: ID or name of subcloud group + """ + context = restcomm.extract_context_from_environ() + + if group_ref is None: + # List of subcloud groups requested + return self._get_subcloud_group_list(context) + + group = self._get_by_ref(context, group_ref) + if group is None: + pecan.abort(httpclient.NOT_FOUND, _('Subcloud Group not found')) + if subclouds: + # Return only the subclouds for this subcloud group + return self._get_subcloud_list_for_group(context, group.id) + subcloud_group_dict = db_api.subcloud_group_db_model_to_dict(group) + return subcloud_group_dict + + def _validate_name(self, name): + # Reject post and update operations for name that: + # - attempt to set to None + # - attempt to set to a number + # - attempt to set to the Default subcloud group + # - exceed the max length + if not name: + return False + if name.isdigit(): + return False + if name == consts.DEFAULT_SUBCLOUD_GROUP_NAME: + return False + if len(name) >= MAX_SUBCLOUD_GROUP_NAME_LEN: + return False + return True + + def _validate_description(self, description): + if not description: + return False + if len(description) >= MAX_SUBCLOUD_GROUP_DESCRIPTION_LEN: + return False + return True + + def _validate_update_apply_type(self, update_apply_type): + if not update_apply_type: + return False + if update_apply_type not in SUPPORTED_GROUP_APPLY_TYPES: + return False + return True + + def _validate_max_parallel_subclouds(self, max_parallel_str): + if not max_parallel_str: + return False + try: + # Check the value is an integer + val = int(max_parallel_str) + except ValueError: + return False + + # We do not support less than min or greater than max + if val < MIN_SUBCLOUD_GROUP_MAX_PARALLEL_SUBCLOUDS: + return False + if val > MAX_SUBCLOUD_GROUP_MAX_PARALLEL_SUBCLOUDS: + return False + return True + + @index.when(method='POST', template='json') + def post(self): + """Create a new subcloud group.""" + context = restcomm.extract_context_from_environ() + + payload = eval(request.body) + if not payload: + pecan.abort(httpclient.BAD_REQUEST, _('Body required')) + + name = payload.get('name') + description = payload.get('description') + update_apply_type = payload.get('update_apply_type') + max_parallel_subclouds = payload.get('max_parallel_subclouds') + + # Validate payload + if not self._validate_name(name): + pecan.abort(httpclient.BAD_REQUEST, _('Invalid group name')) + if not self._validate_description(description): + pecan.abort(httpclient.BAD_REQUEST, _('Invalid group description')) + if not self._validate_update_apply_type(update_apply_type): + pecan.abort(httpclient.BAD_REQUEST, + _('Invalid group update_apply_type')) + if not self._validate_max_parallel_subclouds(max_parallel_subclouds): + pecan.abort(httpclient.BAD_REQUEST, + _('Invalid group max_parallel_subclouds')) + try: + group_ref = db_api.subcloud_group_create(context, + name, + description, + update_apply_type, + max_parallel_subclouds) + return db_api.subcloud_group_db_model_to_dict(group_ref) + except db_exc.DBDuplicateEntry: + LOG.info("Group create failed. Group %s already exists" % name) + pecan.abort(httpclient.BAD_REQUEST, + _('A subcloud group with this name already exists')) + except RemoteError as e: + pecan.abort(httpclient.UNPROCESSABLE_ENTITY, e.value) + except Exception as e: + # TODO(abailey) add support for GROUP already exists (409) + LOG.exception(e) + pecan.abort(httpclient.INTERNAL_SERVER_ERROR, + _('Unable to create subcloud group')) + + @index.when(method='PATCH', template='json') + def patch(self, group_ref): + """Update a subcloud group. + + :param group_ref: ID or name of subcloud group to update + """ + + context = restcomm.extract_context_from_environ() + if group_ref is None: + pecan.abort(httpclient.BAD_REQUEST, + _('Subcloud Group Name or ID required')) + + payload = eval(request.body) + if not payload: + pecan.abort(httpclient.BAD_REQUEST, _('Body required')) + + group = self._get_by_ref(context, group_ref) + if group is None: + pecan.abort(httpclient.NOT_FOUND, _('Subcloud Group not found')) + + name = payload.get('name') + description = payload.get('description') + update_apply_type = payload.get('update_apply_type') + max_parallel_str = payload.get('max_parallel_subclouds') + + if not (name or description or update_apply_type or max_parallel_str): + pecan.abort(httpclient.BAD_REQUEST, _('nothing to update')) + + # Check value is not None or empty before calling validate + if name: + if not self._validate_name(name): + pecan.abort(httpclient.BAD_REQUEST, + _('Invalid group name')) + # Special case. Default group name cannot be changed + if group.id == consts.DEFAULT_SUBCLOUD_GROUP_ID: + pecan.abort(httpclient.BAD_REQUEST, + _('Default group name cannot be changed')) + + if description: + if not self._validate_description(description): + pecan.abort(httpclient.BAD_REQUEST, + _('Invalid group description')) + if update_apply_type: + if not self._validate_update_apply_type(update_apply_type): + pecan.abort(httpclient.BAD_REQUEST, + _('Invalid group update_apply_type')) + if max_parallel_str: + if not self._validate_max_parallel_subclouds(max_parallel_str): + pecan.abort(httpclient.BAD_REQUEST, + _('Invalid group max_parallel_subclouds')) + + try: + updated_group = db_api.subcloud_group_update( + context, + group.id, + name=name, + description=description, + update_apply_type=update_apply_type, + max_parallel_subclouds=max_parallel_str) + return db_api.subcloud_group_db_model_to_dict(updated_group) + except RemoteError as e: + pecan.abort(httpclient.UNPROCESSABLE_ENTITY, e.value) + except Exception as e: + # additional exceptions. + LOG.exception(e) + pecan.abort(httpclient.INTERNAL_SERVER_ERROR, + _('Unable to update subcloud group')) + + @index.when(method='delete', template='json') + def delete(self, group_ref): + """Delete the subcloud group.""" + context = restcomm.extract_context_from_environ() + + if group_ref is None: + pecan.abort(httpclient.BAD_REQUEST, + _('Subcloud Group Name or ID required')) + group = self._get_by_ref(context, group_ref) + if group is None: + pecan.abort(httpclient.NOT_FOUND, _('Subcloud Group not found')) + if group.name == consts.DEFAULT_SUBCLOUD_GROUP_NAME: + pecan.abort(httpclient.BAD_REQUEST, + _('Default Subcloud Group may not be deleted')) + try: + # a subcloud group may not be deleted if it is use by any subclouds + subclouds = db_api.subcloud_get_for_group(context, group.id) + if len(subclouds) > 0: + pecan.abort(httpclient.BAD_REQUEST, + _('Subcloud Group not empty')) + db_api.subcloud_group_destroy(context, group.id) + except RemoteError as e: + pecan.abort(httpclient.UNPROCESSABLE_ENTITY, e.value) + except Exception as e: + LOG.exception(e) + pecan.abort(httpclient.INTERNAL_SERVER_ERROR, + _('Unable to delete subcloud group')) + # This should return nothing + return None diff --git a/distributedcloud/dcmanager/api/controllers/v1/subclouds.py b/distributedcloud/dcmanager/api/controllers/v1/subclouds.py index 0bebabb15..8eff4de9c 100644 --- a/distributedcloud/dcmanager/api/controllers/v1/subclouds.py +++ b/distributedcloud/dcmanager/api/controllers/v1/subclouds.py @@ -32,10 +32,6 @@ import pecan from pecan import expose from pecan import request -from controllerconfig.common.exceptions import ValidateFail -from controllerconfig.utils import validate_address_str -from controllerconfig.utils import validate_network_str - from dccommon.drivers.openstack.keystone_v3 import KeystoneClient from dccommon.drivers.openstack.sysinv_v1 import SysinvClient from dccommon import exceptions as dccommon_exceptions @@ -81,6 +77,14 @@ class SubcloudsController(object): # Route the request to specific methods with parameters pass + def _validate_group_id(self, context, group_id): + try: + # The DB API will raise an exception if the group_id is invalid + db_api.subcloud_group_get(context, group_id) + except Exception as e: + LOG.exception(e) + pecan.abort(400, _("Invalid group_id")) + def _validate_subcloud_config(self, context, name, @@ -91,7 +95,8 @@ class SubcloudsController(object): external_oam_subnet_str, external_oam_gateway_address_str, external_oam_floating_address_str, - systemcontroller_gateway_ip_str): + systemcontroller_gateway_ip_str, + group_id): """Check whether subcloud config is valid.""" # Validate the name @@ -116,28 +121,28 @@ class SubcloudsController(object): management_subnet = None try: - management_subnet = validate_network_str( + management_subnet = utils.validate_network_str( management_subnet_str, minimum_size=MIN_MANAGEMENT_SUBNET_SIZE, existing_networks=subcloud_subnets) - except ValidateFail as e: + except exceptions.ValidateFail as e: LOG.exception(e) pecan.abort(400, _("management_subnet invalid: %s") % e) # Parse/validate the start/end addresses management_start_ip = None try: - management_start_ip = validate_address_str( + management_start_ip = utils.validate_address_str( management_start_ip_str, management_subnet) - except ValidateFail as e: + except exceptions.ValidateFail as e: LOG.exception(e) pecan.abort(400, _("management_start_address invalid: %s") % e) management_end_ip = None try: - management_end_ip = validate_address_str( + management_end_ip = utils.validate_address_str( management_end_ip_str, management_subnet) - except ValidateFail as e: + except exceptions.ValidateFail as e: LOG.exception(e) pecan.abort(400, _("management_end_address invalid: %s") % e) @@ -156,9 +161,9 @@ class SubcloudsController(object): # Parse/validate the gateway try: - validate_address_str( + utils.validate_address_str( management_gateway_ip_str, management_subnet) - except ValidateFail as e: + except exceptions.ValidateFail as e: LOG.exception(e) pecan.abort(400, _("management_gateway_address invalid: %s") % e) @@ -185,9 +190,9 @@ class SubcloudsController(object): management_address_pool.prefix) systemcontroller_subnet = IPNetwork(systemcontroller_subnet_str) try: - validate_address_str( + utils.validate_address_str( systemcontroller_gateway_ip_str, systemcontroller_subnet) - except ValidateFail as e: + except exceptions.ValidateFail as e: LOG.exception(e) pecan.abort(400, _("systemcontroller_gateway_address invalid: %s") % e) @@ -207,28 +212,29 @@ class SubcloudsController(object): MIN_OAM_SUBNET_SIZE = 3 oam_subnet = None try: - oam_subnet = validate_network_str( + oam_subnet = utils.validate_network_str( external_oam_subnet_str, minimum_size=MIN_OAM_SUBNET_SIZE, existing_networks=subcloud_subnets) - except ValidateFail as e: + except exceptions.ValidateFail as e: LOG.exception(e) pecan.abort(400, _("external_oam_subnet invalid: %s") % e) # Parse/validate the addresses try: - validate_address_str( + utils.validate_address_str( external_oam_gateway_address_str, oam_subnet) - except ValidateFail as e: + except exceptions.ValidateFail as e: LOG.exception(e) pecan.abort(400, _("oam_gateway_address invalid: %s") % e) try: - validate_address_str( + utils.validate_address_str( external_oam_floating_address_str, oam_subnet) - except ValidateFail as e: + except exceptions.ValidateFail as e: LOG.exception(e) pecan.abort(400, _("oam_floating_address invalid: %s") % e) + self._validate_group_id(context, group_id) @staticmethod def _validate_install_values(payload): @@ -288,8 +294,8 @@ class SubcloudsController(object): network_str = (install_values['network_address'] + '/' + str(install_values['network_mask'])) try: - network = validate_network_str(network_str, 1) - except ValidateFail as e: + network = utils.validate_network_str(network_str, 1) + except exceptions.ValidateFail as e: LOG.exception(e) pecan.abort(400, _("network address invalid: %s") % e) @@ -546,6 +552,10 @@ class SubcloudsController(object): if not sysadmin_password: pecan.abort(400, _('subcloud sysadmin_password required')) + # If a subcloud group is not passed, use the default + group_id = payload.get('group_id', + consts.DEFAULT_SUBCLOUD_GROUP_ID) + self._validate_subcloud_config(context, name, management_subnet, @@ -555,7 +565,8 @@ class SubcloudsController(object): external_oam_subnet, external_oam_gateway_ip, external_oam_floating_ip, - systemcontroller_gateway_ip) + systemcontroller_gateway_ip, + group_id) if 'install_values' in payload: self._validate_install_values(payload) @@ -609,8 +620,9 @@ class SubcloudsController(object): management_state = payload.get('management-state') description = payload.get('description') location = payload.get('location') + group_id = payload.get('group_id') - if not (management_state or description or location): + if not (management_state or description or location or group_id): pecan.abort(400, _('nothing to update')) # Syntax checking @@ -619,12 +631,19 @@ class SubcloudsController(object): consts.MANAGEMENT_MANAGED]: pecan.abort(400, _('Invalid management-state')) + # Verify the group_id is valid + if group_id: + try: + db_api.subcloud_group_get(context, group_id) + except exceptions.SubcloudGroupNotFound: + pecan.abort(400, _('Invalid group-id')) + try: # Inform dcmanager-manager that subcloud has been updated. # It will do all the real work... subcloud = self.rpc_client.update_subcloud( context, subcloud_id, management_state=management_state, - description=description, location=location) + description=description, location=location, group_id=group_id) return subcloud except RemoteError as e: pecan.abort(422, e.value) diff --git a/distributedcloud/dcmanager/common/consts.py b/distributedcloud/dcmanager/common/consts.py index 3f1f15d74..c0f4f4c14 100644 --- a/distributedcloud/dcmanager/common/consts.py +++ b/distributedcloud/dcmanager/common/consts.py @@ -77,6 +77,13 @@ SW_UPDATE_ACTION_ABORT = "abort" SUBCLOUD_APPLY_TYPE_PARALLEL = "parallel" SUBCLOUD_APPLY_TYPE_SERIAL = "serial" +# Values for the Default Subcloud Group +DEFAULT_SUBCLOUD_GROUP_ID = 1 +DEFAULT_SUBCLOUD_GROUP_NAME = 'Default' +DEFAULT_SUBCLOUD_GROUP_DESCRIPTION = 'Default Subcloud Group' +DEFAULT_SUBCLOUD_GROUP_UPDATE_APPLY_TYPE = SUBCLOUD_APPLY_TYPE_PARALLEL +DEFAULT_SUBCLOUD_GROUP_MAX_PARALLEL_SUBCLOUDS = 2 + # Strategy step states STRATEGY_STATE_INITIAL = "initial" STRATEGY_STATE_UPDATING_PATCHES = "updating patches" diff --git a/distributedcloud/dcmanager/common/exceptions.py b/distributedcloud/dcmanager/common/exceptions.py index fb603943d..335963c4d 100644 --- a/distributedcloud/dcmanager/common/exceptions.py +++ b/distributedcloud/dcmanager/common/exceptions.py @@ -65,6 +65,12 @@ class BadRequest(DCManagerException): message = _('Bad %(resource)s request: %(msg)s') +class ValidateFail(DCManagerException): + def __init__(self, message): + self.message = message + super(ValidateFail, self).__init__() + + class NotFound(DCManagerException): message = _("Not found") @@ -124,6 +130,22 @@ class SubcloudPatchOptsNotFound(NotFound): "defaults will be used.") +class SubcloudGroupNotFound(NotFound): + message = _("Subcloud Group with id %(group_id)s doesn't exist.") + + +class SubcloudGroupNameNotFound(NotFound): + message = _("Subcloud Group with name %(name)s doesn't exist.") + + +class SubcloudGroupNameViolation(DCManagerException): + message = _("Default Subcloud Group name cannot be changed or reused.") + + +class SubcloudGroupDefaultNotDeletable(DCManagerException): + message = _("Default Subcloud Group %(group_id)s may not be deleted.") + + class ConnectionRefused(DCManagerException): message = _("Connection to the service endpoint is refused") diff --git a/distributedcloud/dcmanager/common/utils.py b/distributedcloud/dcmanager/common/utils.py index 1ac04c25c..37c31bed0 100644 --- a/distributedcloud/dcmanager/common/utils.py +++ b/distributedcloud/dcmanager/common/utils.py @@ -22,6 +22,7 @@ import grp import itertools +import netaddr import os import pwd import six.moves @@ -53,6 +54,59 @@ def get_batch_projects(batch_size, project_list, fillvalue=None): return six.moves.zip_longest(fillvalue=fillvalue, *args) +def validate_address_str(ip_address_str, network): + """Determine whether an address is valid.""" + try: + ip_address = netaddr.IPAddress(ip_address_str) + if ip_address.version != network.version: + msg = ("Invalid IP version - must match network version " + + ip_version_to_string(network.version)) + raise exceptions.ValidateFail(msg) + elif ip_address == network: + raise exceptions.ValidateFail("Cannot use network address") + elif ip_address == network.broadcast: + raise exceptions.ValidateFail("Cannot use broadcast address") + elif ip_address not in network: + raise exceptions.ValidateFail( + "Address must be in subnet %s" % str(network)) + return ip_address + except netaddr.AddrFormatError: + raise exceptions.ValidateFail( + "Invalid address - not a valid IP address") + + +def ip_version_to_string(ip_version): + """Returns a string representation of ip_version.""" + if ip_version == 4: + return "IPv4" + elif ip_version == 6: + return "IPv6" + else: + return "IP" + + +def validate_network_str(network_str, minimum_size, + existing_networks=None, multicast=False): + """Determine whether a network is valid.""" + try: + network = netaddr.IPNetwork(network_str) + if network.size < minimum_size: + raise exceptions.ValidateFail("Subnet too small - must have at " + "least %d addresses" % minimum_size) + elif network.version == 6 and network.prefixlen < 64: + raise exceptions.ValidateFail("IPv6 minimum prefix length is 64") + elif existing_networks: + if any(network.ip in subnet for subnet in existing_networks): + raise exceptions.ValidateFail("Subnet overlaps with another " + "configured subnet") + elif multicast and not network.is_multicast(): + raise exceptions.ValidateFail("Invalid subnet - must be multicast") + return network + except netaddr.AddrFormatError: + raise exceptions.ValidateFail( + "Invalid subnet - not a valid IP subnet") + + # to do validate the quota limits def validate_quota_limits(payload): for resource in payload: diff --git a/distributedcloud/dcmanager/db/api.py b/distributedcloud/dcmanager/db/api.py index c6b288765..1e204defa 100644 --- a/distributedcloud/dcmanager/db/api.py +++ b/distributedcloud/dcmanager/db/api.py @@ -67,7 +67,8 @@ def subcloud_db_model_to_dict(subcloud): "systemcontroller-gateway-ip": subcloud.systemcontroller_gateway_ip, "created-at": subcloud.created_at, - "updated-at": subcloud.updated_at} + "updated-at": subcloud.updated_at, + "group_id": subcloud.group_id} return result @@ -75,14 +76,14 @@ def subcloud_create(context, name, description, location, software_version, management_subnet, management_gateway_ip, management_start_ip, management_end_ip, systemcontroller_gateway_ip, deploy_status, - openstack_installed): + openstack_installed, group_id): """Create a subcloud.""" return IMPL.subcloud_create(context, name, description, location, software_version, management_subnet, management_gateway_ip, management_start_ip, management_end_ip, systemcontroller_gateway_ip, deploy_status, - openstack_installed) + openstack_installed, group_id) def subcloud_get(context, subcloud_id): @@ -113,12 +114,13 @@ def subcloud_get_all_with_status(context): def subcloud_update(context, subcloud_id, management_state=None, availability_status=None, software_version=None, description=None, location=None, audit_fail_count=None, - deploy_status=None, openstack_installed=None): + deploy_status=None, openstack_installed=None, + group_id=None): """Update a subcloud or raise if it does not exist.""" return IMPL.subcloud_update(context, subcloud_id, management_state, availability_status, software_version, description, location, audit_fail_count, - deploy_status, openstack_installed) + deploy_status, openstack_installed, group_id) def subcloud_destroy(context, subcloud_id): @@ -196,6 +198,67 @@ def subcloud_status_destroy_all(context, subcloud_id): return IMPL.subcloud_status_destroy_all(context, subcloud_id) +################### +# subcloud_group + +def subcloud_group_db_model_to_dict(subcloud_group): + """Convert subcloud_group db model to dictionary.""" + result = {"id": subcloud_group.id, + "name": subcloud_group.name, + "description": subcloud_group.description, + "update_apply_type": subcloud_group.update_apply_type, + "max_parallel_subclouds": subcloud_group.max_parallel_subclouds, + "created-at": subcloud_group.created_at, + "updated-at": subcloud_group.updated_at} + return result + + +def subcloud_group_create(context, name, description, update_apply_type, + max_parallel_subclouds): + """Create a subcloud_group.""" + return IMPL.subcloud_group_create(context, + name, + description, + update_apply_type, + max_parallel_subclouds) + + +def subcloud_group_get(context, group_id): + """Retrieve a subcloud_group or raise if it does not exist.""" + return IMPL.subcloud_group_get(context, group_id) + + +def subcloud_group_get_by_name(context, name): + """Retrieve a subcloud_group b name or raise if it does not exist.""" + return IMPL.subcloud_group_get_by_name(context, name) + + +def subcloud_group_get_all(context): + """Retrieve all subcloud groups.""" + return IMPL.subcloud_group_get_all(context) + + +def subcloud_get_for_group(context, group_id): + """Retrieve a subcloud_group or raise if it does not exist.""" + return IMPL.subcloud_get_for_group(context, group_id) + + +def subcloud_group_update(context, group_id, name, description, + update_apply_type, max_parallel_subclouds): + """Update the subcloud group or raise if it does not exist.""" + return IMPL.subcloud_group_update(context, + group_id, + name, + description, + update_apply_type, + max_parallel_subclouds) + + +def subcloud_group_destroy(context, group_id): + """Destroy the subcloud group or raise if it does not exist.""" + return IMPL.subcloud_group_destroy(context, group_id) + + ################### def sw_update_strategy_db_model_to_dict(sw_update_strategy): diff --git a/distributedcloud/dcmanager/db/sqlalchemy/api.py b/distributedcloud/dcmanager/db/sqlalchemy/api.py index 88c98051d..7d604452d 100644 --- a/distributedcloud/dcmanager/db/sqlalchemy/api.py +++ b/distributedcloud/dcmanager/db/sqlalchemy/api.py @@ -24,12 +24,13 @@ Implementation of SQLAlchemy backend. """ +import sqlalchemy import sys import threading from oslo_db import exception as db_exc +from oslo_db.exception import DBDuplicateEntry from oslo_db.sqlalchemy import enginefacade - from oslo_log import log as logging from oslo_utils import strutils from oslo_utils import uuidutils @@ -212,7 +213,7 @@ def subcloud_create(context, name, description, location, software_version, management_subnet, management_gateway_ip, management_start_ip, management_end_ip, systemcontroller_gateway_ip, deploy_status, - openstack_installed): + openstack_installed, group_id): with write_session() as session: subcloud_ref = models.Subcloud() subcloud_ref.name = name @@ -229,6 +230,7 @@ def subcloud_create(context, name, description, location, software_version, subcloud_ref.deploy_status = deploy_status subcloud_ref.audit_fail_count = 0 subcloud_ref.openstack_installed = openstack_installed + subcloud_ref.group_id = group_id session.add(subcloud_ref) return subcloud_ref @@ -237,7 +239,8 @@ def subcloud_create(context, name, description, location, software_version, def subcloud_update(context, subcloud_id, management_state=None, availability_status=None, software_version=None, description=None, location=None, audit_fail_count=None, - deploy_status=None, openstack_installed=None): + deploy_status=None, openstack_installed=None, + group_id=None): with write_session() as session: subcloud_ref = subcloud_get(context, subcloud_id) if management_state is not None: @@ -256,6 +259,8 @@ def subcloud_update(context, subcloud_id, management_state=None, subcloud_ref.deploy_status = deploy_status if openstack_installed is not None: subcloud_ref.openstack_installed = openstack_installed + if group_id is not None: + subcloud_ref.group_id = group_id subcloud_ref.save(session) return subcloud_ref @@ -269,7 +274,6 @@ def subcloud_destroy(context, subcloud_id): ########################## - @require_context def subcloud_status_get(context, subcloud_id, endpoint_type): result = model_query(context, models.SubcloudStatus). \ @@ -531,6 +535,138 @@ def sw_update_opts_default_destroy(context): session.delete(sw_update_opts_default_ref) +########################## +# subcloud group +########################## +@require_context +def subcloud_group_get(context, group_id): + try: + result = model_query(context, models.SubcloudGroup). \ + filter_by(deleted=0). \ + filter_by(id=group_id). \ + one() + except NoResultFound: + raise exception.SubcloudGroupNotFound(group_id=group_id) + except MultipleResultsFound: + raise exception.InvalidParameterValue( + err="Multiple entries found for subcloud group %s" % group_id) + + return result + + +@require_context +def subcloud_group_get_by_name(context, name): + try: + result = model_query(context, models.SubcloudGroup). \ + filter_by(deleted=0). \ + filter_by(name=name). \ + one() + except NoResultFound: + raise exception.SubcloudGroupNameNotFound(name=name) + except MultipleResultsFound: + # This exception should never happen due to the UNIQUE setting for name + raise exception.InvalidParameterValue( + err="Multiple entries found for subcloud group %s" % name) + + return result + + +# This method returns all subclouds for a particular subcloud group +@require_context +def subcloud_get_for_group(context, group_id): + return model_query(context, models.Subcloud). \ + filter_by(deleted=0). \ + filter_by(group_id=group_id). \ + order_by(models.Subcloud.id). \ + all() + + +@require_context +def subcloud_group_get_all(context): + result = model_query(context, models.SubcloudGroup). \ + filter_by(deleted=0). \ + order_by(models.SubcloudGroup.id). \ + all() + + return result + + +@require_admin_context +def subcloud_group_create(context, + name, + description, + update_apply_type, + max_parallel_subclouds): + with write_session() as session: + subcloud_group_ref = models.SubcloudGroup() + subcloud_group_ref.name = name + subcloud_group_ref.description = description + subcloud_group_ref.update_apply_type = update_apply_type + subcloud_group_ref.max_parallel_subclouds = max_parallel_subclouds + session.add(subcloud_group_ref) + return subcloud_group_ref + + +@require_admin_context +def subcloud_group_update(context, + group_id, + name=None, + description=None, + update_apply_type=None, + max_parallel_subclouds=None): + with write_session() as session: + subcloud_group_ref = subcloud_group_get(context, group_id) + if name is not None: + # Do not allow the name of the default group to be edited + if subcloud_group_ref.id == consts.DEFAULT_SUBCLOUD_GROUP_ID: + raise exception.SubcloudGroupNameViolation() + # do not allow another group to use the default group name + if name == consts.DEFAULT_SUBCLOUD_GROUP_NAME: + raise exception.SubcloudGroupNameViolation() + subcloud_group_ref.name = name + if description is not None: + subcloud_group_ref.description = description + if update_apply_type is not None: + subcloud_group_ref.update_apply_type = update_apply_type + if max_parallel_subclouds is not None: + subcloud_group_ref.max_parallel_subclouds = max_parallel_subclouds + subcloud_group_ref.save(session) + return subcloud_group_ref + + +@require_admin_context +def subcloud_group_destroy(context, group_id): + with write_session() as session: + subcloud_group_ref = subcloud_group_get(context, group_id) + if subcloud_group_ref.id == consts.DEFAULT_SUBCLOUD_GROUP_ID: + raise exception.SubcloudGroupDefaultNotDeletable(group_id=group_id) + session.delete(subcloud_group_ref) + + +def initialize_subcloud_group_default(engine): + try: + default_group = { + "id": consts.DEFAULT_SUBCLOUD_GROUP_ID, + "name": consts.DEFAULT_SUBCLOUD_GROUP_NAME, + "description": consts.DEFAULT_SUBCLOUD_GROUP_DESCRIPTION, + "update_apply_type": + consts.DEFAULT_SUBCLOUD_GROUP_UPDATE_APPLY_TYPE, + "max_parallel_subclouds": + consts.DEFAULT_SUBCLOUD_GROUP_MAX_PARALLEL_SUBCLOUDS, + "deleted": 0 + } + meta = sqlalchemy.MetaData(bind=engine) + subcloud_group = sqlalchemy.Table('subcloud_group', meta, autoload=True) + try: + with engine.begin() as conn: + conn.execute(subcloud_group.insert(), default_group) + LOG.info("Default Subcloud Group created") + except DBDuplicateEntry: + # The default already exists. + pass + except Exception as ex: + LOG.error("Exception occurred setting up default subcloud group", ex) + ########################## @@ -612,11 +748,18 @@ def strategy_step_destroy_all(context): ########################## +def initialize_db_defaults(engine): + # a default value may already exist. If it does not, create it + initialize_subcloud_group_default(engine) def db_sync(engine, version=None): """Migrate the database to `version` or the most recent version.""" - return migration.db_sync(engine, version=version) + retVal = migration.db_sync(engine, version=version) + # returns None if migration has completed + if retVal is None: + initialize_db_defaults(engine) + return retVal def db_version(engine): diff --git a/distributedcloud/dcmanager/db/sqlalchemy/migrate_repo/versions/006_add_subcloud_group_table.py b/distributedcloud/dcmanager/db/sqlalchemy/migrate_repo/versions/006_add_subcloud_group_table.py new file mode 100644 index 000000000..a8b8b782a --- /dev/null +++ b/distributedcloud/dcmanager/db/sqlalchemy/migrate_repo/versions/006_add_subcloud_group_table.py @@ -0,0 +1,101 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Copyright (c) 2020 Wind River Systems, Inc. +# +# The right to copy, distribute, modify, or otherwise make use +# of this software may be licensed only pursuant to the terms +# of an applicable Wind River license agreement. +# + +from migrate.changeset import constraint +import sqlalchemy + +from dcmanager.common import consts + +ENGINE = 'InnoDB', +CHARSET = 'utf8' + + +def upgrade(migrate_engine): + meta = sqlalchemy.MetaData(bind=migrate_engine) + + # Declare the new subcloud_group table + subcloud_group = sqlalchemy.Table( + 'subcloud_group', meta, + sqlalchemy.Column('id', sqlalchemy.Integer, + primary_key=True, + autoincrement=True, + nullable=False), + sqlalchemy.Column('name', sqlalchemy.String(255), unique=True), + sqlalchemy.Column('description', sqlalchemy.String(255)), + sqlalchemy.Column('update_apply_type', sqlalchemy.String(255)), + sqlalchemy.Column('max_parallel_subclouds', sqlalchemy.Integer), + sqlalchemy.Column('reserved_1', sqlalchemy.Text), + sqlalchemy.Column('reserved_2', sqlalchemy.Text), + sqlalchemy.Column('created_at', sqlalchemy.DateTime), + sqlalchemy.Column('updated_at', sqlalchemy.DateTime), + sqlalchemy.Column('deleted_at', sqlalchemy.DateTime), + sqlalchemy.Column('deleted', sqlalchemy.Integer, default=0), + mysql_engine=ENGINE, + mysql_charset=CHARSET + ) + subcloud_group.create() + + subclouds = sqlalchemy.Table('subclouds', meta, autoload=True) + + # TODO(abailey) do we want to fix the missing constraint for strategy_steps + # strat_steps = sqlalchemy.Table('strategy_steps', meta, autoload=True) + # strat_fkey = constraint.ForeignKeyConstraint( + # columns=[strat_steps.c.subcloud_id], + # refcolumns=[subclouds.c.id], + # name='strat_subcloud_ref') + # strat_steps.append_constraint(strat_fkey) + + # Create a default subcloud group + default_group = { + "id": consts.DEFAULT_SUBCLOUD_GROUP_ID, + "name": consts.DEFAULT_SUBCLOUD_GROUP_NAME, + "description": consts.DEFAULT_SUBCLOUD_GROUP_DESCRIPTION, + "update_apply_type": consts.DEFAULT_SUBCLOUD_GROUP_UPDATE_APPLY_TYPE, + "max_parallel_subclouds": + consts.DEFAULT_SUBCLOUD_GROUP_MAX_PARALLEL_SUBCLOUDS, + "deleted": 0 + } + # Inserting the GROUP as ID 1, + # This should increment the pkey to 2 + with migrate_engine.begin() as conn: + conn.execute(subcloud_group.insert(), default_group) + + # postgres does not increment the subcloud group id sequence + # after the insert above as part of the migrate. + # Note: use different SQL syntax if using mysql or sqlite + if migrate_engine.name == 'postgresql': + with migrate_engine.begin() as conn: + conn.execute("ALTER SEQUENCE subcloud_group_id_seq RESTART WITH 2") + + # Add group_id column to subclouds table + group_id = \ + sqlalchemy.Column('group_id', + sqlalchemy.Integer, + server_default=str(consts.DEFAULT_SUBCLOUD_GROUP_ID)) + group_id.create(subclouds) + + subcloud_fkey = constraint.ForeignKeyConstraint( + columns=[subclouds.c.group_id], + refcolumns=[subcloud_group.c.id], + name='subclouds_group_ref') + subclouds.append_constraint(subcloud_fkey) + + +def downgrade(migrate_engine): + raise NotImplementedError('Database downgrade is unsupported.') diff --git a/distributedcloud/dcmanager/db/sqlalchemy/models.py b/distributedcloud/dcmanager/db/sqlalchemy/models.py index dc106ee6f..76e340839 100644 --- a/distributedcloud/dcmanager/db/sqlalchemy/models.py +++ b/distributedcloud/dcmanager/db/sqlalchemy/models.py @@ -75,6 +75,18 @@ class DCManagerBase(models.ModelBase, session.commit() +class SubcloudGroup(BASE, DCManagerBase): + """Represents a subcloud group""" + + __tablename__ = 'subcloud_group' + + id = Column(Integer, primary_key=True, autoincrement=True, nullable=False) + name = Column(String(255), unique=True) + description = Column(String(255)) + update_apply_type = Column(String(255)) + max_parallel_subclouds = Column(Integer) + + class Subcloud(BASE, DCManagerBase): """Represents a subcloud""" @@ -95,6 +107,11 @@ class Subcloud(BASE, DCManagerBase): openstack_installed = Column(Boolean, nullable=False, default=False) systemcontroller_gateway_ip = Column(String(255)) audit_fail_count = Column(Integer) + # multiple subclouds can be in a particular group + group_id = Column(Integer, + ForeignKey('subcloud_group.id')) + group = relationship(SubcloudGroup, + backref=backref('subcloud')) class SubcloudStatus(BASE, DCManagerBase): diff --git a/distributedcloud/dcmanager/manager/service.py b/distributedcloud/dcmanager/manager/service.py index 6968d401b..841248f9e 100644 --- a/distributedcloud/dcmanager/manager/service.py +++ b/distributedcloud/dcmanager/manager/service.py @@ -149,13 +149,14 @@ class DCManagerService(service.Service): @request_context def update_subcloud(self, context, subcloud_id, management_state=None, - description=None, location=None): + description=None, location=None, group_id=None): # Updates a subcloud LOG.info("Handling update_subcloud request for: %s" % subcloud_id) subcloud = self.subcloud_manager.update_subcloud(context, subcloud_id, management_state, description, - location) + location, + group_id) # If a subcloud has been set to the managed state, trigger the # patching audit so it can update the sync status ASAP. if management_state == consts.MANAGEMENT_MANAGED: diff --git a/distributedcloud/dcmanager/manager/subcloud_manager.py b/distributedcloud/dcmanager/manager/subcloud_manager.py index 07e8bad62..73131fef1 100644 --- a/distributedcloud/dcmanager/manager/subcloud_manager.py +++ b/distributedcloud/dcmanager/manager/subcloud_manager.py @@ -130,6 +130,9 @@ class SubcloudManager(manager.Manager): # controller. software_version = SW_VERSION try: + # if group_id has been omitted from payload, use 'Default'. + group_id = payload.get('group_id', + consts.DEFAULT_SUBCLOUD_GROUP_ID) subcloud = db_api.subcloud_create( context, payload['name'], @@ -142,7 +145,8 @@ class SubcloudManager(manager.Manager): payload['management_end_address'], payload['systemcontroller_gateway_address'], consts.DEPLOY_STATE_NONE, - False) + False, + group_id) except Exception as e: LOG.exception(e) raise e @@ -660,8 +664,13 @@ class SubcloudManager(manager.Manager): "subcloud %s" % subcloud.name) LOG.exception(e) - def update_subcloud(self, context, subcloud_id, management_state=None, - description=None, location=None): + def update_subcloud(self, + context, + subcloud_id, + management_state=None, + description=None, + location=None, + group_id=None): """Update subcloud and notify orchestrators. :param context: request context object @@ -669,6 +678,7 @@ class SubcloudManager(manager.Manager): :param management_state: new management state :param description: new description :param location: new location + :param group_id: new subcloud group id """ LOG.info("Updating subcloud %s." % subcloud_id) @@ -699,10 +709,12 @@ class SubcloudManager(manager.Manager): LOG.error("Invalid management_state %s" % management_state) raise exceptions.InternalError() - subcloud = db_api.subcloud_update(context, subcloud_id, + subcloud = db_api.subcloud_update(context, + subcloud_id, management_state=management_state, description=description, - location=location) + location=location, + group_id=group_id) # Inform orchestrators that subcloud has been updated if management_state: diff --git a/distributedcloud/dcmanager/rpc/client.py b/distributedcloud/dcmanager/rpc/client.py index beebe02a3..d8b1b52fc 100644 --- a/distributedcloud/dcmanager/rpc/client.py +++ b/distributedcloud/dcmanager/rpc/client.py @@ -72,12 +72,13 @@ class ManagerClient(object): subcloud_id=subcloud_id)) def update_subcloud(self, ctxt, subcloud_id, management_state=None, - description=None, location=None): + description=None, location=None, group_id=None): return self.call(ctxt, self.make_msg('update_subcloud', subcloud_id=subcloud_id, management_state=management_state, description=description, - location=location)) + location=location, + group_id=group_id)) def update_subcloud_endpoint_status(self, ctxt, subcloud_name=None, endpoint_type=None, diff --git a/distributedcloud/dcmanager/tests/base.py b/distributedcloud/dcmanager/tests/base.py index 72aa42a2d..dae3868d3 100644 --- a/distributedcloud/dcmanager/tests/base.py +++ b/distributedcloud/dcmanager/tests/base.py @@ -40,18 +40,37 @@ from sqlalchemy.engine import Engine from sqlalchemy import event SUBCLOUD_SAMPLE_DATA_0 = [ - 6, "subcloud-4", "demo subcloud", "Ottawa-Lab-Aisle_3-Rack_C", - "20.01", "managed", "online", "fd01:3::0/64", "fd01:3::1", - "fd01:3::2", "fd01:3::f", "fd01:1::1", 0, "NULL", "NULL", - "2018-05-15 14:45:12.508708", "2018-05-24 10:48:18.090931", - "NULL", 0, "10.10.10.0/24", "10.10.10.1", "10.10.10.12", "testpass" + 6, # id + "subcloud-4", # name + "demo subcloud", # description + "Ottawa-Lab-Aisle_3-Rack_C", # location + "20.01", # software-version + "managed", # management-state + "online", # availability-status + "fd01:3::0/64", # management_subnet + "fd01:3::1", # management_gateway_address + "fd01:3::2", # management_start_address + "fd01:3::f", # management_end_address + "fd01:1::1", # systemcontroller_gateway_address + 0, # audit-fail-count + "NULL", # reserved-1 + "NULL", # reserved-2 + "2018-05-15 14:45:12.508708", # created-at + "2018-05-24 10:48:18.090931", # updated-at + "NULL", # deleted-at + 0, # deleted + "10.10.10.0/24", # external_oam_subnet + "10.10.10.1", # external_oam_gateway_address + "10.10.10.12", # external_oam_floating_address + "testpass", # sysadmin_password + 1 # group_id ] @event.listens_for(Engine, "connect") def set_sqlite_pragma(dbapi_connection, connection_record): cursor = dbapi_connection.cursor() - cursor.execute("PRAGMA foreign_keys=ON") + cursor.execute("PRAGMA foreign_keys=ON;") cursor.close() diff --git a/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_subcloud_group.py b/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_subcloud_group.py new file mode 100644 index 000000000..7a1d24e39 --- /dev/null +++ b/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_subcloud_group.py @@ -0,0 +1,539 @@ +# Copyright (c) 2017 Ericsson AB +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Copyright (c) 2020 Wind River Systems, Inc. +# +# The right to copy, distribute, modify, or otherwise make use +# of this software may be licensed only pursuant to the terms +# of an applicable Wind River license agreement. +# + +import mock +from six.moves import http_client + +from dcmanager.common import consts +from dcmanager.db.sqlalchemy import api as db_api +from dcmanager.rpc import client as rpc_client + +from dcmanager.tests.unit.api import test_root_controller as testroot +from dcmanager.tests.unit.api.v1.controllers.test_subclouds \ + import FAKE_SUBCLOUD_DATA +from dcmanager.tests import utils + +SAMPLE_SUBCLOUD_GROUP_NAME = 'GroupX' +SAMPLE_SUBCLOUD_GROUP_DESCRIPTION = 'A Group of mystery' +SAMPLE_SUBCLOUD_GROUP_UPDATE_APPLY_TYPE = consts.SUBCLOUD_APPLY_TYPE_SERIAL +SAMPLE_SUBCLOUD_GROUP_MAX_PARALLEL_SUBCLOUDS = 3 + + +# APIMixin can be moved to its own file, once the other +# unit tests are refactored to utilize it +class APIMixin(object): + + FAKE_TENANT = utils.UUID1 + + api_headers = { + 'X-Tenant-Id': FAKE_TENANT, + 'X_ROLE': 'admin', + 'X-Identity-Status': 'Confirmed' + } + + # subclasses should provide methods + # get_api_prefix + # get_result_key + + def setUp(self): + super(APIMixin, self).setUp() + + def get_api_headers(self): + return self.api_headers + + def get_single_url(self, uuid): + return '%s/%s' % (self.get_api_prefix(), uuid) + + def get_api_prefix(self): + raise NotImplementedError + + def get_result_key(self): + raise NotImplementedError + + def get_expected_api_fields(self): + raise NotImplementedError + + def get_omitted_api_fields(self): + raise NotImplementedError + + # base mixin subclass MUST override these methods if the api supports them + def _create_db_object(self, context): + raise NotImplementedError + + # base mixin subclass should provide this method for testing of POST + def get_post_object(self): + raise NotImplementedError + + def get_update_object(self): + raise NotImplementedError + + def assert_fields(self, api_object): + # Verify that expected attributes are returned + for field in self.get_expected_api_fields(): + self.assertIn(field, api_object) + + # Verify that hidden attributes are not returned + for field in self.get_omitted_api_fields(): + self.assertNotIn(field, api_object) + + +# +# --------------------- POST ----------------------------------- +# +# An API test will mixin only one of: PostMixin or PostRejectedMixin +# depending on whether or not the API supports a post operation or not +class PostMixin(object): + + @mock.patch.object(rpc_client, 'ManagerClient') + def test_create_success(self, mock_client): + # Test that a POST operation is supported by the API + ndict = self.get_post_object() + response = self.app.post_json(self.get_api_prefix(), + ndict, + headers=self.get_api_headers()) + self.assertEqual(response.content_type, 'application/json') + self.assertEqual(response.status_code, http_client.OK) + self.assert_fields(response.json) + + +class PostRejectedMixin(object): + # Test that a POST operation is blocked by the API + # API should return 400 BAD_REQUEST or FORBIDDEN 403 + @mock.patch.object(rpc_client, 'ManagerClient') + def test_create_not_allowed(self, mock_client): + ndict = self.get_post_object() + response = self.app.post_json(self.API_PREFIX, + ndict, + headers=self.get_api_headers(), + expect_errors=True) + self.assertEqual(response.status_code, http_client.FORBIDDEN) + self.assertTrue(response.json['error_message']) + self.assertIn("Operation not permitted.", + response.json['error_message']) + + +# ------ API GET mixin +class GetMixin(object): + + # Mixins can override initial_list_size if a table is not empty during + # DB creation and migration sync + initial_list_size = 0 + + # Performing a GET on this ID should fail. subclass mixins can override + invalid_id = '123' + + def validate_entry(self, result_item): + self.assert_fields(result_item) + + def validate_list(self, expected_length, results): + self.assertIn(self.get_result_key(), results) + result_list = results.get(self.get_result_key()) + self.assertEqual(expected_length, len(result_list)) + for result_item in result_list: + self.validate_entry(result_item) + + def validate_list_response(self, expected_length, response): + self.assertEqual(response.content_type, 'application/json') + self.assertEqual(response.status_code, http_client.OK) + + # validate the list length + self.validate_list(expected_length, response.json) + + @mock.patch.object(rpc_client, 'ManagerClient') + def test_initial_list_size(self, mock_client): + # Test that a GET operation for a list is supported by the API + response = self.app.get(self.get_api_prefix(), + headers=self.get_api_headers()) + # Validate the initial length + self.validate_list_response(self.initial_list_size, response) + + # Add an entry + context = utils.dummy_context() + self._create_db_object(context) + + response = self.app.get(self.get_api_prefix(), + headers=self.get_api_headers()) + self.validate_list_response(self.initial_list_size + 1, response) + + @mock.patch.object(rpc_client, 'ManagerClient') + def test_fail_get_single(self, mock_client): + # Test that a GET operation for an invalid ID returns the + # appropriate error results + response = self.app.get(self.get_single_url(self.invalid_id), + headers=self.get_api_headers(), + expect_errors=True) + # Failures will return text rather than json + self.assertEqual(response.content_type, 'text/plain') + self.assertEqual(response.status_code, http_client.NOT_FOUND) + + @mock.patch.object(rpc_client, 'ManagerClient') + def test_get_single(self, mock_client): + # create a group + context = utils.dummy_context() + group_name = 'TestGroup' + db_group = self._create_db_object(context, name=group_name) + + # Test that a GET operation for a valid ID works + response = self.app.get(self.get_single_url(db_group.id), + headers=self.get_api_headers()) + self.assertEqual(response.content_type, 'application/json') + self.assertEqual(response.status_code, http_client.OK) + self.validate_entry(response.json) + + +# ------ API Update Mixin +class UpdateMixin(object): + + def validate_updated_fields(self, sub_dict, full_obj): + for key, value in sub_dict.items(): + self.assertEqual(value, full_obj.get(key)) + + @mock.patch.object(rpc_client, 'ManagerClient') + def test_update_success(self, mock_client): + context = utils.dummy_context() + single_obj = self._create_db_object(context) + update_data = self.get_update_object() + response = self.app.patch_json(self.get_single_url(single_obj.id), + headers=self.get_api_headers(), + params=update_data) + self.assertEqual(response.content_type, 'application/json') + self.assertEqual(response.status_code, http_client.OK) + self.validate_updated_fields(update_data, response.json) + + @mock.patch.object(rpc_client, 'ManagerClient') + def test_update_empty_changeset(self, mock_client): + context = utils.dummy_context() + single_obj = self._create_db_object(context) + update_data = {} + response = self.app.patch_json(self.get_single_url(single_obj.id), + headers=self.get_api_headers(), + params=update_data, + expect_errors=True) + # Failures will return text rather than json + self.assertEqual(response.content_type, 'text/plain') + self.assertEqual(response.status_code, http_client.BAD_REQUEST) + + +# ------ API Delete Mixin +class DeleteMixin(object): + + @mock.patch.object(rpc_client, 'ManagerClient') + def test_delete_success(self, mock_client): + context = utils.dummy_context() + single_obj = self._create_db_object(context) + response = self.app.delete(self.get_single_url(single_obj.id), + headers=self.get_api_headers()) + self.assertEqual(response.content_type, 'application/json') + self.assertEqual(response.status_code, http_client.OK) + + @mock.patch.object(rpc_client, 'ManagerClient') + def test_double_delete(self, mock_client): + context = utils.dummy_context() + single_obj = self._create_db_object(context) + response = self.app.delete(self.get_single_url(single_obj.id), + headers=self.get_api_headers()) + self.assertEqual(response.content_type, 'application/json') + self.assertEqual(response.status_code, http_client.OK) + # delete the same object a second time. this should fail (NOT_FOUND) + response = self.app.delete(self.get_single_url(single_obj.id), + headers=self.get_api_headers(), + expect_errors=True) + self.assertEqual(response.content_type, 'text/plain') + self.assertEqual(response.status_code, http_client.NOT_FOUND) + + +class SubcloudGroupAPIMixin(APIMixin): + + API_PREFIX = '/v1.0/subcloud-groups' + RESULT_KEY = 'subcloud_groups' + EXPECTED_FIELDS = ['id', + 'name', + 'description', + 'max_parallel_subclouds', + 'update_apply_type', + 'created-at', + 'updated-at'] + + def setUp(self): + super(SubcloudGroupAPIMixin, self).setUp() + self.fake_rpc_client.some_method = mock.MagicMock() + + def _get_test_subcloud_group_dict(self, **kw): + # id should not be part of the structure + group = { + 'name': kw.get('name', SAMPLE_SUBCLOUD_GROUP_NAME), + 'description': kw.get('description', + SAMPLE_SUBCLOUD_GROUP_DESCRIPTION), + 'update_apply_type': kw.get( + 'update_apply_type', + SAMPLE_SUBCLOUD_GROUP_UPDATE_APPLY_TYPE), + 'max_parallel_subclouds': kw.get( + 'max_parallel_subclouds', + SAMPLE_SUBCLOUD_GROUP_MAX_PARALLEL_SUBCLOUDS) + } + return group + + def _post_get_test_subcloud_group(self, **kw): + post_body = self._get_test_subcloud_group_dict(**kw) + return post_body + + # The following methods are required for subclasses of APIMixin + + def get_api_prefix(self): + return self.API_PREFIX + + def get_result_key(self): + return self.RESULT_KEY + + def get_expected_api_fields(self): + return self.EXPECTED_FIELDS + + def get_omitted_api_fields(self): + return [] + + def _create_db_object(self, context, **kw): + creation_fields = self._get_test_subcloud_group_dict(**kw) + return db_api.subcloud_group_create(context, **creation_fields) + + def get_post_object(self): + return self._post_get_test_subcloud_group() + + def get_update_object(self): + update_object = { + 'description': 'Updated description' + } + return update_object + + +# Combine Subcloud Group API with mixins to test post, get, update and delete +class TestSubcloudGroupPost(testroot.DCManagerApiTest, + SubcloudGroupAPIMixin, + PostMixin): + def setUp(self): + super(TestSubcloudGroupPost, self).setUp() + + def verify_post_failure(self, response): + # Failures will return text rather than json + self.assertEqual(response.content_type, 'text/plain') + self.assertEqual(response.status_code, http_client.BAD_REQUEST) + + @mock.patch.object(rpc_client, 'ManagerClient') + def test_create_with_numerical_name_fails(self, mock_client): + # A numerical name is not permitted. otherwise the 'get' operations + # which support getting by either name or ID could become confused + # if a name for one group was the same as an ID for another. + ndict = self.get_post_object() + ndict['name'] = '123' + response = self.app.post_json(self.get_api_prefix(), + ndict, + headers=self.get_api_headers(), + expect_errors=True) + self.verify_post_failure(response) + + @mock.patch.object(rpc_client, 'ManagerClient') + def test_create_with_blank_name_fails(self, mock_client): + # An empty name is not permitted + ndict = self.get_post_object() + ndict['name'] = '' + response = self.app.post_json(self.get_api_prefix(), + ndict, + headers=self.get_api_headers(), + expect_errors=True) + self.verify_post_failure(response) + + @mock.patch.object(rpc_client, 'ManagerClient') + def test_create_with_default_name_fails(self, mock_client): + # A name that is the same as the 'Default' group is not permitted. + # This would be a duplicate, and names must be unique. + ndict = self.get_post_object() + ndict['name'] = 'Default' + response = self.app.post_json(self.get_api_prefix(), + ndict, + headers=self.get_api_headers(), + expect_errors=True) + self.verify_post_failure(response) + + @mock.patch.object(rpc_client, 'ManagerClient') + def test_create_with_empty_description_fails(self, mock_client): + # An empty description is considered invalid + ndict = self.get_post_object() + ndict['description'] = '' + response = self.app.post_json(self.get_api_prefix(), + ndict, + headers=self.get_api_headers(), + expect_errors=True) + self.verify_post_failure(response) + + @mock.patch.object(rpc_client, 'ManagerClient') + def test_create_with_bad_apply_type(self, mock_client): + # update_apply_type must be either 'serial' or 'parallel' + ndict = self.get_post_object() + ndict['update_apply_type'] = 'something_invalid' + response = self.app.post_json(self.get_api_prefix(), + ndict, + headers=self.get_api_headers(), + expect_errors=True) + self.verify_post_failure(response) + + @mock.patch.object(rpc_client, 'ManagerClient') + def test_create_with_bad_max_parallel_subclouds(self, mock_client): + # max_parallel_subclouds must be an integer between 1 and 100 + ndict = self.get_post_object() + # All the entries in bad_values should be considered invalid + bad_values = [0, 101, -1, 'abc'] + for bad_value in bad_values: + ndict['max_parallel_subclouds'] = bad_value + response = self.app.post_json(self.get_api_prefix(), + ndict, + headers=self.get_api_headers(), + expect_errors=True) + self.verify_post_failure(response) + + +class TestSubcloudGroupGet(testroot.DCManagerApiTest, + SubcloudGroupAPIMixin, + GetMixin): + + def setUp(self): + super(TestSubcloudGroupGet, self).setUp() + # Override initial_list_size. Default group is setup during db sync + self.initial_list_size = 1 + + @mock.patch.object(rpc_client, 'ManagerClient') + def test_get_single_by_name(self, mock_client): + # create a group + context = utils.dummy_context() + # todo(abailey) make this a generic method + group_name = 'TestGroup' + self._create_db_object(context, name=group_name) + + # Test that a GET operation for a valid ID works + response = self.app.get(self.get_single_url(group_name), + headers=self.get_api_headers()) + self.assertEqual(response.content_type, 'application/json') + self.assertEqual(response.status_code, http_client.OK) + self.validate_entry(response.json) + + @mock.patch.object(rpc_client, 'ManagerClient') + def test_list_subclouds_empty(self, mock_client): + # API GET on: subcloud-groups//subclouds + uuid = 1 # The Default Subcloud Group is always ID=1 + url = '%s/%s/subclouds' % (self.get_api_prefix(), uuid) + response = self.app.get(url, + headers=self.get_api_headers()) + # This API returns 'subclouds' rather than 'subcloud-groups' + self.assertIn('subclouds', response.json) + # no subclouds exist yet, so this length should be zero + result_list = response.json.get('subclouds') + self.assertEqual(0, len(result_list)) + + def _create_subcloud_db_object(self, context): + creation_fields = { + 'name': FAKE_SUBCLOUD_DATA.get('name'), + 'description': FAKE_SUBCLOUD_DATA.get('description'), + 'location': FAKE_SUBCLOUD_DATA.get('location'), + 'software_version': FAKE_SUBCLOUD_DATA.get('software_version'), + 'management_subnet': FAKE_SUBCLOUD_DATA.get('management_subnet'), + 'management_gateway_ip': + FAKE_SUBCLOUD_DATA.get('management_gateway_ip'), + 'management_start_ip': + FAKE_SUBCLOUD_DATA.get('management_start_ip'), + 'management_end_ip': FAKE_SUBCLOUD_DATA.get('management_end_ip'), + 'systemcontroller_gateway_ip': + FAKE_SUBCLOUD_DATA.get('systemcontroller_gateway_ip'), + 'deploy_status': FAKE_SUBCLOUD_DATA.get('deploy_status'), + 'openstack_installed': + FAKE_SUBCLOUD_DATA.get('openstack_installed'), + 'group_id': FAKE_SUBCLOUD_DATA.get('group_id', 1) + } + return db_api.subcloud_create(context, **creation_fields) + + @mock.patch.object(rpc_client, 'ManagerClient') + def test_list_subclouds_populated(self, mock_client): + # subclouds are to Default group by default (unless specified) + context = utils.dummy_context() + self._create_subcloud_db_object(context) + + # API GET on: subcloud-groups//subclouds + uuid = 1 # The Default Subcloud Group is always ID=1 + url = '%s/%s/subclouds' % (self.get_api_prefix(), uuid) + response = self.app.get(url, + headers=self.get_api_headers()) + # This API returns 'subclouds' rather than 'subcloud-groups' + self.assertIn('subclouds', response.json) + # the subcloud created earlier will have been queried + result_list = response.json.get('subclouds') + self.assertEqual(1, len(result_list)) + + +class TestSubcloudGroupUpdate(testroot.DCManagerApiTest, + SubcloudGroupAPIMixin, + UpdateMixin): + def setUp(self): + super(TestSubcloudGroupUpdate, self).setUp() + + @mock.patch.object(rpc_client, 'ManagerClient') + def test_update_invalid_apply_type(self, mock_client): + context = utils.dummy_context() + single_obj = self._create_db_object(context) + update_data = { + 'update_apply_type': 'something_bad' + } + response = self.app.patch_json(self.get_single_url(single_obj.id), + headers=self.get_api_headers(), + params=update_data, + expect_errors=True) + # Failures will return text rather than json + self.assertEqual(response.content_type, 'text/plain') + self.assertEqual(response.status_code, http_client.BAD_REQUEST) + + @mock.patch.object(rpc_client, 'ManagerClient') + def test_update_invalid_max_parallel(self, mock_client): + context = utils.dummy_context() + single_obj = self._create_db_object(context) + update_data = { + 'max_parallel_subclouds': -1 + } + response = self.app.patch_json(self.get_single_url(single_obj.id), + headers=self.get_api_headers(), + params=update_data, + expect_errors=True) + # Failures will return text rather than json + self.assertEqual(response.content_type, 'text/plain') + self.assertEqual(response.status_code, http_client.BAD_REQUEST) + + +class TestSubcloudGroupDelete(testroot.DCManagerApiTest, + SubcloudGroupAPIMixin, + DeleteMixin): + def setUp(self): + super(TestSubcloudGroupDelete, self).setUp() + + @mock.patch.object(rpc_client, 'ManagerClient') + def test_delete_default_fails(self, mock_client): + default_zone_id = 1 + response = self.app.delete(self.get_single_url(default_zone_id), + headers=self.get_api_headers(), + expect_errors=True) + # Failures will return text rather than json + self.assertEqual(response.content_type, 'text/plain') + self.assertEqual(response.status_code, http_client.BAD_REQUEST) diff --git a/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_subclouds.py b/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_subclouds.py index 6bebedbd7..b138ac3a8 100644 --- a/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_subclouds.py +++ b/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_subclouds.py @@ -499,7 +499,8 @@ class TestSubclouds(testroot.DCManagerApiTest): mock.ANY, management_state=consts.MANAGEMENT_UNMANAGED, description=None, - location=None) + location=None, + group_id=None) self.assertEqual(response.status_int, 200) @mock.patch.object(rpc_client, 'ManagerClient') diff --git a/distributedcloud/dcmanager/tests/unit/db/test_subcloud_db_api.py b/distributedcloud/dcmanager/tests/unit/db/test_subcloud_db_api.py index 884cee5b7..cec930912 100644 --- a/distributedcloud/dcmanager/tests/unit/db/test_subcloud_db_api.py +++ b/distributedcloud/dcmanager/tests/unit/db/test_subcloud_db_api.py @@ -83,6 +83,7 @@ class DBAPISubcloudTest(base.DCManagerTestCase): 'systemcontroller_gateway_ip': "192.168.204.101", 'deploy_status': "not-deployed", 'openstack_installed': False, + 'group_id': 1, } values.update(kwargs) return db_api.subcloud_create(ctxt, **values) @@ -102,6 +103,7 @@ class DBAPISubcloudTest(base.DCManagerTestCase): 'systemcontroller_gateway_address'], 'deploy_status': "not-deployed", 'openstack_installed': False, + 'group_id': 1, } return db_api.subcloud_create(ctxt, **values) diff --git a/distributedcloud/dcmanager/tests/unit/manager/test_service.py b/distributedcloud/dcmanager/tests/unit/manager/test_service.py index 5213afdbb..8d247f91b 100644 --- a/distributedcloud/dcmanager/tests/unit/manager/test_service.py +++ b/distributedcloud/dcmanager/tests/unit/manager/test_service.py @@ -131,8 +131,9 @@ class TestDCManagerService(base.DCManagerTestCase): self.service_obj.update_subcloud( self.context, subcloud_id=1, management_state='testmgmtstatus') mock_subcloud_manager().update_subcloud.\ - assert_called_once_with(self.context, mock.ANY, mock.ANY, mock.ANY, - mock.ANY) + assert_called_once_with(self.context, mock.ANY, + mock.ANY, mock.ANY, + mock.ANY, mock.ANY) @mock.patch.object(service, 'SwUpdateManager') @mock.patch.object(service, 'SubcloudManager') diff --git a/distributedcloud/dcmanager/tests/unit/manager/test_subcloud_audit_manager.py b/distributedcloud/dcmanager/tests/unit/manager/test_subcloud_audit_manager.py index b0be0cc12..287e5ae2c 100644 --- a/distributedcloud/dcmanager/tests/unit/manager/test_subcloud_audit_manager.py +++ b/distributedcloud/dcmanager/tests/unit/manager/test_subcloud_audit_manager.py @@ -244,6 +244,7 @@ class TestAuditManager(base.DCManagerTestCase): 'systemcontroller_gateway_ip': "192.168.204.101", 'deploy_status': "not-deployed", 'openstack_installed': False, + 'group_id': 1, } values.update(kwargs) return db_api.subcloud_create(ctxt, **values) diff --git a/distributedcloud/dcmanager/tests/unit/manager/test_subcloud_manager.py b/distributedcloud/dcmanager/tests/unit/manager/test_subcloud_manager.py index 9c1869226..366803d37 100644 --- a/distributedcloud/dcmanager/tests/unit/manager/test_subcloud_manager.py +++ b/distributedcloud/dcmanager/tests/unit/manager/test_subcloud_manager.py @@ -34,13 +34,14 @@ from dcmanager.manager import subcloud_manager from dcmanager.tests import base from dcmanager.tests import utils from dcorch.common import consts as dcorch_consts -from dcorch.rpc import client as dcorch_rpc_client class FakeDCOrchAPI(object): def __init__(self): self.update_subcloud_states = mock.MagicMock() self.add_subcloud_sync_endpoint_type = mock.MagicMock() + self.del_subcloud = mock.MagicMock() + self.add_subcloud = mock.MagicMock() class FakeService(object): @@ -147,6 +148,7 @@ class TestSubcloudManager(base.DCManagerTestCase): "systemcontroller_gateway_ip": "192.168.204.101", 'deploy_status': "not-deployed", 'openstack_installed': False, + 'group_id': 1, } values.update(kwargs) return db_api.subcloud_create(ctxt, **values) @@ -160,8 +162,6 @@ class TestSubcloudManager(base.DCManagerTestCase): @mock.patch.object(subcloud_manager.SubcloudManager, '_delete_subcloud_inventory') - @mock.patch.object(dcorch_rpc_client, 'EngineClient') - @mock.patch.object(subcloud_manager, 'context') @mock.patch.object(subcloud_manager, 'KeystoneClient') @mock.patch.object(subcloud_manager, 'db_api') @mock.patch.object(subcloud_manager, 'SysinvClient') @@ -179,13 +179,11 @@ class TestSubcloudManager(base.DCManagerTestCase): mock_write_subcloud_ansible_config, mock_create_subcloud_inventory, mock_create_addn_hosts, mock_sysinv_client, - mock_db_api, mock_keystone_client, mock_context, - mock_dcorch_rpc_client, + mock_db_api, mock_keystone_client, mock_delete_subcloud_inventory): values = utils.create_subcloud_dict(base.SUBCLOUD_SAMPLE_DATA_0) controllers = FAKE_CONTROLLERS services = FAKE_SERVICES - mock_context.get_admin_context.return_value = self.ctx mock_db_api.subcloud_get_by_name.side_effect = \ exceptions.SubcloudNameNotFound() @@ -198,15 +196,13 @@ class TestSubcloudManager(base.DCManagerTestCase): mock_db_api.subcloud_create.assert_called_once() mock_db_api.subcloud_status_create.assert_called() mock_sysinv_client().create_route.assert_called() - mock_dcorch_rpc_client().add_subcloud.assert_called_once() + self.fake_dcorch_api.add_subcloud.assert_called_once() mock_create_addn_hosts.assert_called_once() mock_create_subcloud_inventory.assert_called_once() mock_write_subcloud_ansible_config.assert_called_once() mock_keyring.get_password.assert_called() mock_thread_start.assert_called_once() - @mock.patch.object(dcorch_rpc_client, 'EngineClient') - @mock.patch.object(subcloud_manager, 'context') @mock.patch.object(subcloud_manager, 'db_api') @mock.patch.object(subcloud_manager, 'SysinvClient') @mock.patch.object(subcloud_manager, 'KeystoneClient') @@ -215,11 +211,8 @@ class TestSubcloudManager(base.DCManagerTestCase): def test_delete_subcloud(self, mock_create_addn_hosts, mock_keystone_client, mock_sysinv_client, - mock_db_api, - mock_context, - mock_dcorch_rpc_client): + mock_db_api): controllers = FAKE_CONTROLLERS - mock_context.get_admin_context.return_value = self.ctx data = utils.create_subcloud_dict(base.SUBCLOUD_SAMPLE_DATA_0) fake_subcloud = Subcloud(data, False) mock_db_api.subcloud_get.return_value = fake_subcloud @@ -231,20 +224,15 @@ class TestSubcloudManager(base.DCManagerTestCase): mock_db_api.subcloud_destroy.assert_called_once() mock_create_addn_hosts.assert_called_once() - @mock.patch.object(dcorch_rpc_client, 'EngineClient') - @mock.patch.object(subcloud_manager, 'context') - @mock.patch.object(subcloud_manager, 'KeystoneClient') @mock.patch.object(subcloud_manager, 'db_api') - def test_update_subcloud(self, mock_db_api, - mock_endpoint, mock_context, - mock_dcorch_rpc_client): - mock_context.get_admin_context.return_value = self.ctx + def test_update_subcloud(self, mock_db_api): data = utils.create_subcloud_dict(base.SUBCLOUD_SAMPLE_DATA_0) subcloud_result = Subcloud(data, True) mock_db_api.subcloud_get.return_value = subcloud_result mock_db_api.subcloud_update.return_value = subcloud_result sm = subcloud_manager.SubcloudManager() - sm.update_subcloud(self.ctx, data['id'], + sm.update_subcloud(self.ctx, + data['id'], management_state=consts.MANAGEMENT_MANAGED, description="subcloud new description", location="subcloud new location") @@ -253,7 +241,29 @@ class TestSubcloudManager(base.DCManagerTestCase): data['id'], management_state=consts.MANAGEMENT_MANAGED, description="subcloud new description", - location="subcloud new location") + location="subcloud new location", + group_id=None) + + @mock.patch.object(subcloud_manager, 'db_api') + def test_update_subcloud_group_id(self, mock_db_api): + data = utils.create_subcloud_dict(base.SUBCLOUD_SAMPLE_DATA_0) + subcloud_result = Subcloud(data, True) + mock_db_api.subcloud_get.return_value = subcloud_result + mock_db_api.subcloud_update.return_value = subcloud_result + sm = subcloud_manager.SubcloudManager() + sm.update_subcloud(self.ctx, + data['id'], + management_state=consts.MANAGEMENT_MANAGED, + description="subcloud new description", + location="subcloud new location", + group_id=2) + mock_db_api.subcloud_update.assert_called_once_with( + mock.ANY, + data['id'], + management_state=consts.MANAGEMENT_MANAGED, + description="subcloud new description", + location="subcloud new location", + group_id=2) def test_update_subcloud_endpoint_status(self): # create a subcloud diff --git a/distributedcloud/dcmanager/tests/utils.py b/distributedcloud/dcmanager/tests/utils.py index 32f8e4f63..b1d127fc0 100644 --- a/distributedcloud/dcmanager/tests/utils.py +++ b/distributedcloud/dcmanager/tests/utils.py @@ -117,4 +117,5 @@ def create_subcloud_dict(data_list): 'external_oam_subnet': data_list[19], 'external_oam_gateway_address': data_list[20], 'external_oam_floating_address': data_list[21], - 'sysadmin_password': data_list[22]} + 'sysadmin_password': data_list[22], + 'group_id': data_list[23]} diff --git a/distributedcloud/tox.ini b/distributedcloud/tox.ini index 506421581..e63c93083 100644 --- a/distributedcloud/tox.ini +++ b/distributedcloud/tox.ini @@ -12,7 +12,6 @@ fmclient_src_dir = {[dc]stx_fault_dir}/python-fmclient/fmclient fm_api_src_dir = {[dc]stx_fault_dir}/fm-api sysinv_src_dir = ../../config/sysinv/sysinv/sysinv tsconfig_src_dir = ../../config/tsconfig/tsconfig -controllerconfig_src_dir = ../../config/controllerconfig/controllerconfig cgtsclient_src_dir = ../../config/sysinv/cgts-client/cgts-client cgcs_patch_src_dir = ../../update/cgcs-patch/cgcs-patch @@ -47,7 +46,6 @@ deps = -r{toxinidir}/test-requirements.txt -e{[dc]tsconfig_src_dir} -e{[dc]fmclient_src_dir} -e{[dc]fm_api_src_dir} - -e{[dc]controllerconfig_src_dir} -e{[dc]cgtsclient_src_dir} setenv = CURRENT_CFG_FILE={toxinidir}/.current.cfg @@ -87,7 +85,6 @@ deps = -r{toxinidir}/test-requirements.txt -e../{[dc]tsconfig_src_dir} -e../{[dc]fmclient_src_dir} -e../{[dc]fm_api_src_dir} - -e../{[dc]controllerconfig_src_dir} -e../{[dc]cgtsclient_src_dir} setenv = CURRENT_CFG_FILE={toxinidir}/.current.cfg