From 58a7186beae7b10866aecd53fcde5be34321a52c Mon Sep 17 00:00:00 2001 From: Tao Liu Date: Thu, 16 Apr 2020 10:37:32 -0400 Subject: [PATCH] Support subcloud deploy upload the common files Add new REST APIs to upload and display the deploy manager common files on the System Controller. The deploy manager common files which include playbook, overrides and helm charts are uploaded to /opt/platform/deploy/: /opt/platform/deploy//deploy_playbook_ /opt/platform/deploy//deploy_overrides_ /opt/platform/deploy//deploy_chart_ Modify the subcloud post request to accept the bootstrap-values, install-values and deploy-config as file contents. The deploy config file is only used by the deploy manager and it is uploaded to /opt/dc/ansible. The information that used to create the overrides for the playbook are extracted and sent to the dcmanager, which include bootstrap values, install values and the full path of deploy file names, if the deploy-config is presented in the request. Testcases: REST APIs: 1. curl -X POST -H "X-Auth-Token: $TOKEN" $APIURL/subcloud-deploy \ -F deploy_playbook=@ \ -F deploy_overrides=@ \ -F deploy_chart=@full path of the helm chart name> 2. curl -X GET -H "X-Auth-Token: $TOKEN" $APIURL/subcloud-deploy 3. curl -X POST -H "X-Auth-Token: $TOKEN" \ $APIURL/subclouds \ -F bootstrap_values=@ \ -F sysadmin_password= \ -F bootstrap-address= 4. curl -X POST -H "X-Auth-Token: $TOKEN" \ $APIURL/subclouds \ -F bootstrap_values=@ \ -F install_values=@ \ -F deploy_config=@ \ -F sysadmin_password= \ -F bmc_password= \ -F bootstrap-address= \ CLI: 1. dcmanager subcloud-deploy upload \ --deploy-playbook \ --deploy-chart \ --deploy-overrides 2. dcmanager subcloud-deploy show 3. dcmanager subcloud add --bootstrap-address \ --bootstrap-values \ --deploy-config \ 4. dcmanager subcloud add --bootstrap-address \ --bootstrap-values \ --install-values \ 5.dcmanager subcloud add --bootstrap-address \ --bootstrap-values \ --install-values \ --deploy-config \ Host swact and deploy of a subcloud Closes-Bug: 1864508 Change-Id: I3ce0da6efb8c2d78a213647789fc6bdb3b348b2d Signed-off-by: Tao Liu --- api-ref/source/api-ref-dcmanager-v1.rst | 110 ++++++++++++++-- .../dcmanager/api/controllers/v1/root.py | 3 + .../api/controllers/v1/subcloud_deploy.py | 119 ++++++++++++++++++ .../dcmanager/api/controllers/v1/subclouds.py | 93 +++++++++++++- distributedcloud/dcmanager/common/consts.py | 13 ++ distributedcloud/dcmanager/common/utils.py | 7 ++ .../dcmanager/manager/subcloud_manager.py | 41 +++--- .../v1/controllers/test_subcloud_deploy.py | 81 ++++++++++++ .../unit/api/v1/controllers/test_subclouds.py | 61 +++++++-- 9 files changed, 487 insertions(+), 41 deletions(-) create mode 100644 distributedcloud/dcmanager/api/controllers/v1/subcloud_deploy.py create mode 100644 distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_subcloud_deploy.py diff --git a/api-ref/source/api-ref-dcmanager-v1.rst b/api-ref/source/api-ref-dcmanager-v1.rst index 4eec71ce8..af0328e16 100644 --- a/api-ref/source/api-ref-dcmanager-v1.rst +++ b/api-ref/source/api-ref-dcmanager-v1.rst @@ -1,4 +1,4 @@ -==================================================== +==================================================== Dcmanager API v1 ==================================================== @@ -190,6 +190,9 @@ Creates a subcloud .. rest_method:: POST /v1.0/subclouds +Accepts Content-Type multipart/form-data. + + **Normal response codes** 200 @@ -206,13 +209,12 @@ serviceUnavailable (503) :header: "Parameter", "Style", "Type", "Description" :widths: 20, 20, 20, 60 - "name", "plain", "xsd:string", "The name for the subcloud. Must be a unique name." - "description (Optional)", "plain", "xsd:string", "The description of the subcloud." - "location (Optional)", "plain", "xsd:string", "The location of the subcloud." - "management-subnet", "plain", "xsd:string", "Management subnet for subcloud in CIDR format. Must be unique." - "management-start-ip", "plain", "xsd:string", "Start of management IP address range for subcloud." - "management-end-ip", "plain", "xsd:string", "End of management IP address range for subcloud." - "systemcontroller-gateway-ip", "plain", "xsd:string", "Systemcontroller gateway IP Address." + "bootstrap-address", "plain", "xsd:string", "An OAM IP address of the subcloud controller-0." + "sysadmin_password", "plain", "xsd:string", "The sysadmin password of the subcloud. Must be base64 encoded." + "bmc_password (optional)", "plain", "xsd:string", "The BMC password of the subcloud. Must be base64 encoded." + "bootstrap_values", "plain", "xsd:string", "The content of a file containing the bootstrap overrides such as subcloud name, management and OAM subnet." + "install_values (Optional)", "plain", "xsd:string", "The content of a file containing install variables such as subcloud bootstrap interface and BMC information." + "deploy_config (Optional)", "plain", "xsd:string", "The content of a file containing the resource definitions describing the desired subcloud configuration." "group_id", "plain", "xsd:int", "Id of the subcloud group. Defaults to 1" **Response parameters** @@ -1493,5 +1495,97 @@ Delete per subcloud patch options This operation does not accept a request body. +---------------- +Subcloud Deploy +---------------- + +These APIs allow for the display and upload of the deployment manager common +files which include deploy playbook, deploy overrides, and deploy helm charts. +************************** +Show Subcloud Deploy Files +************************** + +.. rest_method:: GET /v1.0/subcloud-deploy + + +**Normal response codes** + +200 + +**Error response codes** + +badRequest (400), unauthorized (401), forbidden +(403), badMethod (405), HTTPUnprocessableEntity (422), +internalServerError (500), serviceUnavailable (503) + +**Response parameters** + +.. csv-table:: + :header: "Parameter", "Style", "Type", "Description" + :widths: 20, 20, 20, 60 + + "subcloud_deploy", "plain", "xsd:dict", "The dictionary of subcloud deploy files." + "deploy_chart", "plain", "xsd:string", "The file name of the deployment manager helm charts." + "deploy_playbook", "plain", "xsd:string", "The file name of the deployment manager playbook." + "deploy_overrides", "plain", "xsd:string", "The file name of the deployment manager overrides." + +:: + + { + "subcloud_deploy": + { + "deploy_chart": "deployment-manager.tgz", + "deploy_playbook": "deployment-manager-playbook.yaml", + "deploy_overrides": "deployment-manager-overrides-subcloud.yaml" + } + } + +This operation does not accept a request body. + +**************************** +Upload Subcloud Deploy Files +**************************** + +.. rest_method:: POST /v1.0/subcloud-deploy + +Accepts Content-Type multipart/form-data. + +**Normal response codes** + +200 + +**Error response codes** + +badRequest (400), unauthorized (401), forbidden (403), badMethod (405), +HTTPUnprocessableEntity (422), internalServerError (500), +serviceUnavailable (503) + +**Request parameters** + +.. csv-table:: + :header: "Parameter", "Style", "Type", "Description" + :widths: 20, 20, 20, 60 + + "deploy_chart", "plain", "xsd:string", "The content of a file containing the deployment manager helm charts." + "deploy_playbook", "plain", "xsd:string", "The content of a file containing the deployment manager playbook." + "deploy_overrides", "plain", "xsd:string", "The content of a file containing the deployment manager overrides." + +**Response parameters** + +.. csv-table:: + :header: "Parameter", "Style", "Type", "Description" + :widths: 20, 20, 20, 60 + + "deploy_chart", "plain", "xsd:string", "The file name of the deployment manager helm charts." + "deploy_playbook", "plain", "xsd:string", "The file name of the deployment manager playbook." + "deploy_overrides", "plain", "xsd:string", "The file name of the deployment manager overrides." + +:: + + { + "deploy_chart": "deployment-manager.tgz", + "deploy_playbook": "deployment-manager-playbook.yaml", + "deploy_overrides": "deployment-manager-overrides-subcloud.yaml" + } diff --git a/distributedcloud/dcmanager/api/controllers/v1/root.py b/distributedcloud/dcmanager/api/controllers/v1/root.py index 4f9682a45..042a7722b 100644 --- a/distributedcloud/dcmanager/api/controllers/v1/root.py +++ b/distributedcloud/dcmanager/api/controllers/v1/root.py @@ -22,6 +22,7 @@ from dcmanager.api.controllers.v1 import alarm_manager +from dcmanager.api.controllers.v1 import subcloud_deploy from dcmanager.api.controllers.v1 import subcloud_group from dcmanager.api.controllers.v1 import subclouds from dcmanager.api.controllers.v1 import sw_update_options @@ -42,6 +43,8 @@ class Controller(object): sub_controllers = dict() if minor_version == '0': sub_controllers["subclouds"] = subclouds.SubcloudsController + sub_controllers["subcloud-deploy"] = subcloud_deploy.\ + SubcloudDeployController sub_controllers["alarms"] = alarm_manager.SubcloudAlarmController sub_controllers["sw-update-strategy"] = \ sw_update_strategy.SwUpdateStrategyController diff --git a/distributedcloud/dcmanager/api/controllers/v1/subcloud_deploy.py b/distributedcloud/dcmanager/api/controllers/v1/subcloud_deploy.py new file mode 100644 index 000000000..c55cd3e79 --- /dev/null +++ b/distributedcloud/dcmanager/api/controllers/v1/subcloud_deploy.py @@ -0,0 +1,119 @@ +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Copyright (c) 2020 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +import os + +from oslo_config import cfg +from oslo_log import log as logging + +import http.client as httpclient +import pecan +from pecan import expose +from pecan import request + +from dcmanager.common import consts +from dcmanager.common.i18n import _ +from dcmanager.common import utils + +import tsconfig.tsconfig as tsc + + +CONF = cfg.CONF +LOG = logging.getLogger(__name__) + +LOCK_NAME = 'SubcloudDeployController' + + +class SubcloudDeployController(object): + + def __init__(self): + super(SubcloudDeployController, self).__init__() + + @staticmethod + def _upload_files(dir_path, file_option, file_item, binary): + prefix = file_option + '_' + # create the version directory if it does not exist + if not os.path.isdir(dir_path): + os.mkdir(dir_path, 0o755) + else: + # check if the file exists, if so remove it + filename = utils.get_filename_by_prefix(dir_path, prefix) + if filename is not None: + os.remove(dir_path + '/' + filename) + + # upload the new file + file_item.file.seek(0, os.SEEK_SET) + contents = file_item.file.read() + fn = os.path.join(dir_path, prefix + os.path.basename( + file_item.filename)) + if binary: + dst = open(fn, 'wb') + dst.write(contents) + else: + dst = os.open(fn, os.O_WRONLY | os.O_CREAT) + os.write(dst, contents) + + @expose(generic=True, template='json') + def index(self): + # Route the request to specific methods with parameters + pass + + @utils.synchronized(LOCK_NAME) + @index.when(method='POST', template='json') + def post(self): + deploy_dicts = dict() + for f in consts.DEPLOY_COMMON_FILE_OPTIONS: + if f not in request.POST: + pecan.abort(httpclient.BAD_REQUEST, + _("Missing required file for %s") % f) + + file_item = request.POST[f] + filename = getattr(file_item, 'filename', '') + if not filename: + pecan.abort(httpclient.BAD_REQUEST, + _("No %s file uploaded" % f)) + + dir_path = tsc.DEPLOY_PATH + binary = False + if f == consts.DEPLOY_CHART: + binary = True + try: + self._upload_files(dir_path, f, file_item, binary) + except Exception as e: + pecan.abort(httpclient.INTERNAL_SERVER_ERROR, + _("Failed to upload %s file: %s" % (f, e))) + deploy_dicts.update({f: filename}) + + return deploy_dicts + + @index.when(method='GET', template='json') + def get(self): + """Get the subcloud deploy files that has been uploaded and stored""" + + deploy_dicts = dict() + for f in consts.DEPLOY_COMMON_FILE_OPTIONS: + dir_path = tsc.DEPLOY_PATH + filename = None + if os.path.isdir(dir_path): + prefix = f + '_' + filename = utils.get_filename_by_prefix(dir_path, prefix) + if filename is not None: + filename = filename.replace(prefix, '', 1) + deploy_dicts.update({f: filename}) + return dict(subcloud_deploy=deploy_dicts) diff --git a/distributedcloud/dcmanager/api/controllers/v1/subclouds.py b/distributedcloud/dcmanager/api/controllers/v1/subclouds.py index 8eff4de9c..3e4cc398a 100644 --- a/distributedcloud/dcmanager/api/controllers/v1/subclouds.py +++ b/distributedcloud/dcmanager/api/controllers/v1/subclouds.py @@ -18,11 +18,13 @@ # SPDX-License-Identifier: Apache-2.0 # +import base64 import keyring from netaddr import AddrFormatError from netaddr import IPAddress from netaddr import IPNetwork from netaddr import IPRange +import os from oslo_config import cfg from oslo_log import log as logging from oslo_messaging import RemoteError @@ -38,6 +40,8 @@ from dccommon import exceptions as dccommon_exceptions from keystoneauth1 import exceptions as keystone_exceptions +import tsconfig.tsconfig as tsc + from dcmanager.api.controllers import restcomm from dcmanager.common import consts from dcmanager.common import exceptions @@ -48,6 +52,7 @@ from dcmanager.db import api as db_api from dcmanager.rpc import client as rpc_client + CONF = cfg.CONF LOG = logging.getLogger(__name__) # System mode @@ -57,6 +62,18 @@ SYSTEM_MODE_DUPLEX_DIRECT = "duplex-direct" LOCK_NAME = 'SubcloudsController' +BOOTSTRAP_VALUES = 'bootstrap_values' +INSTALL_VALUES = 'install_values' + +SUBCLOUD_ADD_MANDATORY_FILE = [ + BOOTSTRAP_VALUES, +] + +SUBCLOUD_ADD_GET_FILE_CONTENTS = [ + BOOTSTRAP_VALUES, + INSTALL_VALUES, +] + class SubcloudsController(object): VERSION_ALIASES = { @@ -85,6 +102,57 @@ class SubcloudsController(object): LOG.exception(e) pecan.abort(400, _("Invalid group_id")) + @staticmethod + def _get_common_deploy_files(payload): + for f in consts.DEPLOY_COMMON_FILE_OPTIONS: + dir_path = tsc.DEPLOY_PATH + filename = utils.get_filename_by_prefix(dir_path, f + '_') + if filename is None: + pecan.abort(400, _("Missing required deploy file for %s") % f) + payload.update({f: os.path.join(dir_path, filename)}) + + def _upload_deploy_config_file(self, request, payload): + if consts.DEPLOY_CONFIG in request.POST: + file_item = request.POST[consts.DEPLOY_CONFIG] + filename = getattr(file_item, 'filename', '') + if not filename: + pecan.abort(400, _("No %s file uploaded" % + consts.DEPLOY_CONFIG)) + file_item.file.seek(0, os.SEEK_SET) + contents = file_item.file.read() + # the deploy config needs to upload to the override location + fn = os.path.join(consts.ANSIBLE_OVERRIDES_PATH, payload['name'] + + '_' + os.path.basename(filename)) + try: + dst = os.open(fn, os.O_WRONLY | os.O_CREAT) + os.write(dst, contents) + except Exception: + msg = _("Failed to upload %s file" % consts.DEPLOY_CONFIG) + LOG.exception(msg) + pecan.abort(400, msg) + payload.update({consts.DEPLOY_CONFIG: fn}) + self._get_common_deploy_files(payload) + + @staticmethod + def _get_request_data(request): + payload = dict() + for f in SUBCLOUD_ADD_MANDATORY_FILE: + if f not in request.POST: + pecan.abort(400, _("Missing required file for %s") % f) + + for f in SUBCLOUD_ADD_GET_FILE_CONTENTS: + if f in request.POST: + file_item = request.POST[f] + file_item.file.seek(0, os.SEEK_SET) + data = yaml.safe_load(file_item.file.read().decode('utf8')) + if f == BOOTSTRAP_VALUES: + payload.update(data) + else: + payload.update({f: data}) + del request.POST[f] + payload.update(request.POST) + return payload + def _validate_subcloud_config(self, context, name, @@ -242,6 +310,13 @@ class SubcloudsController(object): bmc_password = payload.get('bmc_password') if not bmc_password: pecan.abort(400, _('subcloud bmc_password required')) + try: + bmc_password = base64.b64decode(bmc_password).decode('utf-8') + except Exception: + msg = _('Failed to decode subcloud bmc_password, verify' + ' the password is base64 encoded') + LOG.exception(msg) + pecan.abort(400, msg) payload['install_values'].update({'bmc_password': bmc_password}) for k in install_consts.MANDATORY_INSTALL_VALUES: @@ -500,13 +575,16 @@ class SubcloudsController(object): context = restcomm.extract_context_from_environ() if subcloud_ref is None: - payload = yaml.safe_load(request.body) + + payload = self._get_request_data(request) if not payload: pecan.abort(400, _('Body required')) + name = payload.get('name') if not name: pecan.abort(400, _('name required')) + system_mode = payload.get('system_mode') if not system_mode: pecan.abort(400, _('system_mode required')) @@ -551,6 +629,14 @@ class SubcloudsController(object): payload.get('sysadmin_password') if not sysadmin_password: pecan.abort(400, _('subcloud sysadmin_password required')) + try: + payload['sysadmin_password'] = base64.b64decode( + sysadmin_password).decode('utf-8') + except Exception: + msg = _('Failed to decode subcloud sysadmin_password, ' + 'verify the password is base64 encoded') + LOG.exception(msg) + pecan.abort(400, msg) # If a subcloud group is not passed, use the default group_id = payload.get('group_id', @@ -571,6 +657,11 @@ class SubcloudsController(object): if 'install_values' in payload: self._validate_install_values(payload) + # Upload the deploy config files if it is included in the request + # It has a dependency on the subcloud name, and it is called after + # the name has been validated + self._upload_deploy_config_file(request, payload) + try: # Ask dcmanager-manager to add the subcloud. # It will do all the real work... diff --git a/distributedcloud/dcmanager/common/consts.py b/distributedcloud/dcmanager/common/consts.py index c0f4f4c14..b427412bb 100644 --- a/distributedcloud/dcmanager/common/consts.py +++ b/distributedcloud/dcmanager/common/consts.py @@ -111,3 +111,16 @@ ALARMS_DISABLED = "disabled" ALARM_OK_STATUS = "OK" ALARM_DEGRADED_STATUS = "degraded" ALARM_CRITICAL_STATUS = "critical" + +# subcloud deploy file options +ANSIBLE_OVERRIDES_PATH = '/opt/dc/ansible' +DEPLOY_PLAYBOOK = "deploy_playbook" +DEPLOY_OVERRIDES = "deploy_overrides" +DEPLOY_CHART = "deploy_chart" +DEPLOY_CONFIG = 'deploy_config' + +DEPLOY_COMMON_FILE_OPTIONS = [ + DEPLOY_PLAYBOOK, + DEPLOY_OVERRIDES, + DEPLOY_CHART +] diff --git a/distributedcloud/dcmanager/common/utils.py b/distributedcloud/dcmanager/common/utils.py index 37c31bed0..0175e9fb4 100644 --- a/distributedcloud/dcmanager/common/utils.py +++ b/distributedcloud/dcmanager/common/utils.py @@ -200,3 +200,10 @@ def synchronized(name, external=True, fair=False): return lockutils.synchronized(name, lock_file_prefix=prefix, external=external, lock_path=lock_path, semaphores=None, delay=0.01, fair=fair) + + +def get_filename_by_prefix(dir_path, prefix): + for filename in os.listdir(dir_path): + if filename.startswith(prefix): + return filename + return None diff --git a/distributedcloud/dcmanager/manager/subcloud_manager.py b/distributedcloud/dcmanager/manager/subcloud_manager.py index 0f04a9dd5..3f2eea14f 100644 --- a/distributedcloud/dcmanager/manager/subcloud_manager.py +++ b/distributedcloud/dcmanager/manager/subcloud_manager.py @@ -60,7 +60,6 @@ LOG = logging.getLogger(__name__) ADDN_HOSTS_DC = 'dnsmasq.addn_hosts_dc' # Subcloud configuration paths -ANSIBLE_OVERRIDES_PATH = '/opt/dc/ansible' INVENTORY_FILE_POSTFIX = '_inventory.yml' ANSIBLE_SUBCLOUD_PLAYBOOK = \ '/usr/share/ansible/stx-ansible/playbooks/bootstrap.yml' @@ -160,7 +159,7 @@ class SubcloudManager(manager.Manager): try: # Ansible inventory filename for the specified subcloud ansible_subcloud_inventory_file = os.path.join( - ANSIBLE_OVERRIDES_PATH, + consts.ANSIBLE_OVERRIDES_PATH, subcloud.name + INVENTORY_FILE_POSTFIX) # Create a new route to this subcloud on the management interface @@ -277,12 +276,19 @@ class SubcloudManager(manager.Manager): payload['sysadmin_password'] if "deploy_playbook" in payload: + payload['deploy_values'] = dict() payload['deploy_values']['ansible_become_pass'] = \ payload['sysadmin_password'] payload['deploy_values']['ansible_ssh_pass'] = \ payload['sysadmin_password'] payload['deploy_values']['admin_password'] = \ str(keyring.get_password('CGCS', 'admin')) + payload['deploy_values']['deployment_config'] = \ + payload[consts.DEPLOY_CONFIG] + payload['deploy_values']['deployment_manager_chart'] = \ + payload[consts.DEPLOY_CHART] + payload['deploy_values']['deployment_manager_overrides'] = \ + payload[consts.DEPLOY_OVERRIDES] del payload['sysadmin_password'] @@ -309,7 +315,7 @@ class SubcloudManager(manager.Manager): "ansible-playbook", ANSIBLE_SUBCLOUD_INSTALL_PLAYBOOK, "-i", ansible_subcloud_inventory_file, "--limit", subcloud.name, - "-e", "@%s" % ANSIBLE_OVERRIDES_PATH + "/" + + "-e", "@%s" % consts.ANSIBLE_OVERRIDES_PATH + "/" + payload['name'] + '/' + "install_values.yml" ] @@ -323,17 +329,15 @@ class SubcloudManager(manager.Manager): # which overrides to load apply_command += [ "-e", str("override_files_dir='%s' region_name=%s") % ( - ANSIBLE_OVERRIDES_PATH, subcloud.name)] + consts.ANSIBLE_OVERRIDES_PATH, subcloud.name)] deploy_command = None if "deploy_playbook" in payload: deploy_command = [ - "ansible-playbook", ANSIBLE_OVERRIDES_PATH + '/' + - payload['name'] + "_deploy.yml", - "-e", "@%s" % ANSIBLE_OVERRIDES_PATH + "/" + + "ansible-playbook", payload[consts.DEPLOY_PLAYBOOK], + "-e", "@%s" % consts.ANSIBLE_OVERRIDES_PATH + "/" + payload['name'] + "_deploy_values.yml", - "-i", - ansible_subcloud_inventory_file, + "-i", ansible_subcloud_inventory_file, "--limit", subcloud.name ] @@ -365,7 +369,8 @@ class SubcloudManager(manager.Manager): deploy_status=consts.DEPLOY_STATE_PRE_INSTALL) try: install = SubcloudInstall(context, subcloud.name) - install.prep(ANSIBLE_OVERRIDES_PATH, payload['install_values']) + install.prep(consts.ANSIBLE_OVERRIDES_PATH, + payload['install_values']) except Exception as e: LOG.exception(e) db_api.subcloud_update( @@ -513,7 +518,7 @@ class SubcloudManager(manager.Manager): def _write_subcloud_ansible_config(self, context, payload): """Create the override file for usage with the specified subcloud""" - overrides_file = os.path.join(ANSIBLE_OVERRIDES_PATH, + overrides_file = os.path.join(consts.ANSIBLE_OVERRIDES_PATH, payload['name'] + '.yml') m_ks_client = KeystoneClient() @@ -543,19 +548,17 @@ class SubcloudManager(manager.Manager): for k, v in payload.items(): if k not in ['deploy_playbook', 'deploy_values', - 'install_values']: + 'deploy_config', 'deploy_chart', + 'deploy_overrides', 'install_values']: f_out_overrides_file.write("%s: %s\n" % (k, json.dumps(v))) def _write_deploy_files(self, payload): - """Create the deploy playbook and value files for the subcloud""" + """Create the deploy value files for the subcloud""" - deploy_playbook_file = os.path.join( - ANSIBLE_OVERRIDES_PATH, payload['name'] + '_deploy.yml') deploy_values_file = os.path.join( - ANSIBLE_OVERRIDES_PATH, payload['name'] + '_deploy_values.yml') + consts.ANSIBLE_OVERRIDES_PATH, payload['name'] + + '_deploy_values.yml') - with open(deploy_playbook_file, 'w') as f_out_deploy_playbook_file: - json.dump(payload['deploy_playbook'], f_out_deploy_playbook_file) with open(deploy_values_file, 'w') as f_out_deploy_values_file: json.dump(payload['deploy_values'], f_out_deploy_values_file) @@ -650,7 +653,7 @@ class SubcloudManager(manager.Manager): # Ansible inventory filename for the specified subcloud ansible_subcloud_inventory_file = os.path.join( - ANSIBLE_OVERRIDES_PATH, + consts.ANSIBLE_OVERRIDES_PATH, subcloud.name + INVENTORY_FILE_POSTFIX) self._remove_subcloud_details(context, diff --git a/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_subcloud_deploy.py b/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_subcloud_deploy.py new file mode 100644 index 000000000..73b713d03 --- /dev/null +++ b/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_subcloud_deploy.py @@ -0,0 +1,81 @@ +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Copyright (c) 2020 Wind River Systems, Inc. +# +# The right to copy, distribute, modify, or otherwise make use +# of this software may be licensed only pursuant to the terms +# of an applicable Wind River license agreement. +# + +import mock +from six.moves import http_client + +from dcmanager.api.controllers.v1 import subcloud_deploy +from dcmanager.common import consts +from dcmanager.tests.unit.api import test_root_controller as testroot +from dcmanager.tests import utils + +FAKE_TENANT = utils.UUID1 +FAKE_ID = '1' +FAKE_URL = '/v1.0/subcloud-deploy' +FAKE_HEADERS = {'X-Tenant-Id': FAKE_TENANT, 'X_ROLE': 'admin', + 'X-Identity-Status': 'Confirmed'} + + +class TestSubcloudDeploy(testroot.DCManagerApiTest): + def setUp(self): + super(TestSubcloudDeploy, self).setUp() + self.ctx = utils.dummy_context() + + @mock.patch.object(subcloud_deploy.SubcloudDeployController, + '_upload_files') + def test_post_subcloud_deploy(self, mock_upload_files): + fields = list() + for opt in consts.DEPLOY_COMMON_FILE_OPTIONS: + fake_name = opt + "_fake" + fields.append((opt, fake_name, "fake content")) + mock_upload_files.return_value = True + response = self.app.post(FAKE_URL, + headers=FAKE_HEADERS, + upload_files=fields) + self.assertEqual(response.status_code, http_client.OK) + + @mock.patch.object(subcloud_deploy.SubcloudDeployController, + '_upload_files') + def test_post_subcloud_deploy_missing_file(self, mock_upload_files): + opts = [consts.DEPLOY_PLAYBOOK, consts.DEPLOY_OVERRIDES] + fields = list() + for opt in opts: + fake_name = opt + "_fake" + fields.append((opt, fake_name, "fake content")) + mock_upload_files.return_value = True + response = self.app.post(FAKE_URL, + headers=FAKE_HEADERS, + upload_files=fields, + expect_errors=True) + self.assertEqual(response.status_code, http_client.BAD_REQUEST) + + @mock.patch.object(subcloud_deploy.SubcloudDeployController, + '_upload_files') + def test_post_subcloud_deploy_missing_file_name(self, mock_upload_files): + fields = list() + for opt in consts.DEPLOY_COMMON_FILE_OPTIONS: + fields.append((opt, "", "fake content")) + mock_upload_files.return_value = True + response = self.app.post(FAKE_URL, + headers=FAKE_HEADERS, + upload_files=fields, + expect_errors=True) + self.assertEqual(response.status_code, http_client.BAD_REQUEST) diff --git a/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_subclouds.py b/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_subclouds.py index b138ac3a8..8444580ec 100644 --- a/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_subclouds.py +++ b/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_subclouds.py @@ -13,13 +13,14 @@ # License for the specific language governing permissions and limitations # under the License. # -# Copyright (c) 2017 Wind River Systems, Inc. +# Copyright (c) 2017-2020 Wind River Systems, Inc. # # The right to copy, distribute, modify, or otherwise make use # of this software may be licensed only pursuant to the terms # of an applicable Wind River license agreement. # +import base64 import copy import mock import six @@ -51,8 +52,7 @@ FAKE_SUBCLOUD_DATA = {"name": "subcloud1", "external_oam_subnet": "10.10.10.0/24", "external_oam_gateway_address": "10.10.10.1", "external_oam_floating_address": "10.10.10.12", - "availability-status": "disabled", - "sysadmin_password": "testpass"} + "availability-status": "disabled"} FAKE_SUBCLOUD_INSTALL_VALUES = { "image": "http://192.168.101.2:8080/iso/bootimage.iso", @@ -72,6 +72,11 @@ FAKE_SUBCLOUD_INSTALL_VALUES = { "boot_device": "/dev/disk/by-path/pci-0000:5c:00.0-scsi-0:1:0:0" } +FAKE_BOOTSTRAP_VALUE = { + 'bootstrap-address': '10.10.10.12', + 'sysadmin_password': base64.b64encode('testpass'.encode("utf-8")) +} + class FakeAddressPool(object): def __init__(self, pool_network, pool_prefix, pool_start, pool_end): @@ -103,45 +108,75 @@ class TestSubclouds(testroot.DCManagerApiTest): super(TestSubclouds, self).setUp() self.ctx = utils.dummy_context() + @mock.patch.object(subclouds.SubcloudsController, + '_upload_deploy_config_file') + @mock.patch.object(subclouds.SubcloudsController, + '_get_request_data') @mock.patch.object(subclouds.SubcloudsController, '_get_management_address_pool') @mock.patch.object(rpc_client, 'ManagerClient') @mock.patch.object(subclouds, 'db_api') def test_post_subcloud(self, mock_db_api, mock_rpc_client, - mock_get_management_address_pool): - data = FAKE_SUBCLOUD_DATA + mock_get_management_address_pool, + mock_get_request_data, + mock_upload_deploy_config_file): management_address_pool = FakeAddressPool('192.168.204.0', 24, '192.168.204.2', '192.168.204.100') mock_get_management_address_pool.return_value = management_address_pool mock_rpc_client().add_subcloud.return_value = True - response = self.app.post_json(FAKE_URL, - headers=FAKE_HEADERS, - params=data) + fields = list() + for f in subclouds.SUBCLOUD_ADD_MANDATORY_FILE: + fake_name = f + "_fake" + fields.append((f, fake_name, "fake content")) + data = copy.copy(FAKE_SUBCLOUD_DATA) + data.update(FAKE_BOOTSTRAP_VALUE) + mock_get_request_data.return_value = data + mock_upload_deploy_config_file.return_value = True + response = self.app.post(FAKE_URL, + headers=FAKE_HEADERS, + params=FAKE_BOOTSTRAP_VALUE, + upload_files=fields) mock_rpc_client().add_subcloud.assert_called_once_with( mock.ANY, data) self.assertEqual(response.status_int, 200) + @mock.patch.object(subclouds.SubcloudsController, + '_upload_deploy_config_file') + @mock.patch.object(subclouds.SubcloudsController, + '_get_request_data') @mock.patch.object(subclouds.SubcloudsController, '_get_management_address_pool') @mock.patch.object(rpc_client, 'ManagerClient') @mock.patch.object(subclouds, 'db_api') def test_post_subcloud_with_install_values( self, mock_db_api, mock_rpc_client, - mock_get_management_address_pool): + mock_get_management_address_pool, + mock_get_request_data, + mock_upload_deploy_config_file): data = copy.copy(FAKE_SUBCLOUD_DATA) install_data = copy.copy(FAKE_SUBCLOUD_INSTALL_VALUES) - data['bmc_password'] = 'bmc_password' data.update({'install_values': install_data}) management_address_pool = FakeAddressPool('192.168.204.0', 24, '192.168.204.2', '192.168.204.100') mock_get_management_address_pool.return_value = management_address_pool mock_rpc_client().add_subcloud.return_value = True - response = self.app.post_json(FAKE_URL, - headers=FAKE_HEADERS, - params=data) + fields = list() + for f in subclouds.SUBCLOUD_ADD_GET_FILE_CONTENTS: + fake_name = f + "_fake" + fields.append((f, fake_name, "fake content")) + params = copy.copy(FAKE_BOOTSTRAP_VALUE) + params.update({'bmc_password': + base64.b64encode('bmc_password'.encode("utf-8"))}) + data.update(params) + mock_get_request_data.return_value = data + mock_upload_deploy_config_file.return_value = True + response = self.app.post(FAKE_URL, + headers=FAKE_HEADERS, + params=params, + upload_files=fields) self.assertEqual(response.status_int, 200) @mock.patch.object(subclouds.SubcloudsController,