From adc8e48ac9d29dd76f119df87fe84265cdc73a95 Mon Sep 17 00:00:00 2001 From: Victor Romano Date: Wed, 14 Jun 2023 11:42:39 -0300 Subject: [PATCH] Add subcloud deploy resume option to dcmanager This commit adds the command "subcloud deploy resume" to dcmanager. It will resume subcloud deployment based on current subcloud deploy state. All parameters except sysadmin-password are optional if they were already provided in previous phases. Since install and config are both optional phases, they will only be executed if respective parameters are/have been provided. Test Plan: These options will be referenced on the test cases as the values already present on the system controller before the resume operation or the values passed to it's command: [1] All values (install_values, bootstrap_values, deploy_config) [2] Only install_values and bootstrap_values [3] Only bootstrap_values [4] Only deploy_config Success cases: - PASS: Resume from create-complete previously having [1] without passing any new parameter and verify that the subcloud's deploy state is 'complete'. - PASS: Resume from create-complete previously having [2] without passing any new parameter and verify that the subcloud's deploy state is 'bootstrap-complete'. - PASS: Resume from create-complete previously having [3] without passing any new parameter and manually installing the subcloud and verify that it's deploy state is 'bootstrap-complete'. - PASS: Resume from create-complete previously having [1] passing a previous release (21.12) and verify that the subcloud's deploy state is 'complete' and the installed load is correct. - PASS: Resume from install-complete previously having [1] without passing any new parameter and verify that the subcloud's deploy state is 'complete'. - PASS: Resume from install-complete previously having [2] without passing any new parameter and verify that the subcloud's deploy state is 'bootstrap-complete'. - PASS: Resume from install-complete previously having [3] without passing any new parameter and verify that it's deploy state is 'bootstrap-complete'. - PASS: Resume from install-failed previously having [1] passing new install_values and verify that the subcloud's deploy state is 'complete' and the installation used new values. - PASS: Resume from install-failed previously having [1] without passing any new parameter and verify that the subcloud's deploy state is 'complete'. - PASS: Resume from install-aborted previously having [1] without passing any new parameter and verify that the subcloud's deploy state is 'complete'. - PASS: Resume from bootstrap-complete previously having [1] without passing any new parameter and verify that the subcloud's deploy state is 'complete'. - PASS: Resume from bootstrap-complete previously having [3] and passing [4] and verify that the subcloud's deploy state is 'complete'. - PASS: Resume from bootstrap-failed previously having [1] passing new bootstrap_values and verify that the subcloud's deploy state is 'complete' and the bootstrap used new values. - PASS: Resume from bootstrap-failed previously having [1] without passing any new parameter and verify that the subcloud's deploy state is 'complete'. - PASS: Resume from bootstrap-aborted previously having [1] without passing any new parameter and verify that the subcloud's deploy state is 'complete'. - PASS: Resume from config-failed previously having [1] passing new deploy_config file and verify that the subcloud's deploy state is 'complete' and the config used new values. - PASS: Resume from config-failed previously having [1] without passing any new parameter and verify that the subcloud's deploy state is 'complete'. - PASS: Resume from config-aborted previously having [1] without passing any new parameter and verify that the subcloud's deploy state is 'complete'. - PASS: Repeat previous tests but directly call the API (using CURL) instead of using the CLI. Failure cases: - PASS: Verify that it's not possible to resume deployment if the deploy state is not one of the following: - create-complete - install-complete - pre-install-failed - install-failed - install-aborted - bootstrap-complete - pre-bootstrap-failed - bootstrap-failed - bootstrap-aborted - pre-config-failed - config-failed - config-aborted - PASS: Call the API directly, passing bmc-password and/or sysadmin-password as plain text as opposed to b64encoded and verify that the response contains the correct error code and message. - PASS: Resume from bootstrap-complete previously having [2] and verify that the system alerts that the only remaining phase is config and there's no deploy-config file available Story: 2010756 Task: 48316 Change-Id: I81c0a226b3ede56628e21372b02748013c3f6b35 Signed-off-by: Victor Romano --- api-ref/source/api-ref-dcmanager-v1.rst | 72 +++++ ...-subcloud-deploy-patch-resume-request.json | 10 + ...subcloud-deploy-patch-resume-response.json | 24 ++ .../controllers/v1/phased_subcloud_deploy.py | 186 ++++++++++--- .../api/policies/phased_subcloud_deploy.py | 4 + distributedcloud/dcmanager/common/consts.py | 8 + .../common/phased_subcloud_deploy.py | 38 ++- distributedcloud/dcmanager/common/utils.py | 12 + distributedcloud/dcmanager/manager/service.py | 11 + .../dcmanager/manager/subcloud_manager.py | 259 +++++++++++------- distributedcloud/dcmanager/rpc/client.py | 8 + .../test_phased_subcloud_deploy.py | 190 ++++++++++++- .../unit/manager/test_subcloud_manager.py | 80 +++++- 13 files changed, 743 insertions(+), 159 deletions(-) create mode 100644 api-ref/source/samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-resume-request.json create mode 100644 api-ref/source/samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-resume-response.json diff --git a/api-ref/source/api-ref-dcmanager-v1.rst b/api-ref/source/api-ref-dcmanager-v1.rst index 36fb77ce8..b959fc588 100644 --- a/api-ref/source/api-ref-dcmanager-v1.rst +++ b/api-ref/source/api-ref-dcmanager-v1.rst @@ -2120,4 +2120,76 @@ Response Example ---------------- .. literalinclude:: samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-abort-response.json + :language: json + + +**************************** +Resume subcloud deployment +**************************** + +.. rest_method:: POST /v1.0/phased-subcloud-deploy + +Accepts Content-Type multipart/form-data. + + +**Normal response codes** + +200 + +**Error response codes** + +badRequest (400), unauthorized (401), forbidden (403), badMethod (405), +conflict (409), HTTPUnprocessableEntity (422), internalServerError (500), +serviceUnavailable (503) + +**Request parameters** + +.. rest_parameters:: parameters.yaml + + - bmc_password: bmc_password + - bootstrap-address: bootstrap_address + - bootstrap_values: bootstrap_values + - deploy_config: deploy_config + - install_values: install_values + - release: release + - sysadmin_password: sysadmin_password + +Request Example +---------------- + +.. literalinclude:: samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-resume-request.json + :language: json + + +**Response parameters** + +.. rest_parameters:: parameters.yaml + + - id: subcloud_id + - name: subcloud_name + - description: subcloud_description + - location: subcloud_location + - software-version: software_version + - management-state: management_state + - availability-status: availability_status + - deploy-status: deploy_status + - backup-status: backup_status + - backup-datetime: backup_datetime + - error-description: error_description + - management-subnet: management_subnet + - management-start-ip: management_start_ip + - management-end-ip: management_end_ip + - management-gateway-ip: management_gateway_ip + - openstack-installed: openstack_installed + - systemcontroller-gateway-ip: systemcontroller_gateway_ip + - data_install: data_install + - data_upgrade: data_upgrade + - created-at: created_at + - updated-at: updated_at + - group_id: group_id + +Response Example +---------------- + +.. literalinclude:: samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-resume-response.json :language: json \ No newline at end of file diff --git a/api-ref/source/samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-resume-request.json b/api-ref/source/samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-resume-request.json new file mode 100644 index 000000000..c4dd86591 --- /dev/null +++ b/api-ref/source/samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-resume-request.json @@ -0,0 +1,10 @@ +{ + "bmc_password": "YYYYYYY", + "bootstrap-address": "10.10.10.12", + "bootstrap_values": "content of bootstrap_values file", + "deploy_config": "content of deploy_config file", + "install_values": "content of install_values file", + "location": "Somewhere", + "release": "22.12", + "sysadmin_password": "XXXXXXX" +} \ No newline at end of file diff --git a/api-ref/source/samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-resume-response.json b/api-ref/source/samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-resume-response.json new file mode 100644 index 000000000..d976dcaf3 --- /dev/null +++ b/api-ref/source/samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-resume-response.json @@ -0,0 +1,24 @@ +{ + "id": 1, + "name": "subcloud1", + "description": "Subcloud 1", + "location": "Somewhere", + "software-version": "22.12", + "management-state": "unmanaged", + "availability-status": "offline", + "deploy-status": "pre-install", + "backup-status": null, + "backup-datetime": null, + "error-description": "No errors present", + "management-subnet": "192.168.102.0/24", + "management-start-ip": "192.168.102.2", + "management-end-ip": "192.168.102.50", + "management-gateway-ip": "192.168.102.1", + "openstack-installed": null, + "systemcontroller-gateway-ip": "192.168.204.101", + "data_install": null, + "data_upgrade": null, + "created-at": "2023-05-15 20: 58: 22.992609", + "updated-at": null, + "group_id": 1 +} \ No newline at end of file diff --git a/distributedcloud/dcmanager/api/controllers/v1/phased_subcloud_deploy.py b/distributedcloud/dcmanager/api/controllers/v1/phased_subcloud_deploy.py index 6d7dbaecf..d3d4a75b2 100644 --- a/distributedcloud/dcmanager/api/controllers/v1/phased_subcloud_deploy.py +++ b/distributedcloud/dcmanager/api/controllers/v1/phased_subcloud_deploy.py @@ -5,7 +5,6 @@ # import http.client as httpclient -import json import os from oslo_log import log as logging @@ -14,7 +13,6 @@ import pecan import tsconfig.tsconfig as tsc import yaml -from dccommon import consts as dccommon_consts from dcmanager.api.controllers import restcomm from dcmanager.api.policies import phased_subcloud_deploy as \ phased_subcloud_deploy_policy @@ -33,6 +31,12 @@ from dcmanager.rpc import client as rpc_client LOG = logging.getLogger(__name__) LOCK_NAME = 'PhasedSubcloudDeployController' +INSTALL = consts.DEPLOY_PHASE_INSTALL +BOOTSTRAP = consts.DEPLOY_PHASE_BOOTSTRAP +CONFIG = consts.DEPLOY_PHASE_CONFIG +ABORT = consts.DEPLOY_PHASE_ABORT +RESUME = consts.DEPLOY_PHASE_RESUME + SUBCLOUD_CREATE_REQUIRED_PARAMETERS = ( consts.BOOTSTRAP_VALUES, consts.BOOTSTRAP_ADDRESS @@ -93,6 +97,46 @@ VALID_STATES_FOR_DEPLOY_ABORT = ( consts.DEPLOY_STATE_CONFIGURING ) +FILES_FOR_RESUME_INSTALL = \ + SUBCLOUD_INSTALL_GET_FILE_CONTENTS + \ + SUBCLOUD_BOOTSTRAP_GET_FILE_CONTENTS + \ + SUBCLOUD_CONFIG_GET_FILE_CONTENTS + + +FILES_FOR_RESUME_BOOTSTRAP = \ + SUBCLOUD_BOOTSTRAP_GET_FILE_CONTENTS + \ + SUBCLOUD_CONFIG_GET_FILE_CONTENTS + + +FILES_FOR_RESUME_CONFIG = SUBCLOUD_CONFIG_GET_FILE_CONTENTS + +RESUMABLE_STATES = { + consts.DEPLOY_STATE_CREATED: [INSTALL, BOOTSTRAP, CONFIG], + consts.DEPLOY_STATE_INSTALLED: [BOOTSTRAP, CONFIG], + consts.DEPLOY_STATE_PRE_INSTALL_FAILED: [INSTALL, BOOTSTRAP, CONFIG], + consts.DEPLOY_STATE_INSTALL_FAILED: [INSTALL, BOOTSTRAP, CONFIG], + consts.DEPLOY_STATE_INSTALL_ABORTED: [INSTALL, BOOTSTRAP, CONFIG], + consts.DEPLOY_STATE_BOOTSTRAPPED: [CONFIG], + consts.DEPLOY_STATE_PRE_BOOTSTRAP_FAILED: [BOOTSTRAP, CONFIG], + consts.DEPLOY_STATE_BOOTSTRAP_FAILED: [BOOTSTRAP, CONFIG], + consts.DEPLOY_STATE_BOOTSTRAP_ABORTED: [BOOTSTRAP, CONFIG], + consts.DEPLOY_STATE_PRE_CONFIG_FAILED: [CONFIG], + consts.DEPLOY_STATE_CONFIG_FAILED: [CONFIG], + consts.DEPLOY_STATE_CONFIG_ABORTED: [CONFIG] +} + +FILES_MAPPING = { + INSTALL: SUBCLOUD_INSTALL_GET_FILE_CONTENTS, + BOOTSTRAP: SUBCLOUD_BOOTSTRAP_GET_FILE_CONTENTS, + CONFIG: SUBCLOUD_CONFIG_GET_FILE_CONTENTS +} + +RESUME_PREP_UPDATE_STATUS = { + INSTALL: consts.DEPLOY_STATE_PRE_INSTALL, + BOOTSTRAP: consts.DEPLOY_STATE_PRE_BOOTSTRAP, + CONFIG: consts.DEPLOY_STATE_PRE_CONFIG +} + def get_create_payload(request: pecan.Request) -> dict: payload = dict() @@ -181,30 +225,23 @@ class PhasedSubcloudDeployController(object): pecan.abort(400, _('Subcloud deploy status must be either: %s') % allowed_states_str) - payload['software_version'] = payload.get('release', tsc.SW_VERSION) + payload['software_version'] = payload.get('release', subcloud.software_version) psd_common.populate_payload_with_pre_existing_data( payload, subcloud, SUBCLOUD_INSTALL_GET_FILE_CONTENTS) - psd_common.validate_sysadmin_password(payload) psd_common.pre_deploy_install(payload, subcloud) try: # Align the software version of the subcloud with install # version. Update the deploy status as pre-install. - subcloud = db_api.subcloud_update( - context, - subcloud.id, - description=payload.get('description', subcloud.description), - location=payload.get('location', subcloud.location), - software_version=payload['software_version'], - management_state=dccommon_consts.MANAGEMENT_UNMANAGED, - deploy_status=consts.DEPLOY_STATE_PRE_INSTALL, - data_install=json.dumps(payload['install_values'])) self.dcmanager_rpc_client.subcloud_deploy_install( context, subcloud.id, payload) + subcloud_dict = db_api.subcloud_db_model_to_dict(subcloud) + subcloud_dict['deploy-status'] = consts.DEPLOY_STATE_PRE_INSTALL + subcloud_dict['software-version'] = payload['software_version'] - return db_api.subcloud_db_model_to_dict(subcloud) + return subcloud_dict except RemoteError as e: pecan.abort(422, e.value) except Exception: @@ -238,31 +275,8 @@ class PhasedSubcloudDeployController(object): # Update the existing values with new ones from the request payload.update(request_data) - psd_common.validate_sysadmin_password(payload) - - if has_bootstrap_values: - # Need to validate the new values - playload_name = payload.get('name') - if playload_name != subcloud.name: - pecan.abort(400, _('The bootstrap-values "name" value (%s) ' - 'must match the current subcloud name (%s)' % - (playload_name, subcloud.name))) - - # Verify if payload contains all required bootstrap values - psd_common.validate_bootstrap_values(payload) - - # It's ok for the management subnet to conflict with itself since we - # are only going to update it if it was modified, conflicts with - # other subclouds are still verified. - psd_common.validate_subcloud_config(context, payload, - ignore_conflicts_with=subcloud) - psd_common.format_ip_address(payload) - - # Patch status and fresh_install_k8s_version may have been changed - # between deploy create and deploy bootstrap commands. Validate them - # again: - psd_common.validate_system_controller_patch_status("bootstrap") - psd_common.validate_k8s_version(payload) + psd_common.pre_deploy_bootstrap(context, payload, subcloud, + has_bootstrap_values) try: # Ask dcmanager-manager to bootstrap the subcloud. @@ -329,6 +343,90 @@ class PhasedSubcloudDeployController(object): LOG.exception("Unable to abort subcloud %s deployment" % subcloud.name) pecan.abort(500, _('Unable to abort subcloud deploy')) + def _deploy_resume(self, context: RequestContext, + request: pecan.Request, subcloud): + + if subcloud.deploy_status not in RESUMABLE_STATES: + allowed_states_str = ', '.join(RESUMABLE_STATES) + pecan.abort(400, _('Subcloud deploy status must be either: %s') + % allowed_states_str) + + # Since both install and config are optional phases, + # it's necessary to check if they should be executed + config_file = psd_common.get_config_file_path(subcloud.name, + consts.DEPLOY_CONFIG) + has_original_install_values = subcloud.data_install + has_original_config_values = os.path.exists(config_file) + has_new_install_values = consts.INSTALL_VALUES in request.POST + has_new_config_values = consts.DEPLOY_CONFIG in request.POST + has_bootstrap_values = consts.BOOTSTRAP_VALUES in request.POST + has_config_values = has_original_config_values or has_new_config_values + has_install_values = has_original_install_values or has_new_install_values + + deploy_states_to_run = RESUMABLE_STATES[subcloud.deploy_status] + if deploy_states_to_run == [CONFIG] and not has_config_values: + msg = _("Only deploy phase left is deploy config. " + "Required %s file was not provided and it was not " + "previously available.") % consts.DEPLOY_CONFIG + pecan.abort(400, msg) + + # Since the subcloud can be installed manually and the config is optional, + # skip those phases if the user doesn't provide the install or config values + # and they are not available from previous executions. + files_for_resume = [] + for state in deploy_states_to_run: + if state == INSTALL and not has_install_values: + deploy_states_to_run.remove(state) + elif state == CONFIG and not has_config_values: + deploy_states_to_run.remove(state) + else: + files_for_resume.extend(FILES_MAPPING[state]) + + payload = psd_common.get_request_data(request, subcloud, files_for_resume) + + # Consider the incoming release parameter only if install is one + # of the pending deploy states + if INSTALL in deploy_states_to_run: + payload['software_version'] = payload.get('release', subcloud.software_version) + else: + payload['software_version'] = subcloud.software_version + + # Need to remove bootstrap_values from the list of files to populate + # pre existing data so it does not overwrite newly loaded values + if has_bootstrap_values: + files_for_resume = [f for f in files_for_resume if f + not in FILES_MAPPING[BOOTSTRAP]] + psd_common.populate_payload_with_pre_existing_data( + payload, subcloud, files_for_resume) + + psd_common.validate_sysadmin_password(payload) + for state in deploy_states_to_run: + if state == INSTALL: + psd_common.pre_deploy_install(payload, validate_password=False) + elif state == BOOTSTRAP: + psd_common.pre_deploy_bootstrap(context, payload, subcloud, + has_bootstrap_values, + validate_password=False) + elif state == CONFIG: + # Currently the only pre_deploy_config step is validate_sysadmin_password + # which can't be executed more than once + pass + + try: + self.dcmanager_rpc_client.subcloud_deploy_resume( + context, subcloud.id, subcloud.name, payload, deploy_states_to_run) + subcloud_dict = db_api.subcloud_db_model_to_dict(subcloud) + next_deploy_phase = RESUMABLE_STATES[subcloud.deploy_status][0] + next_deploy_state = RESUME_PREP_UPDATE_STATUS[next_deploy_phase] + subcloud_dict['deploy-status'] = next_deploy_state + subcloud_dict['software-version'] = payload['software_version'] + return subcloud_dict + except RemoteError as e: + pecan.abort(422, e.value) + except Exception: + LOG.exception("Unable to resume subcloud %s deployment" % subcloud.name) + pecan.abort(500, _('Unable to resume subcloud deployment')) + @pecan.expose(generic=True, template='json') def index(self): # Route the request to specific methods with parameters @@ -366,13 +464,15 @@ class PhasedSubcloudDeployController(object): except (exceptions.SubcloudNotFound, exceptions.SubcloudNameNotFound): pecan.abort(404, _('Subcloud not found')) - if verb == 'abort': + if verb == ABORT: subcloud = self._deploy_abort(context, subcloud) - elif verb == 'install': + elif verb == RESUME: + subcloud = self._deploy_resume(context, pecan.request, subcloud) + elif verb == INSTALL: subcloud = self._deploy_install(context, pecan.request, subcloud) - elif verb == 'bootstrap': + elif verb == BOOTSTRAP: subcloud = self._deploy_bootstrap(context, pecan.request, subcloud) - elif verb == 'configure': + elif verb == CONFIG: subcloud = self._deploy_config(context, pecan.request, subcloud) else: pecan.abort(400, _('Invalid request')) diff --git a/distributedcloud/dcmanager/api/policies/phased_subcloud_deploy.py b/distributedcloud/dcmanager/api/policies/phased_subcloud_deploy.py index fec59229e..5281e1aa2 100644 --- a/distributedcloud/dcmanager/api/policies/phased_subcloud_deploy.py +++ b/distributedcloud/dcmanager/api/policies/phased_subcloud_deploy.py @@ -31,6 +31,10 @@ phased_subcloud_deploy_rules = [ 'method': 'PATCH', 'path': '/v1.0/phased-subcloud-deploy/{subcloud}/abort' }, + { + 'method': 'PATCH', + 'path': '/v1.0/phased-subcloud-deploy/{subcloud}/resume' + }, { 'method': 'PATCH', 'path': '/v1.0/phased-subcloud-deploy/{subcloud}/install' diff --git a/distributedcloud/dcmanager/common/consts.py b/distributedcloud/dcmanager/common/consts.py index f6bb1d6e8..3a5274ae6 100644 --- a/distributedcloud/dcmanager/common/consts.py +++ b/distributedcloud/dcmanager/common/consts.py @@ -30,6 +30,14 @@ BOOTSTRAP_VALUES = 'bootstrap_values' BOOTSTRAP_ADDRESS = 'bootstrap-address' INSTALL_VALUES = 'install_values' +# Deploy phases +DEPLOY_PHASE_CREATE = 'create' +DEPLOY_PHASE_INSTALL = 'install' +DEPLOY_PHASE_BOOTSTRAP = 'bootstrap' +DEPLOY_PHASE_CONFIG = 'configure' +DEPLOY_PHASE_ABORT = 'abort' +DEPLOY_PHASE_RESUME = 'resume' + # Admin status for hosts ADMIN_LOCKED = 'locked' ADMIN_UNLOCKED = 'unlocked' diff --git a/distributedcloud/dcmanager/common/phased_subcloud_deploy.py b/distributedcloud/dcmanager/common/phased_subcloud_deploy.py index e1cc7b24f..9b76584d5 100644 --- a/distributedcloud/dcmanager/common/phased_subcloud_deploy.py +++ b/distributedcloud/dcmanager/common/phased_subcloud_deploy.py @@ -21,6 +21,7 @@ from dccommon.drivers.openstack.patching_v1 import PatchingClient from dccommon.drivers.openstack.sdk_platform import OpenStackDriver from dccommon.drivers.openstack.sysinv_v1 import SysinvClient from dcmanager.common import consts +from dcmanager.common.context import RequestContext from dcmanager.common import exceptions from dcmanager.common.i18n import _ from dcmanager.common import utils @@ -845,7 +846,7 @@ def populate_payload_with_pre_existing_data(payload: dict, msg = _("Required %s file was not provided and it was not " "previously available.") % value pecan.abort(400, msg) - payload.update(existing_values) + payload.update(dict(list(existing_values.items()) + list(payload.items()))) elif value == consts.DEPLOY_CONFIG: if not payload.get(consts.DEPLOY_CONFIG): fn = get_config_file_path(subcloud.name, value) @@ -857,8 +858,9 @@ def populate_payload_with_pre_existing_data(payload: dict, get_common_deploy_files(payload, subcloud.software_version) -def pre_deploy_install(payload: dict, - subcloud: models.Subcloud): +def pre_deploy_install(payload: dict, validate_password=False): + if validate_password: + validate_sysadmin_password(payload) install_values = payload['install_values'] @@ -885,3 +887,33 @@ def pre_deploy_install(payload: dict, if not payload.get('bmc_password'): payload.update({'bmc_password': install_values.get('bmc_password')}) payload.update({'install_values': install_values}) + + +def pre_deploy_bootstrap(context: RequestContext, payload: dict, + subcloud: models.Subcloud, has_bootstrap_values: bool, + validate_password=True): + if validate_password: + validate_sysadmin_password(payload) + if has_bootstrap_values: + # Need to validate the new values + payload_name = payload.get('name') + if payload_name != subcloud.name: + pecan.abort(400, _('The bootstrap-values "name" value (%s) ' + 'must match the current subcloud name (%s)' % + (payload_name, subcloud.name))) + + # Verify if payload contains all required bootstrap values + validate_bootstrap_values(payload) + + # It's ok for the management subnet to conflict with itself since we + # are only going to update it if it was modified, conflicts with + # other subclouds are still verified. + validate_subcloud_config(context, payload, + ignore_conflicts_with=subcloud) + format_ip_address(payload) + + # Patch status and fresh_install_k8s_version may have been changed + # between deploy create and deploy bootstrap commands. Validate them + # again: + validate_system_controller_patch_status("bootstrap") + validate_k8s_version(payload) diff --git a/distributedcloud/dcmanager/common/utils.py b/distributedcloud/dcmanager/common/utils.py index 15ed0cef6..9f82c7e20 100644 --- a/distributedcloud/dcmanager/common/utils.py +++ b/distributedcloud/dcmanager/common/utils.py @@ -69,6 +69,18 @@ ABORT_UPDATE_FAIL_STATUS = { consts.DEPLOY_STATE_ABORTING_CONFIG: consts.DEPLOY_STATE_CONFIG_FAILED } +RESUME_PREP_UPDATE_STATUS = { + consts.DEPLOY_PHASE_INSTALL: consts.DEPLOY_STATE_PRE_INSTALL, + consts.DEPLOY_PHASE_BOOTSTRAP: consts.DEPLOY_STATE_PRE_BOOTSTRAP, + consts.DEPLOY_PHASE_CONFIG: consts.DEPLOY_STATE_PRE_CONFIG +} + +RESUME_PREP_UPDATE_FAIL_STATUS = { + consts.DEPLOY_PHASE_INSTALL: consts.DEPLOY_STATE_PRE_INSTALL_FAILED, + consts.DEPLOY_PHASE_BOOTSTRAP: consts.DEPLOY_STATE_PRE_BOOTSTRAP_FAILED, + consts.DEPLOY_PHASE_CONFIG: consts.DEPLOY_STATE_PRE_CONFIG_FAILED +} + def get_import_path(cls): return cls.__module__ + "." + cls.__name__ diff --git a/distributedcloud/dcmanager/manager/service.py b/distributedcloud/dcmanager/manager/service.py index 6401c410b..d02851e20 100644 --- a/distributedcloud/dcmanager/manager/service.py +++ b/distributedcloud/dcmanager/manager/service.py @@ -233,6 +233,17 @@ class DCManagerService(service.Service): subcloud_id, deploy_status) + @request_context + def subcloud_deploy_resume(self, context, subcloud_id, subcloud_name, + payload, deploy_states_to_run): + # Adds a subcloud + LOG.info("Handling subcloud_deploy_resume request for: %s" % subcloud_name) + return self.subcloud_manager.subcloud_deploy_resume(context, + subcloud_id, + subcloud_name, + payload, + deploy_states_to_run) + def _stop_rpc_server(self): # Stop RPC connection to prevent new requests LOG.debug(_("Attempting to stop RPC service...")) diff --git a/distributedcloud/dcmanager/manager/subcloud_manager.py b/distributedcloud/dcmanager/manager/subcloud_manager.py index da7a2bb6d..8c6f34228 100644 --- a/distributedcloud/dcmanager/manager/subcloud_manager.py +++ b/distributedcloud/dcmanager/manager/subcloud_manager.py @@ -762,6 +762,75 @@ class SubcloudManager(manager.Manager): return self._subcloud_operation_notice('restore', restore_subclouds, failed_subclouds, invalid_subclouds) + def _deploy_bootstrap_prep(self, context, subcloud, payload: dict, + ansible_subcloud_inventory_file): + management_subnet = utils.get_management_subnet(payload) + sys_controller_gw_ip = payload.get( + "systemcontroller_gateway_address") + + if (management_subnet != subcloud.management_subnet) or ( + sys_controller_gw_ip != subcloud.systemcontroller_gateway_ip): + m_ks_client = OpenStackDriver( + region_name=dccommon_consts.DEFAULT_REGION_NAME, + region_clients=None).keystone_client + # Create a new route + self._create_subcloud_route(payload, m_ks_client, + sys_controller_gw_ip) + # Delete previous route + self._delete_subcloud_routes(m_ks_client, subcloud) + # Update endpoints + self._update_services_endpoint(context, payload, subcloud.name, + m_ks_client) + + # Update subcloud + subcloud = db_api.subcloud_update( + context, + subcloud.id, + description=payload.get("description", None), + management_subnet=utils.get_management_subnet(payload), + management_gateway_ip=utils.get_management_gateway_address( + payload), + management_start_ip=utils.get_management_start_address( + payload), + management_end_ip=utils.get_management_end_address(payload), + systemcontroller_gateway_ip=payload.get( + "systemcontroller_gateway_address", None), + location=payload.get("location", None), + deploy_status=consts.DEPLOY_STATE_PRE_BOOTSTRAP) + + # Populate payload with passwords + payload['ansible_become_pass'] = payload['sysadmin_password'] + payload['ansible_ssh_pass'] = payload['sysadmin_password'] + payload['admin_password'] = str(keyring.get_password('CGCS', 'admin')) + payload_without_sysadmin_password = payload.copy() + if 'sysadmin_password' in payload_without_sysadmin_password: + del payload_without_sysadmin_password['sysadmin_password'] + + # Update the ansible overrides file + overrides_file = os.path.join(dccommon_consts.ANSIBLE_OVERRIDES_PATH, + subcloud.name + '.yml') + utils.update_values_on_yaml_file(overrides_file, + payload_without_sysadmin_password) + + # Update the ansible inventory for the subcloud + utils.create_subcloud_inventory(payload, + ansible_subcloud_inventory_file) + + apply_command = self.compose_apply_command( + subcloud.name, + ansible_subcloud_inventory_file, + subcloud.software_version) + return apply_command + + def _deploy_config_prep(self, subcloud, payload: dict, + ansible_subcloud_inventory_file): + self._prepare_for_deployment(payload, subcloud.name) + deploy_command = self.compose_deploy_command( + subcloud.name, + ansible_subcloud_inventory_file, + payload) + return deploy_command + def _deploy_install_prep(self, subcloud, payload: dict, ansible_subcloud_inventory_file): payload['install_values']['ansible_ssh_pass'] = \ @@ -836,6 +905,22 @@ class SubcloudManager(manager.Manager): LOG.info("Successfully aborted deployment of %s" % subcloud.name) utils.update_abort_status(context, subcloud.id, subcloud.deploy_status) + def subcloud_deploy_resume(self, context, subcloud_id, subcloud_name, + payload: dict, deploy_states_to_run): + """Resume the subcloud deployment + + :param context: request context object + :param subcloud_id: subcloud id from db + :param subcloud_name: name of the subcloud + :param payload: subcloud resume payload + :param deploy_states_to_run: deploy phases pending execution + """ + LOG.info("Resuming deployment of subcloud %s. Deploy phases to be executed: %s" + % (subcloud_name, ', '.join(deploy_states_to_run))) + + self.run_deploy_phases(context, subcloud_id, payload, + deploy_states_to_run) + def subcloud_deploy_create(self, context, subcloud_id, payload): """Create subcloud and notify orchestrators. @@ -975,7 +1060,7 @@ class SubcloudManager(manager.Manager): deploy_status=consts.DEPLOY_STATE_CREATE_FAILED) return db_api.subcloud_db_model_to_dict(subcloud) - def subcloud_deploy_install(self, context, subcloud_id, payload: dict) -> dict: + def subcloud_deploy_install(self, context, subcloud_id, payload: dict): """Install subcloud :param context: request context object @@ -984,18 +1069,35 @@ class SubcloudManager(manager.Manager): """ # Retrieve the subcloud details from the database - subcloud = db_api.subcloud_get(context, subcloud_id) + subcloud = db_api.subcloud_update( + context, + subcloud_id, + software_version=payload['software_version'], + deploy_status=consts.DEPLOY_STATE_PRE_INSTALL, + data_install=json.dumps(payload['install_values'])) - LOG.info("Installing subcloud %s." % subcloud_id) + LOG.info("Installing subcloud %s." % subcloud.name) try: + log_file = ( + os.path.join(consts.DC_ANSIBLE_LOG_DIR, subcloud.name) + + "_playbook_output.log" + ) ansible_subcloud_inventory_file = self._get_ansible_filename( subcloud.name, INVENTORY_FILE_POSTFIX) install_command = self._deploy_install_prep( subcloud, payload, ansible_subcloud_inventory_file) - self.run_deploy_commands(subcloud, payload, context, - install_command=install_command) + install_success = self._run_subcloud_install( + context, subcloud, install_command, + log_file, payload['install_values'], + abortable=True) + if install_success: + db_api.subcloud_update( + context, subcloud.id, + deploy_status=consts.DEPLOY_STATE_INSTALLED, + error_description=consts.ERROR_DESC_EMPTY) + return install_success except Exception: LOG.exception("Failed to install subcloud %s" % subcloud.name) @@ -1004,6 +1106,7 @@ class SubcloudManager(manager.Manager): db_api.subcloud_update( context, subcloud_id, deploy_status=consts.DEPLOY_STATE_PRE_INSTALL_FAILED) + return False def subcloud_deploy_bootstrap(self, context, subcloud_id, payload): """Bootstrap subcloud @@ -1014,81 +1117,36 @@ class SubcloudManager(manager.Manager): """ LOG.info("Bootstrapping subcloud %s." % payload['name']) + # Retrieve the subcloud details from the database + subcloud = db_api.subcloud_get(context, subcloud_id) + try: - subcloud = db_api.subcloud_get(context, subcloud_id) - - management_subnet = utils.get_management_subnet(payload) - sys_controller_gw_ip = payload.get( - "systemcontroller_gateway_address") - - if (management_subnet != subcloud.management_subnet) or ( - sys_controller_gw_ip != subcloud.systemcontroller_gateway_ip): - m_ks_client = OpenStackDriver( - region_name=dccommon_consts.DEFAULT_REGION_NAME, - region_clients=None).keystone_client - # Create a new route - self._create_subcloud_route(payload, m_ks_client, - sys_controller_gw_ip) - # Delete previous route - self._delete_subcloud_routes(m_ks_client, subcloud) - # Update endpoints - self._update_services_endpoint(context, payload, subcloud.name, - m_ks_client) - - # Update subcloud - subcloud = db_api.subcloud_update( - context, - subcloud.id, - description=payload.get("description", None), - management_subnet=utils.get_management_subnet(payload), - management_gateway_ip=utils.get_management_gateway_address( - payload), - management_start_ip=utils.get_management_start_address( - payload), - management_end_ip=utils.get_management_end_address(payload), - systemcontroller_gateway_ip=payload.get( - "systemcontroller_gateway_address", None), - location=payload.get("location", None), - deploy_status=consts.DEPLOY_STATE_PRE_BOOTSTRAP) - - # Populate payload with passwords - payload['ansible_become_pass'] = payload['sysadmin_password'] - payload['ansible_ssh_pass'] = payload['sysadmin_password'] - payload['admin_password'] = str(keyring.get_password('CGCS', - 'admin')) - del payload['sysadmin_password'] - - # Update the ansible overrides file - overrides_file = os.path.join(dccommon_consts.ANSIBLE_OVERRIDES_PATH, - subcloud.name + '.yml') - utils.update_values_on_yaml_file(overrides_file, payload) - - # Ansible inventory filename for the specified subcloud - ansible_subcloud_inventory_file = utils.get_ansible_filename( + log_file = ( + os.path.join(consts.DC_ANSIBLE_LOG_DIR, subcloud.name) + + "_playbook_output.log" + ) + ansible_subcloud_inventory_file = self._get_ansible_filename( subcloud.name, INVENTORY_FILE_POSTFIX) - # Update the ansible inventory for the subcloud - utils.create_subcloud_inventory(payload, - ansible_subcloud_inventory_file) - - apply_command = self.compose_apply_command( - subcloud.name, - ansible_subcloud_inventory_file, - subcloud.software_version) - - self.run_deploy_commands(subcloud, payload, context, - apply_command=apply_command) + apply_command = self._deploy_bootstrap_prep( + context, subcloud, payload, + ansible_subcloud_inventory_file) + bootstrap_success = self._run_subcloud_bootstrap( + context, subcloud, apply_command, log_file) + return bootstrap_success except Exception: LOG.exception("Failed to bootstrap subcloud %s" % payload['name']) db_api.subcloud_update( context, subcloud_id, deploy_status=consts.DEPLOY_STATE_PRE_BOOTSTRAP_FAILED) + return False def subcloud_deploy_config(self, context, subcloud_id, payload: dict) -> dict: """Configure subcloud :param context: request context object + :param subcloud_id: subcloud_id from db :param payload: subcloud configuration """ LOG.info("Configuring subcloud %s." % subcloud_id) @@ -1097,6 +1155,10 @@ class SubcloudManager(manager.Manager): context, subcloud_id, deploy_status=consts.DEPLOY_STATE_PRE_CONFIG) try: + log_file = ( + os.path.join(consts.DC_ANSIBLE_LOG_DIR, subcloud.name) + + "_playbook_output.log" + ) # Ansible inventory filename for the specified subcloud ansible_subcloud_inventory_file = self._get_ansible_filename( subcloud.name, INVENTORY_FILE_POSTFIX) @@ -1107,14 +1169,16 @@ class SubcloudManager(manager.Manager): ansible_subcloud_inventory_file, payload) - self.run_deploy_commands(subcloud, payload, context, - deploy_command=deploy_command) + config_success = self._run_subcloud_config(subcloud, context, + deploy_command, log_file) + return config_success except Exception: LOG.exception("Failed to configure %s" % subcloud.name) db_api.subcloud_update( context, subcloud_id, deploy_status=consts.DEPLOY_STATE_PRE_CONFIG_FAILED) + return False def _subcloud_operation_notice( self, operation, restore_subclouds, failed_subclouds, @@ -1753,37 +1817,26 @@ class SubcloudManager(manager.Manager): deploy_status=consts.DEPLOY_STATE_DONE, error_description=consts.ERROR_DESC_EMPTY) - def run_deploy_commands(self, subcloud, payload, context, - install_command=None, apply_command=None, - deploy_command=None, rehome_command=None, - network_reconfig=None): + def run_deploy_phases(self, context, subcloud_id, payload, + deploy_states_to_run): + """Run individual phases durring deploy operation.""" try: - log_file = ( - os.path.join(consts.DC_ANSIBLE_LOG_DIR, subcloud.name) - + "_playbook_output.log" - ) - - if install_command: - install_success = self._run_subcloud_install( - context, subcloud, install_command, - log_file, payload['install_values'], - abortable=True) - if not install_success: - return - db_api.subcloud_update( - context, subcloud.id, - deploy_status=consts.DEPLOY_STATE_INSTALLED, - error_description=consts.ERROR_DESC_EMPTY) - - if apply_command: - bootstrap_success = self._run_subcloud_bootstrap( - context, subcloud, apply_command, log_file) - if not bootstrap_success: - return - - if deploy_command: - self._run_subcloud_config(subcloud, context, - deploy_command, log_file) + for state in deploy_states_to_run: + if state == consts.DEPLOY_PHASE_INSTALL: + install_success = self.subcloud_deploy_install( + context, subcloud_id, payload) + if not install_success: + return + elif state == consts.DEPLOY_PHASE_BOOTSTRAP: + bootstrap_success = self.subcloud_deploy_bootstrap( + context, subcloud_id, payload) + if not bootstrap_success: + return + elif state == consts.DEPLOY_PHASE_CONFIG: + config_success = self.subcloud_deploy_config( + context, subcloud_id, payload) + if not config_success: + return except Exception as ex: LOG.exception("run_deploy failed") @@ -1825,10 +1878,12 @@ class SubcloudManager(manager.Manager): software_version = str(payload['software_version']) LOG.info("Preparing remote install of %s, version: %s", subcloud.name, software_version) - db_api.subcloud_update( - context, subcloud.id, - deploy_status=consts.DEPLOY_STATE_PRE_INSTALL, - software_version=software_version) + if (subcloud.deploy_status != consts.DEPLOY_STATE_PRE_INSTALL or + subcloud.software_version != software_version): + db_api.subcloud_update( + context, subcloud.id, + deploy_status=consts.DEPLOY_STATE_PRE_INSTALL, + software_version=software_version) try: install = SubcloudInstall(context, subcloud.name) install.prep(dccommon_consts.ANSIBLE_OVERRIDES_PATH, payload) diff --git a/distributedcloud/dcmanager/rpc/client.py b/distributedcloud/dcmanager/rpc/client.py index a4fdfa8fd..38b79aee5 100644 --- a/distributedcloud/dcmanager/rpc/client.py +++ b/distributedcloud/dcmanager/rpc/client.py @@ -212,6 +212,14 @@ class ManagerClient(RPCClient): subcloud_id=subcloud_id, deploy_status=deploy_status)) + def subcloud_deploy_resume(self, ctxt, subcloud_id, subcloud_name, + payload, deploy_states_to_run): + return self.cast(ctxt, self.make_msg('subcloud_deploy_resume', + subcloud_id=subcloud_id, + subcloud_name=subcloud_name, + payload=payload, + deploy_states_to_run=deploy_states_to_run)) + class DCManagerNotifications(RPCClient): """DC Manager Notification interface to broadcast subcloud state changed diff --git a/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_phased_subcloud_deploy.py b/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_phased_subcloud_deploy.py index 0f4dd0ec5..c372250dc 100644 --- a/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_phased_subcloud_deploy.py +++ b/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_phased_subcloud_deploy.py @@ -9,11 +9,13 @@ import copy import json import mock +import os from os import path as os_path import six from tsconfig.tsconfig import SW_VERSION import webtest +from dcmanager.api.controllers.v1 import phased_subcloud_deploy as psd_api from dcmanager.common import consts from dcmanager.common import phased_subcloud_deploy as psd_common from dcmanager.common import utils as dutils @@ -339,7 +341,8 @@ class TestSubcloudDeployInstall(testroot.DCManagerApiTest): subcloud = fake_subcloud.create_fake_subcloud( self.ctx, - deploy_status=consts.DEPLOY_STATE_CREATED) + deploy_status=consts.DEPLOY_STATE_CREATED, + software_version=SW_VERSION) install_data = copy.copy(FAKE_SUBCLOUD_INSTALL_VALUES) install_data.pop('software_version') @@ -371,7 +374,8 @@ class TestSubcloudDeployInstall(testroot.DCManagerApiTest): subcloud = fake_subcloud.create_fake_subcloud( self.ctx, - deploy_status=consts.DEPLOY_STATE_CREATED) + deploy_status=consts.DEPLOY_STATE_CREATED, + software_version=SW_VERSION) install_data = copy.copy(FAKE_SUBCLOUD_INSTALL_VALUES) install_data.pop('software_version') @@ -398,14 +402,13 @@ class TestSubcloudDeployInstall(testroot.DCManagerApiTest): self.assertEqual(response.status_int, 200) self.assertEqual(consts.DEPLOY_STATE_PRE_INSTALL, response.json['deploy-status']) - self.assertEqual(FAKE_SOFTWARE_VERSION, - json.loads(response.json['data_install'])['software_version']) def test_install_subcloud_no_body(self): subcloud = fake_subcloud.create_fake_subcloud( self.ctx, - deploy_status=consts.DEPLOY_STATE_CREATED) + deploy_status=consts.DEPLOY_STATE_CREATED, + software_version=SW_VERSION) self.mock_get_request_data.return_value = {} @@ -419,6 +422,7 @@ class TestSubcloudDeployInstall(testroot.DCManagerApiTest): subcloud = fake_subcloud.create_fake_subcloud( self.ctx, deploy_status=consts.DEPLOY_STATE_CREATED, + software_version=SW_VERSION, data_install='') fake_sysadmin_password = base64.b64encode( @@ -438,7 +442,8 @@ class TestSubcloudDeployInstall(testroot.DCManagerApiTest): subcloud = fake_subcloud.create_fake_subcloud( self.ctx, - deploy_status=consts.DEPLOY_STATE_CREATED) + deploy_status=consts.DEPLOY_STATE_CREATED, + software_version=SW_VERSION) install_data = copy.copy(FAKE_SUBCLOUD_INSTALL_VALUES) install_data.pop('software_version') @@ -500,3 +505,176 @@ class TestSubcloudDeployAbort(testroot.DCManagerApiTest): self.app.patch_json, FAKE_URL + '/' + str(subcloud.id) + '/abort', headers=FAKE_HEADERS) + + +class TestSubcloudDeployResume(testroot.DCManagerApiTest): + def setUp(self): + super().setUp() + self.ctx = utils.dummy_context() + + p = mock.patch.object(rpc_client, 'ManagerClient') + self.mock_rpc_client = p.start() + self.addCleanup(p.stop) + + p = mock.patch.object(dutils, 'get_vault_load_files') + self.mock_get_vault_load_files = p.start() + self.addCleanup(p.stop) + + p = mock.patch.object(psd_common, 'get_subcloud_db_install_values') + self.mock_get_subcloud_db_install_values = p.start() + self.addCleanup(p.stop) + + p = mock.patch.object(psd_common, 'validate_k8s_version') + self.mock_validate_k8s_version = p.start() + self.addCleanup(p.stop) + + p = mock.patch.object(psd_common, 'get_request_data') + self.mock_get_request_data = p.start() + self.addCleanup(p.stop) + + self.management_address_pool = FakeAddressPool('192.168.204.0', 24, + '192.168.204.2', + '192.168.204.100') + + p = mock.patch.object(psd_common, 'get_network_address_pool') + self.mock_get_network_address_pool = p.start() + self.mock_get_network_address_pool.return_value = \ + self.management_address_pool + self.addCleanup(p.stop) + + p = mock.patch.object(psd_common, 'get_ks_client') + self.mock_get_ks_client = p.start() + self.addCleanup(p.stop) + + p = mock.patch.object(psd_common.PatchingClient, 'query') + self.mock_query = p.start() + self.addCleanup(p.stop) + + @mock.patch.object(os_path, 'isdir') + @mock.patch.object(os, 'listdir') + def test_resume_subcloud(self, + mock_os_listdir, + mock_os_isdir): + mock_os_isdir.return_value = True + mock_os_listdir.return_value = ['deploy_chart_fake.tgz', + 'deploy_overrides_fake.yaml', + 'deploy_playbook_fake.yaml'] + + subcloud = fake_subcloud.create_fake_subcloud( + self.ctx, + name=fake_subcloud.FAKE_BOOTSTRAP_FILE_DATA["name"], + deploy_status=consts.DEPLOY_STATE_CREATED, + software_version=SW_VERSION) + install_data = copy.copy(FAKE_SUBCLOUD_INSTALL_VALUES) + install_data.pop('software_version') + + self.mock_get_vault_load_files.return_value = ('iso_file_path', 'sig_file_path') + self.mock_rpc_client().subcloud_deploy_resume.return_value = True + + for state in psd_api.RESUMABLE_STATES: + fake_sysadmin_password = base64.b64encode( + 'testpass'.encode("utf-8")).decode('utf-8') + fake_bmc_password = base64.b64encode( + 'bmc_password'.encode("utf-8")).decode('utf-8') + bmc_password = {'bmc_password': fake_bmc_password} + install_data.update(bmc_password) + install_request = {'install_values': install_data, + 'sysadmin_password': fake_sysadmin_password, + 'bmc_password': fake_bmc_password} + bootstrap_request = {'bootstrap_values': fake_subcloud.FAKE_BOOTSTRAP_FILE_DATA} + config_request = {'deploy_config': 'deploy config values', + 'sysadmin_password': fake_sysadmin_password} + resume_request = {**install_request, + **bootstrap_request, + **config_request} + resume_payload = {**install_request, + **fake_subcloud.FAKE_BOOTSTRAP_FILE_DATA, + **config_request} + + subcloud = db_api.subcloud_update(self.ctx, + subcloud.id, + deploy_status=state) + next_deploy_phase = psd_api.RESUMABLE_STATES[subcloud.deploy_status][0] + next_deploy_state = psd_api.RESUME_PREP_UPDATE_STATUS[next_deploy_phase] + + self.mock_get_request_data.return_value = resume_payload + response = self.app.patch( + FAKE_URL + '/' + str(subcloud.id) + '/resume', + headers=FAKE_HEADERS, params=resume_request) + + self.assertEqual(response.status_int, 200) + self.assertEqual(next_deploy_state, + response.json['deploy-status']) + self.assertEqual(SW_VERSION, response.json['software-version']) + + def test_resume_subcloud_invalid_state(self): + + subcloud = fake_subcloud.create_fake_subcloud( + self.ctx, + name=fake_subcloud.FAKE_BOOTSTRAP_FILE_DATA["name"], + deploy_status=consts.DEPLOY_STATE_CREATED, + software_version=SW_VERSION) + + self.mock_get_vault_load_files.return_value = ('iso_file_path', 'sig_file_path') + self.mock_rpc_client().subcloud_deploy_resume.return_value = True + invalid_resume_states = [consts.DEPLOY_STATE_INSTALLING, + consts.DEPLOY_STATE_BOOTSTRAPPING, + consts.DEPLOY_STATE_CONFIGURING] + + for state in invalid_resume_states: + subcloud = db_api.subcloud_update(self.ctx, + subcloud.id, + deploy_status=state) + + six.assertRaisesRegex(self, webtest.app.AppError, "400 *", + self.app.patch_json, FAKE_URL + '/' + + str(subcloud.id) + '/resume', + headers=FAKE_HEADERS) + + @mock.patch.object(dutils, 'load_yaml_file') + @mock.patch.object(os_path, 'exists') + @mock.patch.object(os_path, 'isdir') + @mock.patch.object(os, 'listdir') + def test_resume_subcloud_no_request_data(self, + mock_os_listdir, + mock_os_isdir, + mock_path_exists, + mock_load_yaml): + subcloud = fake_subcloud.create_fake_subcloud( + self.ctx, + name=fake_subcloud.FAKE_BOOTSTRAP_FILE_DATA["name"], + deploy_status=consts.DEPLOY_STATE_CREATED, + software_version=SW_VERSION) + + config_file = psd_common.get_config_file_path(subcloud.name, + consts.DEPLOY_CONFIG) + mock_path_exists.side_effect = lambda x: True if x == config_file else False + mock_load_yaml.return_value = { + "software_version": fake_subcloud.FAKE_SOFTWARE_VERSION} + mock_os_isdir.return_value = True + mock_os_listdir.return_value = ['deploy_chart_fake.tgz', + 'deploy_overrides_fake.yaml', + 'deploy_playbook_fake.yaml'] + self.mock_get_vault_load_files.return_value = ('iso_file_path', 'sig_file_path') + self.mock_rpc_client().subcloud_deploy_resume.return_value = True + + for state in psd_api.RESUMABLE_STATES: + fake_sysadmin_password = base64.b64encode( + 'testpass'.encode("utf-8")).decode('utf-8') + resume_request = {'sysadmin_password': fake_sysadmin_password} + + subcloud = db_api.subcloud_update(self.ctx, + subcloud.id, + deploy_status=state) + next_deploy_phase = psd_api.RESUMABLE_STATES[subcloud.deploy_status][0] + next_deploy_state = psd_api.RESUME_PREP_UPDATE_STATUS[next_deploy_phase] + + self.mock_get_request_data.return_value = resume_request + response = self.app.patch( + FAKE_URL + '/' + str(subcloud.id) + '/resume', + headers=FAKE_HEADERS, params=resume_request) + + self.assertEqual(response.status_int, 200) + self.assertEqual(next_deploy_state, + response.json['deploy-status']) + self.assertEqual(SW_VERSION, response.json['software-version']) diff --git a/distributedcloud/dcmanager/tests/unit/manager/test_subcloud_manager.py b/distributedcloud/dcmanager/tests/unit/manager/test_subcloud_manager.py index a64c86647..9caeb8bb9 100644 --- a/distributedcloud/dcmanager/tests/unit/manager/test_subcloud_manager.py +++ b/distributedcloud/dcmanager/tests/unit/manager/test_subcloud_manager.py @@ -33,7 +33,6 @@ from dcmanager.common import consts from dcmanager.common import exceptions from dcmanager.common import prestage from dcmanager.common import utils as cutils -from dcmanager.db import api as dc_db_api from dcmanager.db.sqlalchemy import api as db_api from dcmanager.manager import subcloud_manager from dcmanager.state import subcloud_state_manager @@ -428,8 +427,12 @@ class TestSubcloudManager(base.DCManagerTestCase): @mock.patch.object( subcloud_manager.SubcloudManager, 'compose_install_command') - def test_deploy_install_subcloud(self, + @mock.patch.object( + subcloud_manager.SubcloudManager, '_run_subcloud_install') + def test_subcloud_deploy_install(self, + mock_run_subcloud_install, mock_compose_install_command): + mock_run_subcloud_install.return_value = True subcloud_name = 'subcloud1' subcloud = self.create_subcloud_static( @@ -453,6 +456,12 @@ class TestSubcloudManager(base.DCManagerTestCase): sm._get_ansible_filename(subcloud_name, consts.INVENTORY_FILE_POSTFIX), FAKE_PREVIOUS_SW_VERSION) + # Verify subcloud was updated with correct values + updated_subcloud = db_api.subcloud_get_by_name(self.ctx, + subcloud.name) + self.assertEqual(consts.DEPLOY_STATE_INSTALLED, + updated_subcloud.deploy_status) + @mock.patch.object(subcloud_manager.SubcloudManager, '_create_intermediate_ca_cert') @mock.patch.object(cutils, 'delete_subcloud_inventory') @@ -564,9 +573,10 @@ class TestSubcloudManager(base.DCManagerTestCase): self.assertEqual(consts.DEPLOY_STATE_BOOTSTRAPPED, updated_subcloud.deploy_status) - @mock.patch.object(dc_db_api, 'subcloud_get') - def test_subcloud_deploy_bootstrap_failed(self, mock_subcloud_get): - mock_subcloud_get.side_effect = FakeException('boom') + @mock.patch.object(subcloud_manager.SubcloudManager, + '_deploy_bootstrap_prep') + def test_subcloud_deploy_bootstrap_failed(self, mock_bootstrap_prep): + mock_bootstrap_prep.side_effect = FakeException('boom') subcloud = fake_subcloud.create_fake_subcloud( self.ctx, @@ -605,6 +615,66 @@ class TestSubcloudManager(base.DCManagerTestCase): payload=fake_payload) mock_prepare_for_deployment.assert_called_once() + @mock.patch.object(subcloud_manager.SubcloudManager, + '_run_subcloud_install') + @mock.patch.object(subcloud_manager.SubcloudManager, + '_prepare_for_deployment') + @mock.patch.object(cutils, 'create_subcloud_inventory') + @mock.patch.object(subcloud_manager, 'keyring') + @mock.patch.object(cutils, 'get_playbook_for_software_version') + @mock.patch.object(cutils, 'update_values_on_yaml_file') + @mock.patch.object(RunAnsible, 'exec_playbook') + def test_subcloud_deploy_resume(self, mock_exec_playbook, mock_update_yml, + mock_get_playbook_for_software_version, + mock_keyring, create_subcloud_inventory, + mock_prepare_for_deployment, + mock_run_subcloud_install): + mock_get_playbook_for_software_version.return_value = "22.12" + mock_keyring.get_password.return_value = "testpass" + mock_exec_playbook.return_value = False + mock_run_subcloud_install.return_value = True + + subcloud = self.create_subcloud_static( + self.ctx, + name='subcloud1', + deploy_status=consts.DEPLOY_STATE_CREATED) + + deploy_states_to_run = [consts.DEPLOY_PHASE_INSTALL, + consts.DEPLOY_PHASE_BOOTSTRAP, + consts.DEPLOY_PHASE_CONFIG] + + fake_install_values = \ + copy.copy(fake_subcloud.FAKE_SUBCLOUD_INSTALL_VALUES) + fake_install_values['software_version'] = SW_VERSION + fake_payload_install = {'bmc_password': 'bmc_pass', + 'install_values': fake_install_values, + 'software_version': SW_VERSION, + 'sysadmin_password': 'sys_pass'} + + fake_payload_bootstrap = {**fake_subcloud.FAKE_BOOTSTRAP_VALUE, + **fake_subcloud.FAKE_BOOTSTRAP_FILE_DATA} + fake_payload_bootstrap["sysadmin_password"] = "testpass" + + fake_payload_config = {"sysadmin_password": "testpass", + "deploy_playbook": "test_playbook.yaml", + "deploy_overrides": "test_overrides.yaml", + "deploy_chart": "test_chart.yaml", + "deploy_config": "subcloud1.yaml"} + + fake_payload = {**fake_payload_install, + **fake_payload_bootstrap, + **fake_payload_config} + + sm = subcloud_manager.SubcloudManager() + sm.subcloud_deploy_resume(self.ctx, subcloud.id, subcloud.name, + fake_payload, deploy_states_to_run) + + # Verify subcloud was updated with correct values + updated_subcloud = db_api.subcloud_get_by_name(self.ctx, + subcloud.name) + self.assertEqual(consts.DEPLOY_STATE_DONE, + updated_subcloud.deploy_status) + @mock.patch.object(subcloud_manager.SubcloudManager, 'compose_apply_command') @mock.patch.object(subcloud_manager.SubcloudManager,