diff --git a/api-ref/source/api-ref-dcmanager-v1.rst b/api-ref/source/api-ref-dcmanager-v1.rst index 36fb77ce8..b959fc588 100644 --- a/api-ref/source/api-ref-dcmanager-v1.rst +++ b/api-ref/source/api-ref-dcmanager-v1.rst @@ -2120,4 +2120,76 @@ Response Example ---------------- .. literalinclude:: samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-abort-response.json + :language: json + + +**************************** +Resume subcloud deployment +**************************** + +.. rest_method:: POST /v1.0/phased-subcloud-deploy + +Accepts Content-Type multipart/form-data. + + +**Normal response codes** + +200 + +**Error response codes** + +badRequest (400), unauthorized (401), forbidden (403), badMethod (405), +conflict (409), HTTPUnprocessableEntity (422), internalServerError (500), +serviceUnavailable (503) + +**Request parameters** + +.. rest_parameters:: parameters.yaml + + - bmc_password: bmc_password + - bootstrap-address: bootstrap_address + - bootstrap_values: bootstrap_values + - deploy_config: deploy_config + - install_values: install_values + - release: release + - sysadmin_password: sysadmin_password + +Request Example +---------------- + +.. literalinclude:: samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-resume-request.json + :language: json + + +**Response parameters** + +.. rest_parameters:: parameters.yaml + + - id: subcloud_id + - name: subcloud_name + - description: subcloud_description + - location: subcloud_location + - software-version: software_version + - management-state: management_state + - availability-status: availability_status + - deploy-status: deploy_status + - backup-status: backup_status + - backup-datetime: backup_datetime + - error-description: error_description + - management-subnet: management_subnet + - management-start-ip: management_start_ip + - management-end-ip: management_end_ip + - management-gateway-ip: management_gateway_ip + - openstack-installed: openstack_installed + - systemcontroller-gateway-ip: systemcontroller_gateway_ip + - data_install: data_install + - data_upgrade: data_upgrade + - created-at: created_at + - updated-at: updated_at + - group_id: group_id + +Response Example +---------------- + +.. literalinclude:: samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-resume-response.json :language: json \ No newline at end of file diff --git a/api-ref/source/samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-resume-request.json b/api-ref/source/samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-resume-request.json new file mode 100644 index 000000000..c4dd86591 --- /dev/null +++ b/api-ref/source/samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-resume-request.json @@ -0,0 +1,10 @@ +{ + "bmc_password": "YYYYYYY", + "bootstrap-address": "10.10.10.12", + "bootstrap_values": "content of bootstrap_values file", + "deploy_config": "content of deploy_config file", + "install_values": "content of install_values file", + "location": "Somewhere", + "release": "22.12", + "sysadmin_password": "XXXXXXX" +} \ No newline at end of file diff --git a/api-ref/source/samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-resume-response.json b/api-ref/source/samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-resume-response.json new file mode 100644 index 000000000..d976dcaf3 --- /dev/null +++ b/api-ref/source/samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-resume-response.json @@ -0,0 +1,24 @@ +{ + "id": 1, + "name": "subcloud1", + "description": "Subcloud 1", + "location": "Somewhere", + "software-version": "22.12", + "management-state": "unmanaged", + "availability-status": "offline", + "deploy-status": "pre-install", + "backup-status": null, + "backup-datetime": null, + "error-description": "No errors present", + "management-subnet": "192.168.102.0/24", + "management-start-ip": "192.168.102.2", + "management-end-ip": "192.168.102.50", + "management-gateway-ip": "192.168.102.1", + "openstack-installed": null, + "systemcontroller-gateway-ip": "192.168.204.101", + "data_install": null, + "data_upgrade": null, + "created-at": "2023-05-15 20: 58: 22.992609", + "updated-at": null, + "group_id": 1 +} \ No newline at end of file diff --git a/distributedcloud/dcmanager/api/controllers/v1/phased_subcloud_deploy.py b/distributedcloud/dcmanager/api/controllers/v1/phased_subcloud_deploy.py index 6d7dbaecf..d3d4a75b2 100644 --- a/distributedcloud/dcmanager/api/controllers/v1/phased_subcloud_deploy.py +++ b/distributedcloud/dcmanager/api/controllers/v1/phased_subcloud_deploy.py @@ -5,7 +5,6 @@ # import http.client as httpclient -import json import os from oslo_log import log as logging @@ -14,7 +13,6 @@ import pecan import tsconfig.tsconfig as tsc import yaml -from dccommon import consts as dccommon_consts from dcmanager.api.controllers import restcomm from dcmanager.api.policies import phased_subcloud_deploy as \ phased_subcloud_deploy_policy @@ -33,6 +31,12 @@ from dcmanager.rpc import client as rpc_client LOG = logging.getLogger(__name__) LOCK_NAME = 'PhasedSubcloudDeployController' +INSTALL = consts.DEPLOY_PHASE_INSTALL +BOOTSTRAP = consts.DEPLOY_PHASE_BOOTSTRAP +CONFIG = consts.DEPLOY_PHASE_CONFIG +ABORT = consts.DEPLOY_PHASE_ABORT +RESUME = consts.DEPLOY_PHASE_RESUME + SUBCLOUD_CREATE_REQUIRED_PARAMETERS = ( consts.BOOTSTRAP_VALUES, consts.BOOTSTRAP_ADDRESS @@ -93,6 +97,46 @@ VALID_STATES_FOR_DEPLOY_ABORT = ( consts.DEPLOY_STATE_CONFIGURING ) +FILES_FOR_RESUME_INSTALL = \ + SUBCLOUD_INSTALL_GET_FILE_CONTENTS + \ + SUBCLOUD_BOOTSTRAP_GET_FILE_CONTENTS + \ + SUBCLOUD_CONFIG_GET_FILE_CONTENTS + + +FILES_FOR_RESUME_BOOTSTRAP = \ + SUBCLOUD_BOOTSTRAP_GET_FILE_CONTENTS + \ + SUBCLOUD_CONFIG_GET_FILE_CONTENTS + + +FILES_FOR_RESUME_CONFIG = SUBCLOUD_CONFIG_GET_FILE_CONTENTS + +RESUMABLE_STATES = { + consts.DEPLOY_STATE_CREATED: [INSTALL, BOOTSTRAP, CONFIG], + consts.DEPLOY_STATE_INSTALLED: [BOOTSTRAP, CONFIG], + consts.DEPLOY_STATE_PRE_INSTALL_FAILED: [INSTALL, BOOTSTRAP, CONFIG], + consts.DEPLOY_STATE_INSTALL_FAILED: [INSTALL, BOOTSTRAP, CONFIG], + consts.DEPLOY_STATE_INSTALL_ABORTED: [INSTALL, BOOTSTRAP, CONFIG], + consts.DEPLOY_STATE_BOOTSTRAPPED: [CONFIG], + consts.DEPLOY_STATE_PRE_BOOTSTRAP_FAILED: [BOOTSTRAP, CONFIG], + consts.DEPLOY_STATE_BOOTSTRAP_FAILED: [BOOTSTRAP, CONFIG], + consts.DEPLOY_STATE_BOOTSTRAP_ABORTED: [BOOTSTRAP, CONFIG], + consts.DEPLOY_STATE_PRE_CONFIG_FAILED: [CONFIG], + consts.DEPLOY_STATE_CONFIG_FAILED: [CONFIG], + consts.DEPLOY_STATE_CONFIG_ABORTED: [CONFIG] +} + +FILES_MAPPING = { + INSTALL: SUBCLOUD_INSTALL_GET_FILE_CONTENTS, + BOOTSTRAP: SUBCLOUD_BOOTSTRAP_GET_FILE_CONTENTS, + CONFIG: SUBCLOUD_CONFIG_GET_FILE_CONTENTS +} + +RESUME_PREP_UPDATE_STATUS = { + INSTALL: consts.DEPLOY_STATE_PRE_INSTALL, + BOOTSTRAP: consts.DEPLOY_STATE_PRE_BOOTSTRAP, + CONFIG: consts.DEPLOY_STATE_PRE_CONFIG +} + def get_create_payload(request: pecan.Request) -> dict: payload = dict() @@ -181,30 +225,23 @@ class PhasedSubcloudDeployController(object): pecan.abort(400, _('Subcloud deploy status must be either: %s') % allowed_states_str) - payload['software_version'] = payload.get('release', tsc.SW_VERSION) + payload['software_version'] = payload.get('release', subcloud.software_version) psd_common.populate_payload_with_pre_existing_data( payload, subcloud, SUBCLOUD_INSTALL_GET_FILE_CONTENTS) - psd_common.validate_sysadmin_password(payload) psd_common.pre_deploy_install(payload, subcloud) try: # Align the software version of the subcloud with install # version. Update the deploy status as pre-install. - subcloud = db_api.subcloud_update( - context, - subcloud.id, - description=payload.get('description', subcloud.description), - location=payload.get('location', subcloud.location), - software_version=payload['software_version'], - management_state=dccommon_consts.MANAGEMENT_UNMANAGED, - deploy_status=consts.DEPLOY_STATE_PRE_INSTALL, - data_install=json.dumps(payload['install_values'])) self.dcmanager_rpc_client.subcloud_deploy_install( context, subcloud.id, payload) + subcloud_dict = db_api.subcloud_db_model_to_dict(subcloud) + subcloud_dict['deploy-status'] = consts.DEPLOY_STATE_PRE_INSTALL + subcloud_dict['software-version'] = payload['software_version'] - return db_api.subcloud_db_model_to_dict(subcloud) + return subcloud_dict except RemoteError as e: pecan.abort(422, e.value) except Exception: @@ -238,31 +275,8 @@ class PhasedSubcloudDeployController(object): # Update the existing values with new ones from the request payload.update(request_data) - psd_common.validate_sysadmin_password(payload) - - if has_bootstrap_values: - # Need to validate the new values - playload_name = payload.get('name') - if playload_name != subcloud.name: - pecan.abort(400, _('The bootstrap-values "name" value (%s) ' - 'must match the current subcloud name (%s)' % - (playload_name, subcloud.name))) - - # Verify if payload contains all required bootstrap values - psd_common.validate_bootstrap_values(payload) - - # It's ok for the management subnet to conflict with itself since we - # are only going to update it if it was modified, conflicts with - # other subclouds are still verified. - psd_common.validate_subcloud_config(context, payload, - ignore_conflicts_with=subcloud) - psd_common.format_ip_address(payload) - - # Patch status and fresh_install_k8s_version may have been changed - # between deploy create and deploy bootstrap commands. Validate them - # again: - psd_common.validate_system_controller_patch_status("bootstrap") - psd_common.validate_k8s_version(payload) + psd_common.pre_deploy_bootstrap(context, payload, subcloud, + has_bootstrap_values) try: # Ask dcmanager-manager to bootstrap the subcloud. @@ -329,6 +343,90 @@ class PhasedSubcloudDeployController(object): LOG.exception("Unable to abort subcloud %s deployment" % subcloud.name) pecan.abort(500, _('Unable to abort subcloud deploy')) + def _deploy_resume(self, context: RequestContext, + request: pecan.Request, subcloud): + + if subcloud.deploy_status not in RESUMABLE_STATES: + allowed_states_str = ', '.join(RESUMABLE_STATES) + pecan.abort(400, _('Subcloud deploy status must be either: %s') + % allowed_states_str) + + # Since both install and config are optional phases, + # it's necessary to check if they should be executed + config_file = psd_common.get_config_file_path(subcloud.name, + consts.DEPLOY_CONFIG) + has_original_install_values = subcloud.data_install + has_original_config_values = os.path.exists(config_file) + has_new_install_values = consts.INSTALL_VALUES in request.POST + has_new_config_values = consts.DEPLOY_CONFIG in request.POST + has_bootstrap_values = consts.BOOTSTRAP_VALUES in request.POST + has_config_values = has_original_config_values or has_new_config_values + has_install_values = has_original_install_values or has_new_install_values + + deploy_states_to_run = RESUMABLE_STATES[subcloud.deploy_status] + if deploy_states_to_run == [CONFIG] and not has_config_values: + msg = _("Only deploy phase left is deploy config. " + "Required %s file was not provided and it was not " + "previously available.") % consts.DEPLOY_CONFIG + pecan.abort(400, msg) + + # Since the subcloud can be installed manually and the config is optional, + # skip those phases if the user doesn't provide the install or config values + # and they are not available from previous executions. + files_for_resume = [] + for state in deploy_states_to_run: + if state == INSTALL and not has_install_values: + deploy_states_to_run.remove(state) + elif state == CONFIG and not has_config_values: + deploy_states_to_run.remove(state) + else: + files_for_resume.extend(FILES_MAPPING[state]) + + payload = psd_common.get_request_data(request, subcloud, files_for_resume) + + # Consider the incoming release parameter only if install is one + # of the pending deploy states + if INSTALL in deploy_states_to_run: + payload['software_version'] = payload.get('release', subcloud.software_version) + else: + payload['software_version'] = subcloud.software_version + + # Need to remove bootstrap_values from the list of files to populate + # pre existing data so it does not overwrite newly loaded values + if has_bootstrap_values: + files_for_resume = [f for f in files_for_resume if f + not in FILES_MAPPING[BOOTSTRAP]] + psd_common.populate_payload_with_pre_existing_data( + payload, subcloud, files_for_resume) + + psd_common.validate_sysadmin_password(payload) + for state in deploy_states_to_run: + if state == INSTALL: + psd_common.pre_deploy_install(payload, validate_password=False) + elif state == BOOTSTRAP: + psd_common.pre_deploy_bootstrap(context, payload, subcloud, + has_bootstrap_values, + validate_password=False) + elif state == CONFIG: + # Currently the only pre_deploy_config step is validate_sysadmin_password + # which can't be executed more than once + pass + + try: + self.dcmanager_rpc_client.subcloud_deploy_resume( + context, subcloud.id, subcloud.name, payload, deploy_states_to_run) + subcloud_dict = db_api.subcloud_db_model_to_dict(subcloud) + next_deploy_phase = RESUMABLE_STATES[subcloud.deploy_status][0] + next_deploy_state = RESUME_PREP_UPDATE_STATUS[next_deploy_phase] + subcloud_dict['deploy-status'] = next_deploy_state + subcloud_dict['software-version'] = payload['software_version'] + return subcloud_dict + except RemoteError as e: + pecan.abort(422, e.value) + except Exception: + LOG.exception("Unable to resume subcloud %s deployment" % subcloud.name) + pecan.abort(500, _('Unable to resume subcloud deployment')) + @pecan.expose(generic=True, template='json') def index(self): # Route the request to specific methods with parameters @@ -366,13 +464,15 @@ class PhasedSubcloudDeployController(object): except (exceptions.SubcloudNotFound, exceptions.SubcloudNameNotFound): pecan.abort(404, _('Subcloud not found')) - if verb == 'abort': + if verb == ABORT: subcloud = self._deploy_abort(context, subcloud) - elif verb == 'install': + elif verb == RESUME: + subcloud = self._deploy_resume(context, pecan.request, subcloud) + elif verb == INSTALL: subcloud = self._deploy_install(context, pecan.request, subcloud) - elif verb == 'bootstrap': + elif verb == BOOTSTRAP: subcloud = self._deploy_bootstrap(context, pecan.request, subcloud) - elif verb == 'configure': + elif verb == CONFIG: subcloud = self._deploy_config(context, pecan.request, subcloud) else: pecan.abort(400, _('Invalid request')) diff --git a/distributedcloud/dcmanager/api/policies/phased_subcloud_deploy.py b/distributedcloud/dcmanager/api/policies/phased_subcloud_deploy.py index fec59229e..5281e1aa2 100644 --- a/distributedcloud/dcmanager/api/policies/phased_subcloud_deploy.py +++ b/distributedcloud/dcmanager/api/policies/phased_subcloud_deploy.py @@ -31,6 +31,10 @@ phased_subcloud_deploy_rules = [ 'method': 'PATCH', 'path': '/v1.0/phased-subcloud-deploy/{subcloud}/abort' }, + { + 'method': 'PATCH', + 'path': '/v1.0/phased-subcloud-deploy/{subcloud}/resume' + }, { 'method': 'PATCH', 'path': '/v1.0/phased-subcloud-deploy/{subcloud}/install' diff --git a/distributedcloud/dcmanager/common/consts.py b/distributedcloud/dcmanager/common/consts.py index f6bb1d6e8..3a5274ae6 100644 --- a/distributedcloud/dcmanager/common/consts.py +++ b/distributedcloud/dcmanager/common/consts.py @@ -30,6 +30,14 @@ BOOTSTRAP_VALUES = 'bootstrap_values' BOOTSTRAP_ADDRESS = 'bootstrap-address' INSTALL_VALUES = 'install_values' +# Deploy phases +DEPLOY_PHASE_CREATE = 'create' +DEPLOY_PHASE_INSTALL = 'install' +DEPLOY_PHASE_BOOTSTRAP = 'bootstrap' +DEPLOY_PHASE_CONFIG = 'configure' +DEPLOY_PHASE_ABORT = 'abort' +DEPLOY_PHASE_RESUME = 'resume' + # Admin status for hosts ADMIN_LOCKED = 'locked' ADMIN_UNLOCKED = 'unlocked' diff --git a/distributedcloud/dcmanager/common/phased_subcloud_deploy.py b/distributedcloud/dcmanager/common/phased_subcloud_deploy.py index e1cc7b24f..9b76584d5 100644 --- a/distributedcloud/dcmanager/common/phased_subcloud_deploy.py +++ b/distributedcloud/dcmanager/common/phased_subcloud_deploy.py @@ -21,6 +21,7 @@ from dccommon.drivers.openstack.patching_v1 import PatchingClient from dccommon.drivers.openstack.sdk_platform import OpenStackDriver from dccommon.drivers.openstack.sysinv_v1 import SysinvClient from dcmanager.common import consts +from dcmanager.common.context import RequestContext from dcmanager.common import exceptions from dcmanager.common.i18n import _ from dcmanager.common import utils @@ -845,7 +846,7 @@ def populate_payload_with_pre_existing_data(payload: dict, msg = _("Required %s file was not provided and it was not " "previously available.") % value pecan.abort(400, msg) - payload.update(existing_values) + payload.update(dict(list(existing_values.items()) + list(payload.items()))) elif value == consts.DEPLOY_CONFIG: if not payload.get(consts.DEPLOY_CONFIG): fn = get_config_file_path(subcloud.name, value) @@ -857,8 +858,9 @@ def populate_payload_with_pre_existing_data(payload: dict, get_common_deploy_files(payload, subcloud.software_version) -def pre_deploy_install(payload: dict, - subcloud: models.Subcloud): +def pre_deploy_install(payload: dict, validate_password=False): + if validate_password: + validate_sysadmin_password(payload) install_values = payload['install_values'] @@ -885,3 +887,33 @@ def pre_deploy_install(payload: dict, if not payload.get('bmc_password'): payload.update({'bmc_password': install_values.get('bmc_password')}) payload.update({'install_values': install_values}) + + +def pre_deploy_bootstrap(context: RequestContext, payload: dict, + subcloud: models.Subcloud, has_bootstrap_values: bool, + validate_password=True): + if validate_password: + validate_sysadmin_password(payload) + if has_bootstrap_values: + # Need to validate the new values + payload_name = payload.get('name') + if payload_name != subcloud.name: + pecan.abort(400, _('The bootstrap-values "name" value (%s) ' + 'must match the current subcloud name (%s)' % + (payload_name, subcloud.name))) + + # Verify if payload contains all required bootstrap values + validate_bootstrap_values(payload) + + # It's ok for the management subnet to conflict with itself since we + # are only going to update it if it was modified, conflicts with + # other subclouds are still verified. + validate_subcloud_config(context, payload, + ignore_conflicts_with=subcloud) + format_ip_address(payload) + + # Patch status and fresh_install_k8s_version may have been changed + # between deploy create and deploy bootstrap commands. Validate them + # again: + validate_system_controller_patch_status("bootstrap") + validate_k8s_version(payload) diff --git a/distributedcloud/dcmanager/common/utils.py b/distributedcloud/dcmanager/common/utils.py index 15ed0cef6..9f82c7e20 100644 --- a/distributedcloud/dcmanager/common/utils.py +++ b/distributedcloud/dcmanager/common/utils.py @@ -69,6 +69,18 @@ ABORT_UPDATE_FAIL_STATUS = { consts.DEPLOY_STATE_ABORTING_CONFIG: consts.DEPLOY_STATE_CONFIG_FAILED } +RESUME_PREP_UPDATE_STATUS = { + consts.DEPLOY_PHASE_INSTALL: consts.DEPLOY_STATE_PRE_INSTALL, + consts.DEPLOY_PHASE_BOOTSTRAP: consts.DEPLOY_STATE_PRE_BOOTSTRAP, + consts.DEPLOY_PHASE_CONFIG: consts.DEPLOY_STATE_PRE_CONFIG +} + +RESUME_PREP_UPDATE_FAIL_STATUS = { + consts.DEPLOY_PHASE_INSTALL: consts.DEPLOY_STATE_PRE_INSTALL_FAILED, + consts.DEPLOY_PHASE_BOOTSTRAP: consts.DEPLOY_STATE_PRE_BOOTSTRAP_FAILED, + consts.DEPLOY_PHASE_CONFIG: consts.DEPLOY_STATE_PRE_CONFIG_FAILED +} + def get_import_path(cls): return cls.__module__ + "." + cls.__name__ diff --git a/distributedcloud/dcmanager/manager/service.py b/distributedcloud/dcmanager/manager/service.py index 6401c410b..d02851e20 100644 --- a/distributedcloud/dcmanager/manager/service.py +++ b/distributedcloud/dcmanager/manager/service.py @@ -233,6 +233,17 @@ class DCManagerService(service.Service): subcloud_id, deploy_status) + @request_context + def subcloud_deploy_resume(self, context, subcloud_id, subcloud_name, + payload, deploy_states_to_run): + # Adds a subcloud + LOG.info("Handling subcloud_deploy_resume request for: %s" % subcloud_name) + return self.subcloud_manager.subcloud_deploy_resume(context, + subcloud_id, + subcloud_name, + payload, + deploy_states_to_run) + def _stop_rpc_server(self): # Stop RPC connection to prevent new requests LOG.debug(_("Attempting to stop RPC service...")) diff --git a/distributedcloud/dcmanager/manager/subcloud_manager.py b/distributedcloud/dcmanager/manager/subcloud_manager.py index da7a2bb6d..8c6f34228 100644 --- a/distributedcloud/dcmanager/manager/subcloud_manager.py +++ b/distributedcloud/dcmanager/manager/subcloud_manager.py @@ -762,6 +762,75 @@ class SubcloudManager(manager.Manager): return self._subcloud_operation_notice('restore', restore_subclouds, failed_subclouds, invalid_subclouds) + def _deploy_bootstrap_prep(self, context, subcloud, payload: dict, + ansible_subcloud_inventory_file): + management_subnet = utils.get_management_subnet(payload) + sys_controller_gw_ip = payload.get( + "systemcontroller_gateway_address") + + if (management_subnet != subcloud.management_subnet) or ( + sys_controller_gw_ip != subcloud.systemcontroller_gateway_ip): + m_ks_client = OpenStackDriver( + region_name=dccommon_consts.DEFAULT_REGION_NAME, + region_clients=None).keystone_client + # Create a new route + self._create_subcloud_route(payload, m_ks_client, + sys_controller_gw_ip) + # Delete previous route + self._delete_subcloud_routes(m_ks_client, subcloud) + # Update endpoints + self._update_services_endpoint(context, payload, subcloud.name, + m_ks_client) + + # Update subcloud + subcloud = db_api.subcloud_update( + context, + subcloud.id, + description=payload.get("description", None), + management_subnet=utils.get_management_subnet(payload), + management_gateway_ip=utils.get_management_gateway_address( + payload), + management_start_ip=utils.get_management_start_address( + payload), + management_end_ip=utils.get_management_end_address(payload), + systemcontroller_gateway_ip=payload.get( + "systemcontroller_gateway_address", None), + location=payload.get("location", None), + deploy_status=consts.DEPLOY_STATE_PRE_BOOTSTRAP) + + # Populate payload with passwords + payload['ansible_become_pass'] = payload['sysadmin_password'] + payload['ansible_ssh_pass'] = payload['sysadmin_password'] + payload['admin_password'] = str(keyring.get_password('CGCS', 'admin')) + payload_without_sysadmin_password = payload.copy() + if 'sysadmin_password' in payload_without_sysadmin_password: + del payload_without_sysadmin_password['sysadmin_password'] + + # Update the ansible overrides file + overrides_file = os.path.join(dccommon_consts.ANSIBLE_OVERRIDES_PATH, + subcloud.name + '.yml') + utils.update_values_on_yaml_file(overrides_file, + payload_without_sysadmin_password) + + # Update the ansible inventory for the subcloud + utils.create_subcloud_inventory(payload, + ansible_subcloud_inventory_file) + + apply_command = self.compose_apply_command( + subcloud.name, + ansible_subcloud_inventory_file, + subcloud.software_version) + return apply_command + + def _deploy_config_prep(self, subcloud, payload: dict, + ansible_subcloud_inventory_file): + self._prepare_for_deployment(payload, subcloud.name) + deploy_command = self.compose_deploy_command( + subcloud.name, + ansible_subcloud_inventory_file, + payload) + return deploy_command + def _deploy_install_prep(self, subcloud, payload: dict, ansible_subcloud_inventory_file): payload['install_values']['ansible_ssh_pass'] = \ @@ -836,6 +905,22 @@ class SubcloudManager(manager.Manager): LOG.info("Successfully aborted deployment of %s" % subcloud.name) utils.update_abort_status(context, subcloud.id, subcloud.deploy_status) + def subcloud_deploy_resume(self, context, subcloud_id, subcloud_name, + payload: dict, deploy_states_to_run): + """Resume the subcloud deployment + + :param context: request context object + :param subcloud_id: subcloud id from db + :param subcloud_name: name of the subcloud + :param payload: subcloud resume payload + :param deploy_states_to_run: deploy phases pending execution + """ + LOG.info("Resuming deployment of subcloud %s. Deploy phases to be executed: %s" + % (subcloud_name, ', '.join(deploy_states_to_run))) + + self.run_deploy_phases(context, subcloud_id, payload, + deploy_states_to_run) + def subcloud_deploy_create(self, context, subcloud_id, payload): """Create subcloud and notify orchestrators. @@ -975,7 +1060,7 @@ class SubcloudManager(manager.Manager): deploy_status=consts.DEPLOY_STATE_CREATE_FAILED) return db_api.subcloud_db_model_to_dict(subcloud) - def subcloud_deploy_install(self, context, subcloud_id, payload: dict) -> dict: + def subcloud_deploy_install(self, context, subcloud_id, payload: dict): """Install subcloud :param context: request context object @@ -984,18 +1069,35 @@ class SubcloudManager(manager.Manager): """ # Retrieve the subcloud details from the database - subcloud = db_api.subcloud_get(context, subcloud_id) + subcloud = db_api.subcloud_update( + context, + subcloud_id, + software_version=payload['software_version'], + deploy_status=consts.DEPLOY_STATE_PRE_INSTALL, + data_install=json.dumps(payload['install_values'])) - LOG.info("Installing subcloud %s." % subcloud_id) + LOG.info("Installing subcloud %s." % subcloud.name) try: + log_file = ( + os.path.join(consts.DC_ANSIBLE_LOG_DIR, subcloud.name) + + "_playbook_output.log" + ) ansible_subcloud_inventory_file = self._get_ansible_filename( subcloud.name, INVENTORY_FILE_POSTFIX) install_command = self._deploy_install_prep( subcloud, payload, ansible_subcloud_inventory_file) - self.run_deploy_commands(subcloud, payload, context, - install_command=install_command) + install_success = self._run_subcloud_install( + context, subcloud, install_command, + log_file, payload['install_values'], + abortable=True) + if install_success: + db_api.subcloud_update( + context, subcloud.id, + deploy_status=consts.DEPLOY_STATE_INSTALLED, + error_description=consts.ERROR_DESC_EMPTY) + return install_success except Exception: LOG.exception("Failed to install subcloud %s" % subcloud.name) @@ -1004,6 +1106,7 @@ class SubcloudManager(manager.Manager): db_api.subcloud_update( context, subcloud_id, deploy_status=consts.DEPLOY_STATE_PRE_INSTALL_FAILED) + return False def subcloud_deploy_bootstrap(self, context, subcloud_id, payload): """Bootstrap subcloud @@ -1014,81 +1117,36 @@ class SubcloudManager(manager.Manager): """ LOG.info("Bootstrapping subcloud %s." % payload['name']) + # Retrieve the subcloud details from the database + subcloud = db_api.subcloud_get(context, subcloud_id) + try: - subcloud = db_api.subcloud_get(context, subcloud_id) - - management_subnet = utils.get_management_subnet(payload) - sys_controller_gw_ip = payload.get( - "systemcontroller_gateway_address") - - if (management_subnet != subcloud.management_subnet) or ( - sys_controller_gw_ip != subcloud.systemcontroller_gateway_ip): - m_ks_client = OpenStackDriver( - region_name=dccommon_consts.DEFAULT_REGION_NAME, - region_clients=None).keystone_client - # Create a new route - self._create_subcloud_route(payload, m_ks_client, - sys_controller_gw_ip) - # Delete previous route - self._delete_subcloud_routes(m_ks_client, subcloud) - # Update endpoints - self._update_services_endpoint(context, payload, subcloud.name, - m_ks_client) - - # Update subcloud - subcloud = db_api.subcloud_update( - context, - subcloud.id, - description=payload.get("description", None), - management_subnet=utils.get_management_subnet(payload), - management_gateway_ip=utils.get_management_gateway_address( - payload), - management_start_ip=utils.get_management_start_address( - payload), - management_end_ip=utils.get_management_end_address(payload), - systemcontroller_gateway_ip=payload.get( - "systemcontroller_gateway_address", None), - location=payload.get("location", None), - deploy_status=consts.DEPLOY_STATE_PRE_BOOTSTRAP) - - # Populate payload with passwords - payload['ansible_become_pass'] = payload['sysadmin_password'] - payload['ansible_ssh_pass'] = payload['sysadmin_password'] - payload['admin_password'] = str(keyring.get_password('CGCS', - 'admin')) - del payload['sysadmin_password'] - - # Update the ansible overrides file - overrides_file = os.path.join(dccommon_consts.ANSIBLE_OVERRIDES_PATH, - subcloud.name + '.yml') - utils.update_values_on_yaml_file(overrides_file, payload) - - # Ansible inventory filename for the specified subcloud - ansible_subcloud_inventory_file = utils.get_ansible_filename( + log_file = ( + os.path.join(consts.DC_ANSIBLE_LOG_DIR, subcloud.name) + + "_playbook_output.log" + ) + ansible_subcloud_inventory_file = self._get_ansible_filename( subcloud.name, INVENTORY_FILE_POSTFIX) - # Update the ansible inventory for the subcloud - utils.create_subcloud_inventory(payload, - ansible_subcloud_inventory_file) - - apply_command = self.compose_apply_command( - subcloud.name, - ansible_subcloud_inventory_file, - subcloud.software_version) - - self.run_deploy_commands(subcloud, payload, context, - apply_command=apply_command) + apply_command = self._deploy_bootstrap_prep( + context, subcloud, payload, + ansible_subcloud_inventory_file) + bootstrap_success = self._run_subcloud_bootstrap( + context, subcloud, apply_command, log_file) + return bootstrap_success except Exception: LOG.exception("Failed to bootstrap subcloud %s" % payload['name']) db_api.subcloud_update( context, subcloud_id, deploy_status=consts.DEPLOY_STATE_PRE_BOOTSTRAP_FAILED) + return False def subcloud_deploy_config(self, context, subcloud_id, payload: dict) -> dict: """Configure subcloud :param context: request context object + :param subcloud_id: subcloud_id from db :param payload: subcloud configuration """ LOG.info("Configuring subcloud %s." % subcloud_id) @@ -1097,6 +1155,10 @@ class SubcloudManager(manager.Manager): context, subcloud_id, deploy_status=consts.DEPLOY_STATE_PRE_CONFIG) try: + log_file = ( + os.path.join(consts.DC_ANSIBLE_LOG_DIR, subcloud.name) + + "_playbook_output.log" + ) # Ansible inventory filename for the specified subcloud ansible_subcloud_inventory_file = self._get_ansible_filename( subcloud.name, INVENTORY_FILE_POSTFIX) @@ -1107,14 +1169,16 @@ class SubcloudManager(manager.Manager): ansible_subcloud_inventory_file, payload) - self.run_deploy_commands(subcloud, payload, context, - deploy_command=deploy_command) + config_success = self._run_subcloud_config(subcloud, context, + deploy_command, log_file) + return config_success except Exception: LOG.exception("Failed to configure %s" % subcloud.name) db_api.subcloud_update( context, subcloud_id, deploy_status=consts.DEPLOY_STATE_PRE_CONFIG_FAILED) + return False def _subcloud_operation_notice( self, operation, restore_subclouds, failed_subclouds, @@ -1753,37 +1817,26 @@ class SubcloudManager(manager.Manager): deploy_status=consts.DEPLOY_STATE_DONE, error_description=consts.ERROR_DESC_EMPTY) - def run_deploy_commands(self, subcloud, payload, context, - install_command=None, apply_command=None, - deploy_command=None, rehome_command=None, - network_reconfig=None): + def run_deploy_phases(self, context, subcloud_id, payload, + deploy_states_to_run): + """Run individual phases durring deploy operation.""" try: - log_file = ( - os.path.join(consts.DC_ANSIBLE_LOG_DIR, subcloud.name) - + "_playbook_output.log" - ) - - if install_command: - install_success = self._run_subcloud_install( - context, subcloud, install_command, - log_file, payload['install_values'], - abortable=True) - if not install_success: - return - db_api.subcloud_update( - context, subcloud.id, - deploy_status=consts.DEPLOY_STATE_INSTALLED, - error_description=consts.ERROR_DESC_EMPTY) - - if apply_command: - bootstrap_success = self._run_subcloud_bootstrap( - context, subcloud, apply_command, log_file) - if not bootstrap_success: - return - - if deploy_command: - self._run_subcloud_config(subcloud, context, - deploy_command, log_file) + for state in deploy_states_to_run: + if state == consts.DEPLOY_PHASE_INSTALL: + install_success = self.subcloud_deploy_install( + context, subcloud_id, payload) + if not install_success: + return + elif state == consts.DEPLOY_PHASE_BOOTSTRAP: + bootstrap_success = self.subcloud_deploy_bootstrap( + context, subcloud_id, payload) + if not bootstrap_success: + return + elif state == consts.DEPLOY_PHASE_CONFIG: + config_success = self.subcloud_deploy_config( + context, subcloud_id, payload) + if not config_success: + return except Exception as ex: LOG.exception("run_deploy failed") @@ -1825,10 +1878,12 @@ class SubcloudManager(manager.Manager): software_version = str(payload['software_version']) LOG.info("Preparing remote install of %s, version: %s", subcloud.name, software_version) - db_api.subcloud_update( - context, subcloud.id, - deploy_status=consts.DEPLOY_STATE_PRE_INSTALL, - software_version=software_version) + if (subcloud.deploy_status != consts.DEPLOY_STATE_PRE_INSTALL or + subcloud.software_version != software_version): + db_api.subcloud_update( + context, subcloud.id, + deploy_status=consts.DEPLOY_STATE_PRE_INSTALL, + software_version=software_version) try: install = SubcloudInstall(context, subcloud.name) install.prep(dccommon_consts.ANSIBLE_OVERRIDES_PATH, payload) diff --git a/distributedcloud/dcmanager/rpc/client.py b/distributedcloud/dcmanager/rpc/client.py index a4fdfa8fd..38b79aee5 100644 --- a/distributedcloud/dcmanager/rpc/client.py +++ b/distributedcloud/dcmanager/rpc/client.py @@ -212,6 +212,14 @@ class ManagerClient(RPCClient): subcloud_id=subcloud_id, deploy_status=deploy_status)) + def subcloud_deploy_resume(self, ctxt, subcloud_id, subcloud_name, + payload, deploy_states_to_run): + return self.cast(ctxt, self.make_msg('subcloud_deploy_resume', + subcloud_id=subcloud_id, + subcloud_name=subcloud_name, + payload=payload, + deploy_states_to_run=deploy_states_to_run)) + class DCManagerNotifications(RPCClient): """DC Manager Notification interface to broadcast subcloud state changed diff --git a/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_phased_subcloud_deploy.py b/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_phased_subcloud_deploy.py index 0f4dd0ec5..c372250dc 100644 --- a/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_phased_subcloud_deploy.py +++ b/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_phased_subcloud_deploy.py @@ -9,11 +9,13 @@ import copy import json import mock +import os from os import path as os_path import six from tsconfig.tsconfig import SW_VERSION import webtest +from dcmanager.api.controllers.v1 import phased_subcloud_deploy as psd_api from dcmanager.common import consts from dcmanager.common import phased_subcloud_deploy as psd_common from dcmanager.common import utils as dutils @@ -339,7 +341,8 @@ class TestSubcloudDeployInstall(testroot.DCManagerApiTest): subcloud = fake_subcloud.create_fake_subcloud( self.ctx, - deploy_status=consts.DEPLOY_STATE_CREATED) + deploy_status=consts.DEPLOY_STATE_CREATED, + software_version=SW_VERSION) install_data = copy.copy(FAKE_SUBCLOUD_INSTALL_VALUES) install_data.pop('software_version') @@ -371,7 +374,8 @@ class TestSubcloudDeployInstall(testroot.DCManagerApiTest): subcloud = fake_subcloud.create_fake_subcloud( self.ctx, - deploy_status=consts.DEPLOY_STATE_CREATED) + deploy_status=consts.DEPLOY_STATE_CREATED, + software_version=SW_VERSION) install_data = copy.copy(FAKE_SUBCLOUD_INSTALL_VALUES) install_data.pop('software_version') @@ -398,14 +402,13 @@ class TestSubcloudDeployInstall(testroot.DCManagerApiTest): self.assertEqual(response.status_int, 200) self.assertEqual(consts.DEPLOY_STATE_PRE_INSTALL, response.json['deploy-status']) - self.assertEqual(FAKE_SOFTWARE_VERSION, - json.loads(response.json['data_install'])['software_version']) def test_install_subcloud_no_body(self): subcloud = fake_subcloud.create_fake_subcloud( self.ctx, - deploy_status=consts.DEPLOY_STATE_CREATED) + deploy_status=consts.DEPLOY_STATE_CREATED, + software_version=SW_VERSION) self.mock_get_request_data.return_value = {} @@ -419,6 +422,7 @@ class TestSubcloudDeployInstall(testroot.DCManagerApiTest): subcloud = fake_subcloud.create_fake_subcloud( self.ctx, deploy_status=consts.DEPLOY_STATE_CREATED, + software_version=SW_VERSION, data_install='') fake_sysadmin_password = base64.b64encode( @@ -438,7 +442,8 @@ class TestSubcloudDeployInstall(testroot.DCManagerApiTest): subcloud = fake_subcloud.create_fake_subcloud( self.ctx, - deploy_status=consts.DEPLOY_STATE_CREATED) + deploy_status=consts.DEPLOY_STATE_CREATED, + software_version=SW_VERSION) install_data = copy.copy(FAKE_SUBCLOUD_INSTALL_VALUES) install_data.pop('software_version') @@ -500,3 +505,176 @@ class TestSubcloudDeployAbort(testroot.DCManagerApiTest): self.app.patch_json, FAKE_URL + '/' + str(subcloud.id) + '/abort', headers=FAKE_HEADERS) + + +class TestSubcloudDeployResume(testroot.DCManagerApiTest): + def setUp(self): + super().setUp() + self.ctx = utils.dummy_context() + + p = mock.patch.object(rpc_client, 'ManagerClient') + self.mock_rpc_client = p.start() + self.addCleanup(p.stop) + + p = mock.patch.object(dutils, 'get_vault_load_files') + self.mock_get_vault_load_files = p.start() + self.addCleanup(p.stop) + + p = mock.patch.object(psd_common, 'get_subcloud_db_install_values') + self.mock_get_subcloud_db_install_values = p.start() + self.addCleanup(p.stop) + + p = mock.patch.object(psd_common, 'validate_k8s_version') + self.mock_validate_k8s_version = p.start() + self.addCleanup(p.stop) + + p = mock.patch.object(psd_common, 'get_request_data') + self.mock_get_request_data = p.start() + self.addCleanup(p.stop) + + self.management_address_pool = FakeAddressPool('192.168.204.0', 24, + '192.168.204.2', + '192.168.204.100') + + p = mock.patch.object(psd_common, 'get_network_address_pool') + self.mock_get_network_address_pool = p.start() + self.mock_get_network_address_pool.return_value = \ + self.management_address_pool + self.addCleanup(p.stop) + + p = mock.patch.object(psd_common, 'get_ks_client') + self.mock_get_ks_client = p.start() + self.addCleanup(p.stop) + + p = mock.patch.object(psd_common.PatchingClient, 'query') + self.mock_query = p.start() + self.addCleanup(p.stop) + + @mock.patch.object(os_path, 'isdir') + @mock.patch.object(os, 'listdir') + def test_resume_subcloud(self, + mock_os_listdir, + mock_os_isdir): + mock_os_isdir.return_value = True + mock_os_listdir.return_value = ['deploy_chart_fake.tgz', + 'deploy_overrides_fake.yaml', + 'deploy_playbook_fake.yaml'] + + subcloud = fake_subcloud.create_fake_subcloud( + self.ctx, + name=fake_subcloud.FAKE_BOOTSTRAP_FILE_DATA["name"], + deploy_status=consts.DEPLOY_STATE_CREATED, + software_version=SW_VERSION) + install_data = copy.copy(FAKE_SUBCLOUD_INSTALL_VALUES) + install_data.pop('software_version') + + self.mock_get_vault_load_files.return_value = ('iso_file_path', 'sig_file_path') + self.mock_rpc_client().subcloud_deploy_resume.return_value = True + + for state in psd_api.RESUMABLE_STATES: + fake_sysadmin_password = base64.b64encode( + 'testpass'.encode("utf-8")).decode('utf-8') + fake_bmc_password = base64.b64encode( + 'bmc_password'.encode("utf-8")).decode('utf-8') + bmc_password = {'bmc_password': fake_bmc_password} + install_data.update(bmc_password) + install_request = {'install_values': install_data, + 'sysadmin_password': fake_sysadmin_password, + 'bmc_password': fake_bmc_password} + bootstrap_request = {'bootstrap_values': fake_subcloud.FAKE_BOOTSTRAP_FILE_DATA} + config_request = {'deploy_config': 'deploy config values', + 'sysadmin_password': fake_sysadmin_password} + resume_request = {**install_request, + **bootstrap_request, + **config_request} + resume_payload = {**install_request, + **fake_subcloud.FAKE_BOOTSTRAP_FILE_DATA, + **config_request} + + subcloud = db_api.subcloud_update(self.ctx, + subcloud.id, + deploy_status=state) + next_deploy_phase = psd_api.RESUMABLE_STATES[subcloud.deploy_status][0] + next_deploy_state = psd_api.RESUME_PREP_UPDATE_STATUS[next_deploy_phase] + + self.mock_get_request_data.return_value = resume_payload + response = self.app.patch( + FAKE_URL + '/' + str(subcloud.id) + '/resume', + headers=FAKE_HEADERS, params=resume_request) + + self.assertEqual(response.status_int, 200) + self.assertEqual(next_deploy_state, + response.json['deploy-status']) + self.assertEqual(SW_VERSION, response.json['software-version']) + + def test_resume_subcloud_invalid_state(self): + + subcloud = fake_subcloud.create_fake_subcloud( + self.ctx, + name=fake_subcloud.FAKE_BOOTSTRAP_FILE_DATA["name"], + deploy_status=consts.DEPLOY_STATE_CREATED, + software_version=SW_VERSION) + + self.mock_get_vault_load_files.return_value = ('iso_file_path', 'sig_file_path') + self.mock_rpc_client().subcloud_deploy_resume.return_value = True + invalid_resume_states = [consts.DEPLOY_STATE_INSTALLING, + consts.DEPLOY_STATE_BOOTSTRAPPING, + consts.DEPLOY_STATE_CONFIGURING] + + for state in invalid_resume_states: + subcloud = db_api.subcloud_update(self.ctx, + subcloud.id, + deploy_status=state) + + six.assertRaisesRegex(self, webtest.app.AppError, "400 *", + self.app.patch_json, FAKE_URL + '/' + + str(subcloud.id) + '/resume', + headers=FAKE_HEADERS) + + @mock.patch.object(dutils, 'load_yaml_file') + @mock.patch.object(os_path, 'exists') + @mock.patch.object(os_path, 'isdir') + @mock.patch.object(os, 'listdir') + def test_resume_subcloud_no_request_data(self, + mock_os_listdir, + mock_os_isdir, + mock_path_exists, + mock_load_yaml): + subcloud = fake_subcloud.create_fake_subcloud( + self.ctx, + name=fake_subcloud.FAKE_BOOTSTRAP_FILE_DATA["name"], + deploy_status=consts.DEPLOY_STATE_CREATED, + software_version=SW_VERSION) + + config_file = psd_common.get_config_file_path(subcloud.name, + consts.DEPLOY_CONFIG) + mock_path_exists.side_effect = lambda x: True if x == config_file else False + mock_load_yaml.return_value = { + "software_version": fake_subcloud.FAKE_SOFTWARE_VERSION} + mock_os_isdir.return_value = True + mock_os_listdir.return_value = ['deploy_chart_fake.tgz', + 'deploy_overrides_fake.yaml', + 'deploy_playbook_fake.yaml'] + self.mock_get_vault_load_files.return_value = ('iso_file_path', 'sig_file_path') + self.mock_rpc_client().subcloud_deploy_resume.return_value = True + + for state in psd_api.RESUMABLE_STATES: + fake_sysadmin_password = base64.b64encode( + 'testpass'.encode("utf-8")).decode('utf-8') + resume_request = {'sysadmin_password': fake_sysadmin_password} + + subcloud = db_api.subcloud_update(self.ctx, + subcloud.id, + deploy_status=state) + next_deploy_phase = psd_api.RESUMABLE_STATES[subcloud.deploy_status][0] + next_deploy_state = psd_api.RESUME_PREP_UPDATE_STATUS[next_deploy_phase] + + self.mock_get_request_data.return_value = resume_request + response = self.app.patch( + FAKE_URL + '/' + str(subcloud.id) + '/resume', + headers=FAKE_HEADERS, params=resume_request) + + self.assertEqual(response.status_int, 200) + self.assertEqual(next_deploy_state, + response.json['deploy-status']) + self.assertEqual(SW_VERSION, response.json['software-version']) diff --git a/distributedcloud/dcmanager/tests/unit/manager/test_subcloud_manager.py b/distributedcloud/dcmanager/tests/unit/manager/test_subcloud_manager.py index a64c86647..9caeb8bb9 100644 --- a/distributedcloud/dcmanager/tests/unit/manager/test_subcloud_manager.py +++ b/distributedcloud/dcmanager/tests/unit/manager/test_subcloud_manager.py @@ -33,7 +33,6 @@ from dcmanager.common import consts from dcmanager.common import exceptions from dcmanager.common import prestage from dcmanager.common import utils as cutils -from dcmanager.db import api as dc_db_api from dcmanager.db.sqlalchemy import api as db_api from dcmanager.manager import subcloud_manager from dcmanager.state import subcloud_state_manager @@ -428,8 +427,12 @@ class TestSubcloudManager(base.DCManagerTestCase): @mock.patch.object( subcloud_manager.SubcloudManager, 'compose_install_command') - def test_deploy_install_subcloud(self, + @mock.patch.object( + subcloud_manager.SubcloudManager, '_run_subcloud_install') + def test_subcloud_deploy_install(self, + mock_run_subcloud_install, mock_compose_install_command): + mock_run_subcloud_install.return_value = True subcloud_name = 'subcloud1' subcloud = self.create_subcloud_static( @@ -453,6 +456,12 @@ class TestSubcloudManager(base.DCManagerTestCase): sm._get_ansible_filename(subcloud_name, consts.INVENTORY_FILE_POSTFIX), FAKE_PREVIOUS_SW_VERSION) + # Verify subcloud was updated with correct values + updated_subcloud = db_api.subcloud_get_by_name(self.ctx, + subcloud.name) + self.assertEqual(consts.DEPLOY_STATE_INSTALLED, + updated_subcloud.deploy_status) + @mock.patch.object(subcloud_manager.SubcloudManager, '_create_intermediate_ca_cert') @mock.patch.object(cutils, 'delete_subcloud_inventory') @@ -564,9 +573,10 @@ class TestSubcloudManager(base.DCManagerTestCase): self.assertEqual(consts.DEPLOY_STATE_BOOTSTRAPPED, updated_subcloud.deploy_status) - @mock.patch.object(dc_db_api, 'subcloud_get') - def test_subcloud_deploy_bootstrap_failed(self, mock_subcloud_get): - mock_subcloud_get.side_effect = FakeException('boom') + @mock.patch.object(subcloud_manager.SubcloudManager, + '_deploy_bootstrap_prep') + def test_subcloud_deploy_bootstrap_failed(self, mock_bootstrap_prep): + mock_bootstrap_prep.side_effect = FakeException('boom') subcloud = fake_subcloud.create_fake_subcloud( self.ctx, @@ -605,6 +615,66 @@ class TestSubcloudManager(base.DCManagerTestCase): payload=fake_payload) mock_prepare_for_deployment.assert_called_once() + @mock.patch.object(subcloud_manager.SubcloudManager, + '_run_subcloud_install') + @mock.patch.object(subcloud_manager.SubcloudManager, + '_prepare_for_deployment') + @mock.patch.object(cutils, 'create_subcloud_inventory') + @mock.patch.object(subcloud_manager, 'keyring') + @mock.patch.object(cutils, 'get_playbook_for_software_version') + @mock.patch.object(cutils, 'update_values_on_yaml_file') + @mock.patch.object(RunAnsible, 'exec_playbook') + def test_subcloud_deploy_resume(self, mock_exec_playbook, mock_update_yml, + mock_get_playbook_for_software_version, + mock_keyring, create_subcloud_inventory, + mock_prepare_for_deployment, + mock_run_subcloud_install): + mock_get_playbook_for_software_version.return_value = "22.12" + mock_keyring.get_password.return_value = "testpass" + mock_exec_playbook.return_value = False + mock_run_subcloud_install.return_value = True + + subcloud = self.create_subcloud_static( + self.ctx, + name='subcloud1', + deploy_status=consts.DEPLOY_STATE_CREATED) + + deploy_states_to_run = [consts.DEPLOY_PHASE_INSTALL, + consts.DEPLOY_PHASE_BOOTSTRAP, + consts.DEPLOY_PHASE_CONFIG] + + fake_install_values = \ + copy.copy(fake_subcloud.FAKE_SUBCLOUD_INSTALL_VALUES) + fake_install_values['software_version'] = SW_VERSION + fake_payload_install = {'bmc_password': 'bmc_pass', + 'install_values': fake_install_values, + 'software_version': SW_VERSION, + 'sysadmin_password': 'sys_pass'} + + fake_payload_bootstrap = {**fake_subcloud.FAKE_BOOTSTRAP_VALUE, + **fake_subcloud.FAKE_BOOTSTRAP_FILE_DATA} + fake_payload_bootstrap["sysadmin_password"] = "testpass" + + fake_payload_config = {"sysadmin_password": "testpass", + "deploy_playbook": "test_playbook.yaml", + "deploy_overrides": "test_overrides.yaml", + "deploy_chart": "test_chart.yaml", + "deploy_config": "subcloud1.yaml"} + + fake_payload = {**fake_payload_install, + **fake_payload_bootstrap, + **fake_payload_config} + + sm = subcloud_manager.SubcloudManager() + sm.subcloud_deploy_resume(self.ctx, subcloud.id, subcloud.name, + fake_payload, deploy_states_to_run) + + # Verify subcloud was updated with correct values + updated_subcloud = db_api.subcloud_get_by_name(self.ctx, + subcloud.name) + self.assertEqual(consts.DEPLOY_STATE_DONE, + updated_subcloud.deploy_status) + @mock.patch.object(subcloud_manager.SubcloudManager, 'compose_apply_command') @mock.patch.object(subcloud_manager.SubcloudManager,