From f081453b3c47a6d7e8d029f0bacf8cdabe631ac7 Mon Sep 17 00:00:00 2001 From: Victor Romano Date: Mon, 29 May 2023 17:45:20 -0300 Subject: [PATCH] Add subcloud deploy abort option to dcmanager This commit adds the command "subcloud deploy abort" to dcmanager. It allows the user to abort the on-going execution of a playbook against the subcloud. Any task will be aborted immediately if the unabortable flag is not set by the playbook [1]. If current operation is install, a shutdown command will also be issued. Test Plan: Success cases: - PASS: Trigger an abort during installation and verify that the playbook execution was aborted immediately, the subcloud was shut dows after it and the RVMC pod and job were terminated. - PASS: Trigger an abort during config and verify that the playbook execution was aborted immediately. - PASS: Trigger an abort during bootstrap without the presence of unabortable flag and verify that the playbook execution was aborted immidiately. - PASS: Trigger an abort during bootstrap with the presence of unabortable flag and verify that the playbook execution was aborted only after the flag was deleted. - PASS: Trigger an abort directly calling the API (using CURL instead of using the CLI. Failure cases: - PASS: Verify that the abort request is rejected if deploy state is not 'installing', 'bootstrapping' or 'configuring'. - PASS: Abort when an unabortable task is running and then force an external error during this task restarting dcmanager-manager service, verify that deploy state is set to a failed state e.g. bootstrap-failed. - PASS: Abort when an unabortable task is running and then force an internal error during this task using ansible.builtin.fail module, verify that deploy state is set to a failed state e.g. bootstrap-failed. - PASS: Abort when an unabortable task is running and then force the playbook to halt the execution during this task using ansible.builtin.pause module, verify that deploy state is set to a failed state after 10 minutes e.g. bootstrap-failed. Story: 2010756 Task: 48102 Co-Authored-By: Gustavo Herzmann Signed-off-by: Victor Romano Change-Id: Ic5311324a76bf7ce1215692e934d5577ff82868e --- api-ref/source/api-ref-dcmanager-v1.rst | 67 +++++++ ...d-subcloud-deploy-patch-abort-request.json | 3 + ...-subcloud-deploy-patch-abort-response.json | 23 +++ distributedcloud/dccommon/consts.py | 5 + distributedcloud/dccommon/exceptions.py | 4 + distributedcloud/dccommon/kubeoperator.py | 57 +++++- distributedcloud/dccommon/subcloud_install.py | 122 +++++++++++- distributedcloud/dccommon/utils.py | 154 ++++++++++++++- .../controllers/v1/phased_subcloud_deploy.py | 44 ++++- .../dcmanager/api/controllers/v1/subclouds.py | 6 +- .../api/policies/phased_subcloud_deploy.py | 4 + distributedcloud/dcmanager/common/consts.py | 8 +- .../common/phased_subcloud_deploy.py | 2 +- distributedcloud/dcmanager/common/utils.py | 35 +++- distributedcloud/dcmanager/manager/service.py | 11 +- .../dcmanager/manager/subcloud_manager.py | 184 ++++++++++++------ .../states/upgrade/migrating_data.py | 3 +- .../states/upgrade/upgrading_simplex.py | 6 +- distributedcloud/dcmanager/rpc/client.py | 7 +- .../test_phased_subcloud_deploy.py | 37 ++++ .../v1/controllers/test_subcloud_deploy.py | 7 +- .../unit/api/v1/controllers/test_subclouds.py | 6 +- .../unit/manager/test_subcloud_manager.py | 47 ++--- 23 files changed, 720 insertions(+), 122 deletions(-) create mode 100644 api-ref/source/samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-abort-request.json create mode 100644 api-ref/source/samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-abort-response.json diff --git a/api-ref/source/api-ref-dcmanager-v1.rst b/api-ref/source/api-ref-dcmanager-v1.rst index 995f778c3..36fb77ce8 100644 --- a/api-ref/source/api-ref-dcmanager-v1.rst +++ b/api-ref/source/api-ref-dcmanager-v1.rst @@ -2053,4 +2053,71 @@ Response Example ---------------- .. literalinclude:: samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-configure-response.json + :language: json + + +********************************** +Abort subcloud deployment +********************************** + +.. rest_method:: PATCH /v1.0/phased-subcloud-deploy/{subcloud}/abort + +**Normal response codes** + +200 + +**Error response codes** + +badRequest (400), unauthorized (401), forbidden (403), badMethod (405), +HTTPUnprocessableEntity (422), internalServerError (500), +serviceUnavailable (503) + +**Request parameters** + +.. rest_parameters:: parameters.yaml + + - subcloud: subcloud_uri + +Accepts Content-Type multipart/form-data + +Request Example +---------------- + +.. literalinclude:: samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-abort-request.json + :language: json + +**Response parameters** + +.. rest_parameters:: parameters.yaml + + - id: subcloud_id + - group_id: group_id + - name: subcloud_name + - description: subcloud_description + - location: subcloud_location + - software-version: software_version + - availability-status: availability_status + - error-description: error_description + - deploy-status: deploy_status + - backup-status: backup_status + - backup-datetime: backup_datetime + - openstack-installed: openstack_installed + - management-state: management_state + - systemcontroller-gateway-ip: systemcontroller_gateway_ip + - management-start-ip: management_start_ip + - management-end-ip: management_end_ip + - management-subnet: management_subnet + - management-gateway-ip: management_gateway_ip + - created-at: created_at + - updated-at: updated_at + - data_install: data_install + - data_upgrade: data_upgrade + - endpoint_sync_status: endpoint_sync_status + - sync_status: sync_status + - endpoint_type: sync_status_type + +Response Example +---------------- + +.. literalinclude:: samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-abort-response.json :language: json \ No newline at end of file diff --git a/api-ref/source/samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-abort-request.json b/api-ref/source/samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-abort-request.json new file mode 100644 index 000000000..e4f22488d --- /dev/null +++ b/api-ref/source/samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-abort-request.json @@ -0,0 +1,3 @@ +{ + "subcloud": "subcloud1" +} \ No newline at end of file diff --git a/api-ref/source/samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-abort-response.json b/api-ref/source/samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-abort-response.json new file mode 100644 index 000000000..18b325132 --- /dev/null +++ b/api-ref/source/samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-abort-response.json @@ -0,0 +1,23 @@ +{ + "id": 1, + "name": "subcloud1", + "created-at": "2023-01-02T03:04:05.678987", + "updated-at": "2023-04-08T15:16:23.424851", + "availability-status": "offline", + "data_install": null, + "data_upgrade": null, + "deploy-status": "aborting-install", + "backup-status": null, + "backup-datetime": null, + "description": "Ottawa Site", + "group_id": 1, + "location": "YOW", + "management-end-ip": "192.168.101.50", + "management-gateway-ip": "192.168.101.1", + "management-start-ip": "192.168.101.2", + "management-state": "unmanaged", + "management-subnet": "192.168.101.0/24", + "openstack-installed": false, + "software-version": "22.12", + "systemcontroller-gateway-ip": "192.168.204.101" +} \ No newline at end of file diff --git a/distributedcloud/dccommon/consts.py b/distributedcloud/dccommon/consts.py index 853cc6a24..a4102c2e7 100644 --- a/distributedcloud/dccommon/consts.py +++ b/distributedcloud/dccommon/consts.py @@ -28,6 +28,7 @@ CLOUD_0 = "RegionOne" VIRTUAL_MASTER_CLOUD = "SystemController" SW_UPDATE_DEFAULT_TITLE = "all clouds default" +ANSIBLE_OVERRIDES_PATH = '/opt/dc-vault/ansible' LOAD_VAULT_DIR = '/opt/dc-vault/loads' DEPLOY_DIR = '/opt/platform/deploy' @@ -140,6 +141,10 @@ CERT_CA_FILE_CENTOS = "ca-cert.pem" CERT_CA_FILE_DEBIAN = "ca-cert.crt" SSL_CERT_CA_DIR = "/etc/pki/ca-trust/source/anchors/" +# RVMC +RVMC_NAME_PREFIX = 'rvmc' +RVMC_CONFIG_FILE_NAME = 'rvmc-config.yaml' + # Subcloud installation values BMC_INSTALL_VALUES = [ 'bmc_username', diff --git a/distributedcloud/dccommon/exceptions.py b/distributedcloud/dccommon/exceptions.py index 5ed8d7bf9..27a792eca 100644 --- a/distributedcloud/dccommon/exceptions.py +++ b/distributedcloud/dccommon/exceptions.py @@ -119,3 +119,7 @@ class ImageNotInLocalRegistry(NotFound): message = _("Image %(image_name)s:%(image_tag)s not found in the local registry. " "Please check with command: system registry-image-list or " "system registry-image-tags %(image_name)s") + + +class SubcloudShutdownError(PlaybookExecutionFailed): + message = _("Subcloud %(subcloud_name)s could not be shut down.") diff --git a/distributedcloud/dccommon/kubeoperator.py b/distributedcloud/dccommon/kubeoperator.py index 2bfd9e416..d16697b66 100644 --- a/distributedcloud/dccommon/kubeoperator.py +++ b/distributedcloud/dccommon/kubeoperator.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2020-2022 Wind River Systems, Inc. +# Copyright (c) 2020-2023 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -149,3 +149,58 @@ class KubeOperator(object): if e.status != httplib.NOT_FOUND: LOG.error("Fail to delete %s:%s. %s" % (namespace, name, e)) raise + + def get_pods_by_namespace(self, namespace): + c = self._get_kubernetesclient_core() + try: + pods = c.list_namespaced_pod(namespace) + return [pod.metadata.name for pod in pods.items] + except ApiException as e: + if e.status == httplib.NOT_FOUND: + return [] + LOG.error("Failed to get pod name under " + "Namespace %s." % (namespace)) + raise + except Exception as e: + LOG.error("Kubernetes exception in get_pods_by_namespace: %s" % e) + raise + + def pod_exists(self, pod, namespace): + pods = self.get_pods_by_namespace(namespace) + if pod in pods: + return True + return False + + def kube_delete_job(self, name, namespace, **kwargs): + body = {} + if kwargs: + body.update(kwargs) + + b = self._get_kubernetesclient_batch() + try: + b.delete_namespaced_job(name, namespace, body=body) + except ApiException as e: + if e.status != httplib.NOT_FOUND: + LOG.error("Failed to delete job %s under " + "Namespace %s: %s" % (name, namespace, e.body)) + raise + except Exception as e: + LOG.error("Kubernetes exception in kube_delete_job: %s" % e) + raise + + def kube_delete_pod(self, name, namespace, **kwargs): + body = {} + if kwargs: + body.update(kwargs) + + c = self._get_kubernetesclient_core() + try: + c.delete_namespaced_pod(name, namespace, body=body) + except ApiException as e: + if e.status != httplib.NOT_FOUND: + LOG.error("Failed to delete pod %s under " + "Namespace %s: %s" % (name, namespace, e.body)) + raise + except Exception as e: + LOG.error("Kubernetes exception in kube_delete_pod: %s" % e) + raise diff --git a/distributedcloud/dccommon/subcloud_install.py b/distributedcloud/dccommon/subcloud_install.py index 3f9338c2f..d1263231b 100644 --- a/distributedcloud/dccommon/subcloud_install.py +++ b/distributedcloud/dccommon/subcloud_install.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021-2022 Wind River Systems, Inc. +# Copyright (c) 2021-2023 Wind River Systems, Inc. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -13,10 +13,13 @@ # limitations under the License. # +import json import os import shutil import socket +import ssl import tempfile +import time from eventlet.green import subprocess import netaddr @@ -24,13 +27,14 @@ from oslo_log import log as logging from six.moves.urllib import error as urllib_error from six.moves.urllib import parse from six.moves.urllib import request +import yaml from dccommon import consts from dccommon.drivers.openstack.keystone_v3 import KeystoneClient from dccommon.drivers.openstack.sysinv_v1 import SysinvClient from dccommon import exceptions from dccommon import utils as dccommon_utils -from dcmanager.common import consts as common_consts +from dcmanager.common import consts as dcmanager_consts from dcmanager.common import utils LOG = logging.getLogger(__name__) @@ -39,7 +43,6 @@ BOOT_MENU_TIMEOUT = '5' # The RVMC_IMAGE_NAME:RVMC_IMAGE_TAG must align with the one specified # in system images in the ansible install/upgrade playbook -RVMC_NAME_PREFIX = 'rvmc' RVMC_IMAGE_NAME = 'docker.io/starlingx/rvmc' RVMC_IMAGE_TAG = 'stx.8.0-v1.0.1' @@ -55,6 +58,99 @@ NETWORK_INTERFACE_PREFIX = 'ifcfg' NETWORK_ROUTE_PREFIX = 'route' LOCAL_REGISTRY_PREFIX = 'registry.local:9001/' +# Redfish constants +ACTION_URL = '/Actions/ComputerSystem.Reset' +POWER_OFF_PAYLOAD = {'Action': 'Reset', 'ResetType': 'ForceOff'} +REDFISH_HEADER = {'Content-Type': 'application/json', + 'Accept': 'application/json'} +REDFISH_SYSTEMS_URL = '/redfish/v1/Systems' +SUCCESSFUL_STATUS_CODES = [200, 202, 204] + + +class SubcloudShutdown(object): + """Sends a shutdown signal to a Redfish controlled subcloud + + Approach: + + To shutdown a Redfish controlled subcloud, it's needed to first + send a GET request to find the @odata.id of the member, and then + send a POST request with the shutdown signal. Since this is + intended as a way to turn off the subcloud during the deploy abort + process, only the ForceOff option is considered. + """ + def __init__(self, subcloud_name): + self.target = subcloud_name + self.rvmc_data = self._get_subcloud_data() + + def _get_subcloud_data(self): + rvmc_config_file_path = os.path.join(consts.ANSIBLE_OVERRIDES_PATH, + self.target, consts.RVMC_CONFIG_FILE_NAME) + if not os.path.isfile(rvmc_config_file_path): + raise Exception('Missing rvmc files for %s' % self.target) + with open(os.path.abspath(rvmc_config_file_path), 'r') as f: + rvmc_data = f.read() + rvmc_config_values = yaml.load(rvmc_data, Loader=yaml.SafeLoader) + base_url = "https://" + rvmc_config_values['bmc_address'] + bmc_username = rvmc_config_values['bmc_username'] + bmc_password = rvmc_config_values['bmc_password'] + credentials = ("%s:%s" % (bmc_username.rstrip(), bmc_password)).encode("utf-8") + return {'base_url': base_url, 'credentials': credentials} + + def _make_request(self, url, credentials, method, retry=5): + if method == 'get': + payload = None + else: + payload = json.dumps(POWER_OFF_PAYLOAD).encode('utf-8') + + try: + context = ssl._create_unverified_context() + req = request.Request(url, headers=REDFISH_HEADER, method=method) + req.add_header('Authorization', 'Basic %s' % credentials) + response = request.urlopen(req, data=payload, context=context) + status_code = response.getcode() + + if status_code not in SUCCESSFUL_STATUS_CODES: + if retry <= 0: + raise exceptions.SubcloudShutdownError( + subcloud_name=self.target) + retry -= retry + time.sleep(2) + self._make_request(url, credentials, method, retry=retry) + except urllib_error.URLError: + # This occurs when the BMC is not available anymore, + # so we just ignore it. + return None + except Exception as ex: + raise ex + + return response + + def _get_data_id(self): + base_url = self.rvmc_data['base_url'] + credentials = self.rvmc_data['credentials'] + url = base_url + REDFISH_SYSTEMS_URL + response = self._make_request(url, credentials, method='GET') + if not response: + return None + r = json.loads(response.read().decode()) + + for member in r['Members']: + if member.get('@odata.id'): + url_with_id = member['@odata.id'] + break + + return url_with_id + + def send_shutdown_signal(self): + base_url = self.rvmc_data['base_url'] + credentials = self.rvmc_data['credentials'] + url_with_id = self._get_data_id() + if not url_with_id: + return None + url = base_url + url_with_id + ACTION_URL + response = self._make_request(url, credentials, method='POST') + return response + class SubcloudInstall(object): """Class to encapsulate the subcloud install operations""" @@ -169,7 +265,7 @@ class SubcloudInstall(object): LOG.debug("create rvmc config file, path: %s, payload: %s", override_path, payload) - rvmc_config_file = os.path.join(override_path, 'rvmc-config.yaml') + rvmc_config_file = os.path.join(override_path, consts.RVMC_CONFIG_FILE_NAME) with open(rvmc_config_file, 'w') as f_out_rvmc_config_file: for k, v in payload.items(): @@ -184,7 +280,7 @@ class SubcloudInstall(object): RVMC_IMAGE_TAG install_override_file = os.path.join(override_path, 'install_values.yml') - rvmc_name = "%s-%s" % (RVMC_NAME_PREFIX, self.name) + rvmc_name = "%s-%s" % (consts.RVMC_NAME_PREFIX, self.name) host_name = socket.gethostname() with open(install_override_file, 'w') as f_out_override_file: @@ -600,16 +696,26 @@ class SubcloudInstall(object): # create the install override file self.create_install_override_file(override_path, payload) - def install(self, log_file_dir, install_command): + def install(self, log_file_dir, install_command, abortable=False): LOG.info("Start remote install %s", self.name) log_file = os.path.join(log_file_dir, self.name) + '_playbook_output.log' try: # Since this is a long-running task we want to register # for cleanup on process restart/SWACT. - dccommon_utils.run_playbook(log_file, install_command) + if abortable: + # Install phase of subcloud deployment + run_ansible = dccommon_utils.RunAnsible() + aborted = run_ansible.exec_playbook(log_file, install_command, self.name) + # Returns True if the playbook was aborted and False otherwise + return aborted + else: + dccommon_utils.run_playbook(log_file, install_command) + # Always return false because this playbook execution + # method cannot be aborted + return False except exceptions.PlaybookExecutionFailed: msg = ("Failed to install %s, check individual " "log at %s or run %s for details" - % (self.name, log_file, common_consts.ERROR_DESC_CMD)) + % (self.name, log_file, dcmanager_consts.ERROR_DESC_CMD)) raise Exception(msg) diff --git a/distributedcloud/dccommon/utils.py b/distributedcloud/dccommon/utils.py index bc2d990a1..42a0d7de4 100644 --- a/distributedcloud/dccommon/utils.py +++ b/distributedcloud/dccommon/utils.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2022 Wind River Systems, Inc. +# Copyright (c) 2020-2023 Wind River Systems, Inc. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -19,6 +19,8 @@ import functools import os import random import re +import threading +import time from eventlet.green import subprocess from oslo_log import log as logging @@ -28,6 +30,7 @@ from dccommon import consts from dccommon import exceptions from dccommon.exceptions import PlaybookExecutionFailed from dccommon.exceptions import PlaybookExecutionTimeout +from dccommon.subprocess_cleanup import kill_subprocess_group from dccommon.subprocess_cleanup import SubprocessCleanup from dcorch.common.i18n import _ @@ -86,6 +89,153 @@ class memoized(object): return functools.partial(self.__call__, obj) +class RunAnsible(object): + """Class to run Ansible playbooks with the abort option + + Approach: + + At the start of the playbook execution, the abort status + (default value is False) and PID of the subprocess for the + specified subcloud are set on the class variable dict (abort_status). + When the user sends the abort command, the subcloud_manager changes + the abort status to True and the subprocess is killed. + + If Ansible is currently executing a task that cannot be interrupted, + a deploy_not_abortable flag is created in the overrides folder by the + playbook itself, and the abort process will wait for said flag to be + deleted before killing the subprocess. If the task fails while abort + is waiting, the playbook_failed flag will indicate to the + original process to raise PlaybookExecutionFailed. + """ + abort_status = {} + lock = threading.Lock() + + def _unregister_subcloud(self, subcloud_name): + with RunAnsible.lock: + if RunAnsible.abort_status.get(subcloud_name): + del RunAnsible.abort_status[subcloud_name] + + def run_abort(self, subcloud_name, timeout=600): + """Set abort status for a subcloud. + + :param subcloud_name: Name of the subcloud + param timeout: Timeout in seconds. + """ + with RunAnsible.lock: + RunAnsible.abort_status[subcloud_name]['abort'] = True + unabortable_flag = os.path.join(consts.ANSIBLE_OVERRIDES_PATH, + '.%s_deploy_not_abortable' % subcloud_name) + subp = RunAnsible.abort_status[subcloud_name]['subp'] + while os.path.exists(unabortable_flag) and timeout > 0: + time.sleep(1) + timeout -= 1 + kill_subprocess_group(subp) + return True + + def exec_playbook(self, log_file, playbook_command, subcloud_name, + timeout=None, register_cleanup=True): + """Run ansible playbook via subprocess. + + :param log_file: Logs output to file + :param timeout: Timeout in seconds. Raises PlaybookExecutionTimeout + on timeout + :param register_cleanup: Register the subprocess group for cleanup on + shutdown, if the underlying service supports cleanup. + """ + exec_env = os.environ.copy() + exec_env["ANSIBLE_LOG_PATH"] = "/dev/null" + + aborted = False + + if timeout: + timeout_log_str = " (timeout: %ss)" % timeout + else: + timeout_log_str = '' + + with open(log_file, "a+") as f_out_log: + try: + logged_playbook_command = \ + _strip_password_from_command(playbook_command) + txt = "%s Executing playbook command%s: %s\n" \ + % (datetime.today().strftime('%Y-%m-%d-%H:%M:%S'), + timeout_log_str, + logged_playbook_command) + f_out_log.write(txt) + f_out_log.flush() + + # Remove unabortable flag created by the playbook + # if present from previous executions + unabortable_flag = os.path.join(consts.ANSIBLE_OVERRIDES_PATH, + '.%s_deploy_not_abortable' % subcloud_name) + if os.path.exists(unabortable_flag): + os.remove(unabortable_flag) + + subp = subprocess.Popen(playbook_command, + stdout=f_out_log, + stderr=f_out_log, + env=exec_env, + start_new_session=register_cleanup) + try: + if register_cleanup: + SubprocessCleanup.register_subprocess_group(subp) + with RunAnsible.lock: + RunAnsible.abort_status[subcloud_name] = { + 'abort': False, + 'subp': subp} + + subp.wait(timeout) + subp_rc = subp.poll() + + # There are 5 possible outcomes of the subprocess execution: + # 1: Playbook completed (process exited) + # - playbook_failure is False with subp_rc == 0, + # aborted is False, unabortable_flag_exists is False + # 2: Playbook was aborted (process killed) + # - playbook_failure is False with subp_rc != 0, + # aborted is True, unabortable_flag_exists is False + # 3: Playbook failed (process exited) + # - playbook_failure is True with subp_rc != 0, + # aborted is False, unabortable_flag_exists is False + # 4: Playbook failed during unabortable task (process exited) + # - playbook_failure is True with subp_rc != 0, + # aborted is False, unabortable_flag_exists is True + # 5: Playbook failed while waiting to be aborted (process exited) + # - playbook_failure is True with subp_rc != 0, + # aborted is True, unabortable_flag_exists is False + with RunAnsible.lock: + aborted = RunAnsible.abort_status[subcloud_name]['abort'] + unabortable_flag_exists = os.path.exists(unabortable_flag) + playbook_failure = (subp_rc != 0 and + (not aborted or unabortable_flag_exists)) + + # Raise PlaybookExecutionFailed if the playbook fails when + # on normal conditions (no abort issued) or fails while + # waiting for the unabortable flag to be cleared. + if playbook_failure: + raise PlaybookExecutionFailed(playbook_cmd=playbook_command) + + except subprocess.TimeoutExpired: + kill_subprocess_group(subp) + f_out_log.write( + "%s TIMEOUT (%ss) - playbook is terminated\n" % + (datetime.today().strftime('%Y-%m-%d-%H:%M:%S'), timeout) + ) + raise PlaybookExecutionTimeout(playbook_cmd=playbook_command, + timeout=timeout) + finally: + f_out_log.flush() + if register_cleanup: + SubprocessCleanup.unregister_subprocess_group(subp) + self._unregister_subcloud(subcloud_name) + + except PlaybookExecutionFailed: + raise + except Exception as ex: + LOG.error(str(ex)) + raise + return aborted + + def _strip_password_from_command(script_command): """Strip out any known password arguments from given command""" logged_command = list() @@ -105,6 +255,8 @@ def _strip_password_from_command(script_command): return logged_command +# TODO(vgluzrom): remove this function and replace all calls +# with RunAnsible class def run_playbook(log_file, playbook_command, timeout=None, register_cleanup=True): """Run ansible playbook via subprocess. diff --git a/distributedcloud/dcmanager/api/controllers/v1/phased_subcloud_deploy.py b/distributedcloud/dcmanager/api/controllers/v1/phased_subcloud_deploy.py index b84ca06e9..6d7dbaecf 100644 --- a/distributedcloud/dcmanager/api/controllers/v1/phased_subcloud_deploy.py +++ b/distributedcloud/dcmanager/api/controllers/v1/phased_subcloud_deploy.py @@ -61,11 +61,13 @@ VALID_STATES_FOR_DEPLOY_INSTALL = ( consts.DEPLOY_STATE_CREATED, consts.DEPLOY_STATE_PRE_INSTALL_FAILED, consts.DEPLOY_STATE_INSTALL_FAILED, - consts.DEPLOY_STATE_INSTALLED + consts.DEPLOY_STATE_INSTALLED, + consts.DEPLOY_STATE_INSTALL_ABORTED ) VALID_STATES_FOR_DEPLOY_BOOTSTRAP = [ consts.DEPLOY_STATE_INSTALLED, + consts.DEPLOY_STATE_PRE_BOOTSTRAP_FAILED, consts.DEPLOY_STATE_BOOTSTRAP_FAILED, consts.DEPLOY_STATE_BOOTSTRAP_ABORTED, consts.DEPLOY_STATE_BOOTSTRAPPED, @@ -81,7 +83,14 @@ VALID_STATES_FOR_DEPLOY_CONFIG = ( consts.DEPLOY_STATE_PRE_CONFIG_FAILED, consts.DEPLOY_STATE_CONFIG_FAILED, consts.DEPLOY_STATE_DEPLOY_FAILED, - consts.DEPLOY_STATE_BOOTSTRAPPED + consts.DEPLOY_STATE_BOOTSTRAPPED, + consts.DEPLOY_STATE_CONFIG_ABORTED +) + +VALID_STATES_FOR_DEPLOY_ABORT = ( + consts.DEPLOY_STATE_INSTALLING, + consts.DEPLOY_STATE_BOOTSTRAPPING, + consts.DEPLOY_STATE_CONFIGURING ) @@ -288,15 +297,38 @@ class PhasedSubcloudDeployController(object): psd_common.validate_sysadmin_password(payload) try: - subcloud = self.dcmanager_rpc_client.subcloud_deploy_config( + self.dcmanager_rpc_client.subcloud_deploy_config( context, subcloud.id, payload) - return subcloud + subcloud_dict = db_api.subcloud_db_model_to_dict(subcloud) + subcloud_dict['deploy-status'] = consts.DEPLOY_STATE_PRE_CONFIG + return subcloud_dict except RemoteError as e: pecan.abort(422, e.value) except Exception: LOG.exception("Unable to configure subcloud %s" % subcloud.name) pecan.abort(500, _('Unable to configure subcloud')) + def _deploy_abort(self, context, subcloud): + + if subcloud.deploy_status not in VALID_STATES_FOR_DEPLOY_ABORT: + allowed_states_str = ', '.join(VALID_STATES_FOR_DEPLOY_ABORT) + pecan.abort(400, _('Subcloud deploy status must be in one ' + 'of the following states: %s') + % allowed_states_str) + + try: + self.dcmanager_rpc_client.subcloud_deploy_abort( + context, subcloud.id, subcloud.deploy_status) + subcloud_dict = db_api.subcloud_db_model_to_dict(subcloud) + subcloud_dict['deploy-status'] = \ + utils.ABORT_UPDATE_STATUS[subcloud.deploy_status] + return subcloud_dict + except RemoteError as e: + pecan.abort(422, e.value) + except Exception: + LOG.exception("Unable to abort subcloud %s deployment" % subcloud.name) + pecan.abort(500, _('Unable to abort subcloud deploy')) + @pecan.expose(generic=True, template='json') def index(self): # Route the request to specific methods with parameters @@ -334,7 +366,9 @@ class PhasedSubcloudDeployController(object): except (exceptions.SubcloudNotFound, exceptions.SubcloudNameNotFound): pecan.abort(404, _('Subcloud not found')) - if verb == 'install': + if verb == 'abort': + subcloud = self._deploy_abort(context, subcloud) + elif verb == 'install': subcloud = self._deploy_install(context, pecan.request, subcloud) elif verb == 'bootstrap': subcloud = self._deploy_bootstrap(context, pecan.request, subcloud) diff --git a/distributedcloud/dcmanager/api/controllers/v1/subclouds.py b/distributedcloud/dcmanager/api/controllers/v1/subclouds.py index b49db8df4..ef2f7b6fb 100644 --- a/distributedcloud/dcmanager/api/controllers/v1/subclouds.py +++ b/distributedcloud/dcmanager/api/controllers/v1/subclouds.py @@ -281,17 +281,17 @@ class SubcloudsController(object): def _get_config_file_path(self, subcloud_name, config_file_type=None): if config_file_type == consts.DEPLOY_CONFIG: file_path = os.path.join( - consts.ANSIBLE_OVERRIDES_PATH, + dccommon_consts.ANSIBLE_OVERRIDES_PATH, subcloud_name + '_' + config_file_type + '.yml' ) elif config_file_type == INSTALL_VALUES: file_path = os.path.join( - consts.ANSIBLE_OVERRIDES_PATH + '/' + subcloud_name, + dccommon_consts.ANSIBLE_OVERRIDES_PATH + '/' + subcloud_name, config_file_type + '.yml' ) else: file_path = os.path.join( - consts.ANSIBLE_OVERRIDES_PATH, + dccommon_consts.ANSIBLE_OVERRIDES_PATH, subcloud_name + '.yml' ) return file_path diff --git a/distributedcloud/dcmanager/api/policies/phased_subcloud_deploy.py b/distributedcloud/dcmanager/api/policies/phased_subcloud_deploy.py index fb848bb78..fec59229e 100644 --- a/distributedcloud/dcmanager/api/policies/phased_subcloud_deploy.py +++ b/distributedcloud/dcmanager/api/policies/phased_subcloud_deploy.py @@ -27,6 +27,10 @@ phased_subcloud_deploy_rules = [ check_str='rule:' + base.ADMIN_IN_SYSTEM_PROJECTS, description="Modify the subcloud deployment.", operations=[ + { + 'method': 'PATCH', + 'path': '/v1.0/phased-subcloud-deploy/{subcloud}/abort' + }, { 'method': 'PATCH', 'path': '/v1.0/phased-subcloud-deploy/{subcloud}/install' diff --git a/distributedcloud/dcmanager/common/consts.py b/distributedcloud/dcmanager/common/consts.py index f94c7b3ba..f6bb1d6e8 100644 --- a/distributedcloud/dcmanager/common/consts.py +++ b/distributedcloud/dcmanager/common/consts.py @@ -185,6 +185,12 @@ DEPLOY_STATE_CONFIGURING = 'configuring' DEPLOY_STATE_CONFIG_FAILED = 'config-failed' DEPLOY_STATE_DEPLOYING = 'deploying' DEPLOY_STATE_DEPLOY_FAILED = 'deploy-failed' +DEPLOY_STATE_ABORTING_INSTALL = 'aborting-install' +DEPLOY_STATE_INSTALL_ABORTED = 'install-aborted' +DEPLOY_STATE_ABORTING_BOOTSTRAP = 'aborting-bootstrap' +DEPLOY_STATE_BOOTSTRAP_ABORTED = 'bootstrap-aborted' +DEPLOY_STATE_ABORTING_CONFIG = 'aborting-config' +DEPLOY_STATE_CONFIG_ABORTED = 'config-aborted' DEPLOY_STATE_MIGRATING_DATA = 'migrating-data' DEPLOY_STATE_DATA_MIGRATION_FAILED = 'data-migration-failed' DEPLOY_STATE_MIGRATED = 'migrated' @@ -281,8 +287,6 @@ ALARM_OK_STATUS = "OK" ALARM_DEGRADED_STATUS = "degraded" ALARM_CRITICAL_STATUS = "critical" -# subcloud deploy file options -ANSIBLE_OVERRIDES_PATH = '/opt/dc-vault/ansible' DEPLOY_PLAYBOOK = "deploy_playbook" DEPLOY_OVERRIDES = "deploy_overrides" DEPLOY_CHART = "deploy_chart" diff --git a/distributedcloud/dcmanager/common/phased_subcloud_deploy.py b/distributedcloud/dcmanager/common/phased_subcloud_deploy.py index 39c2193ec..e1cc7b24f 100644 --- a/distributedcloud/dcmanager/common/phased_subcloud_deploy.py +++ b/distributedcloud/dcmanager/common/phased_subcloud_deploy.py @@ -692,7 +692,7 @@ def upload_deploy_config_file(request, payload): def get_config_file_path(subcloud_name, config_file_type=None): - basepath = consts.ANSIBLE_OVERRIDES_PATH + basepath = dccommon_consts.ANSIBLE_OVERRIDES_PATH if config_file_type == consts.DEPLOY_CONFIG: filename = f"{subcloud_name}_{config_file_type}.yml" elif config_file_type == consts.INSTALL_VALUES: diff --git a/distributedcloud/dcmanager/common/utils.py b/distributedcloud/dcmanager/common/utils.py index eec88db80..15ed0cef6 100644 --- a/distributedcloud/dcmanager/common/utils.py +++ b/distributedcloud/dcmanager/common/utils.py @@ -54,6 +54,21 @@ DC_MANAGER_GRPNAME = "root" # Max lines output msg from logs MAX_LINES_MSG = 10 +ABORT_UPDATE_STATUS = { + consts.DEPLOY_STATE_INSTALLING: consts.DEPLOY_STATE_ABORTING_INSTALL, + consts.DEPLOY_STATE_BOOTSTRAPPING: consts.DEPLOY_STATE_ABORTING_BOOTSTRAP, + consts.DEPLOY_STATE_CONFIGURING: consts.DEPLOY_STATE_ABORTING_CONFIG, + consts.DEPLOY_STATE_ABORTING_INSTALL: consts.DEPLOY_STATE_INSTALL_ABORTED, + consts.DEPLOY_STATE_ABORTING_BOOTSTRAP: consts.DEPLOY_STATE_BOOTSTRAP_ABORTED, + consts.DEPLOY_STATE_ABORTING_CONFIG: consts.DEPLOY_STATE_CONFIG_ABORTED +} + +ABORT_UPDATE_FAIL_STATUS = { + consts.DEPLOY_STATE_ABORTING_INSTALL: consts.DEPLOY_STATE_INSTALL_FAILED, + consts.DEPLOY_STATE_ABORTING_BOOTSTRAP: consts.DEPLOY_STATE_BOOTSTRAP_FAILED, + consts.DEPLOY_STATE_ABORTING_CONFIG: consts.DEPLOY_STATE_CONFIG_FAILED +} + def get_import_path(cls): return cls.__module__ + "." + cls.__name__ @@ -526,7 +541,7 @@ def get_oam_addresses(subcloud_name, sc_ks_client): def get_ansible_filename(subcloud_name, postfix='.yml'): """Build ansible filename using subcloud and given postfix""" - ansible_filename = os.path.join(consts.ANSIBLE_OVERRIDES_PATH, + ansible_filename = os.path.join(dccommon_consts.ANSIBLE_OVERRIDES_PATH, subcloud_name + postfix) return ansible_filename @@ -1025,3 +1040,21 @@ def get_failure_msg(subcloud_name): except Exception as e: LOG.exception("{}: {}".format(subcloud_name, e)) return consts.ERROR_DESC_FAILED + + +def update_abort_status(context, subcloud_id, deploy_status, abort_failed=False): + """Update the subcloud deploy status during deploy abort operation. + + :param context: request context object + :param subcloud_id: subcloud id from db + :param deploy_status: subcloud deploy status from db + :param abort_failed: if abort process fails (default False) + """ + if abort_failed: + abort_status_dict = ABORT_UPDATE_FAIL_STATUS + else: + abort_status_dict = ABORT_UPDATE_STATUS + new_deploy_status = abort_status_dict[deploy_status] + updated_subcloud = db_api.subcloud_update(context, subcloud_id, + deploy_status=new_deploy_status) + return updated_subcloud diff --git a/distributedcloud/dcmanager/manager/service.py b/distributedcloud/dcmanager/manager/service.py index a54853641..6401c410b 100644 --- a/distributedcloud/dcmanager/manager/service.py +++ b/distributedcloud/dcmanager/manager/service.py @@ -22,6 +22,7 @@ import oslo_messaging from oslo_service import service from oslo_utils import uuidutils +from dccommon import consts as dccommon_consts from dccommon.subprocess_cleanup import SubprocessCleanup from dcmanager.audit import rpcapi as dcmanager_audit_rpc_client from dcmanager.common import consts @@ -92,7 +93,7 @@ class DCManagerService(service.Service): if not os.path.isdir(consts.DC_ANSIBLE_LOG_DIR): os.mkdir(consts.DC_ANSIBLE_LOG_DIR, 0o755) - os.makedirs(consts.ANSIBLE_OVERRIDES_PATH, 0o600, exist_ok=True) + os.makedirs(dccommon_consts.ANSIBLE_OVERRIDES_PATH, 0o600, exist_ok=True) self.subcloud_manager.handle_subcloud_operations_in_progress() super(DCManagerService, self).start() @@ -224,6 +225,14 @@ class DCManagerService(service.Service): subcloud_id, payload) + @request_context + def subcloud_deploy_abort(self, context, subcloud_id, deploy_status): + # Abort the subcloud deployment + LOG.info("Handling subcloud_deploy_abort request for: %s" % subcloud_id) + return self.subcloud_manager.subcloud_deploy_abort(context, + subcloud_id, + deploy_status) + def _stop_rpc_server(self): # Stop RPC connection to prevent new requests LOG.debug(_("Attempting to stop RPC service...")) diff --git a/distributedcloud/dcmanager/manager/subcloud_manager.py b/distributedcloud/dcmanager/manager/subcloud_manager.py index ba52de907..da7a2bb6d 100644 --- a/distributedcloud/dcmanager/manager/subcloud_manager.py +++ b/distributedcloud/dcmanager/manager/subcloud_manager.py @@ -44,7 +44,9 @@ from dccommon.drivers.openstack.sysinv_v1 import SysinvClient from dccommon.exceptions import PlaybookExecutionFailed from dccommon import kubeoperator from dccommon.subcloud_install import SubcloudInstall +from dccommon.subcloud_install import SubcloudShutdown from dccommon.utils import run_playbook +from dccommon.utils import RunAnsible from dcmanager.audit import rpcapi as dcmanager_audit_rpc_client from dcmanager.common import consts from dcmanager.common.consts import INVENTORY_FILE_POSTFIX @@ -110,6 +112,9 @@ TRANSITORY_STATES = { consts.DEPLOY_STATE_PRE_CONFIG: consts.DEPLOY_STATE_PRE_CONFIG_FAILED, consts.DEPLOY_STATE_CONFIGURING: consts.DEPLOY_STATE_CONFIG_FAILED, consts.DEPLOY_STATE_DEPLOYING: consts.DEPLOY_STATE_DEPLOY_FAILED, + consts.DEPLOY_STATE_ABORTING_INSTALL: consts.DEPLOY_STATE_INSTALL_FAILED, + consts.DEPLOY_STATE_ABORTING_BOOTSTRAP: consts.DEPLOY_STATE_BOOTSTRAP_FAILED, + consts.DEPLOY_STATE_ABORTING_CONFIG: consts.DEPLOY_STATE_CONFIG_FAILED, consts.DEPLOY_STATE_MIGRATING_DATA: consts.DEPLOY_STATE_DATA_MIGRATION_FAILED, consts.DEPLOY_STATE_PRE_RESTORE: consts.DEPLOY_STATE_RESTORE_PREP_FAILED, consts.DEPLOY_STATE_RESTORING: consts.DEPLOY_STATE_RESTORE_FAILED, @@ -224,7 +229,7 @@ class SubcloudManager(manager.Manager): @staticmethod def _get_ansible_filename(subcloud_name, postfix='.yml'): ansible_filename = os.path.join( - consts.ANSIBLE_OVERRIDES_PATH, + dccommon_consts.ANSIBLE_OVERRIDES_PATH, subcloud_name + postfix) return ansible_filename @@ -235,7 +240,7 @@ class SubcloudManager(manager.Manager): "ansible-playbook", dccommon_consts.ANSIBLE_SUBCLOUD_INSTALL_PLAYBOOK, "-i", ansible_subcloud_inventory_file, "--limit", subcloud_name, - "-e", "@%s" % consts.ANSIBLE_OVERRIDES_PATH + "/" + + "-e", "@%s" % dccommon_consts.ANSIBLE_OVERRIDES_PATH + "/" + subcloud_name + '/' + "install_values.yml", "-e", "install_release_version=%s" % software_version if software_version else SW_VERSION] @@ -256,7 +261,7 @@ class SubcloudManager(manager.Manager): # which overrides to load apply_command += [ "-e", str("override_files_dir='%s' region_name=%s") % ( - consts.ANSIBLE_OVERRIDES_PATH, subcloud_name), + dccommon_consts.ANSIBLE_OVERRIDES_PATH, subcloud_name), "-e", "install_release_version=%s" % software_version if software_version else SW_VERSION] return apply_command @@ -265,7 +270,7 @@ class SubcloudManager(manager.Manager): def compose_deploy_command(self, subcloud_name, ansible_subcloud_inventory_file, payload): deploy_command = [ "ansible-playbook", payload[consts.DEPLOY_PLAYBOOK], - "-e", "@%s" % consts.ANSIBLE_OVERRIDES_PATH + "/" + + "-e", "@%s" % dccommon_consts.ANSIBLE_OVERRIDES_PATH + "/" + subcloud_name + '_deploy_values.yml', "-i", ansible_subcloud_inventory_file, "--limit", subcloud_name @@ -277,7 +282,7 @@ class SubcloudManager(manager.Manager): "ansible-playbook", ANSIBLE_SUBCLOUD_BACKUP_CREATE_PLAYBOOK, "-i", ansible_subcloud_inventory_file, "--limit", subcloud_name, - "-e", "subcloud_bnr_overrides=%s" % consts.ANSIBLE_OVERRIDES_PATH + "/" + + "-e", "subcloud_bnr_overrides=%s" % dccommon_consts.ANSIBLE_OVERRIDES_PATH + "/" + subcloud_name + "_backup_create_values.yml"] return backup_command @@ -288,7 +293,7 @@ class SubcloudManager(manager.Manager): "ansible-playbook", ANSIBLE_SUBCLOUD_BACKUP_DELETE_PLAYBOOK, "-i", ansible_subcloud_inventory_file, "--limit", subcloud_name, - "-e", "subcloud_bnr_overrides=%s" % consts.ANSIBLE_OVERRIDES_PATH + "/" + + "-e", "subcloud_bnr_overrides=%s" % dccommon_consts.ANSIBLE_OVERRIDES_PATH + "/" + subcloud_name + "_backup_delete_values.yml"] return backup_command @@ -298,7 +303,7 @@ class SubcloudManager(manager.Manager): "ansible-playbook", ANSIBLE_SUBCLOUD_BACKUP_RESTORE_PLAYBOOK, "-i", ansible_subcloud_inventory_file, "--limit", subcloud_name, - "-e", "subcloud_bnr_overrides=%s" % consts.ANSIBLE_OVERRIDES_PATH + "/" + + "-e", "subcloud_bnr_overrides=%s" % dccommon_consts.ANSIBLE_OVERRIDES_PATH + "/" + subcloud_name + "_backup_restore_values.yml"] return backup_command @@ -309,7 +314,7 @@ class SubcloudManager(manager.Manager): "-i", ansible_subcloud_inventory_file, "--limit", subcloud_name, "--timeout", UPDATE_PLAYBOOK_TIMEOUT, - "-e", "subcloud_update_overrides=%s" % consts.ANSIBLE_OVERRIDES_PATH + "/" + + "-e", "subcloud_update_overrides=%s" % dccommon_consts.ANSIBLE_OVERRIDES_PATH + "/" + subcloud_name + "_update_values.yml"] return subcloud_update_command @@ -324,7 +329,7 @@ class SubcloudManager(manager.Manager): "--limit", subcloud_name, "--timeout", REHOME_PLAYBOOK_TIMEOUT, "-e", str("override_files_dir='%s' region_name=%s") % ( - consts.ANSIBLE_OVERRIDES_PATH, subcloud_name)] + dccommon_consts.ANSIBLE_OVERRIDES_PATH, subcloud_name)] return rehome_command def add_subcloud(self, context, payload): @@ -784,6 +789,53 @@ class SubcloudManager(manager.Manager): return install_command + def subcloud_deploy_abort(self, context, subcloud_id, deploy_status): + """Abort the subcloud deploy + + :param context: request context object + :param subcloud_id: subcloud id from db + :param deploy_status: subcloud deploy status from db + """ + + LOG.info("Aborting deployment of subcloud %s." % subcloud_id) + subcloud = utils.update_abort_status(context, subcloud_id, deploy_status) + + try: + run_ansible = RunAnsible() + aborted = run_ansible.run_abort(subcloud.name) + if not aborted: + LOG.warning("Ansible deploy phase subprocess of %s " + "was terminated before it could be aborted" + % subcloud.name) + # let the main phase thread handle the state update + return + + if subcloud.deploy_status == consts.DEPLOY_STATE_ABORTING_INSTALL: + # First delete the k8s job and pod, stopping the current + # installation attempt if exists + # Then send shutdown signal to subcloud + kube = kubeoperator.KubeOperator() + shutdown_subcloud = SubcloudShutdown(subcloud.name) + namespace = dccommon_consts.RVMC_NAME_PREFIX + jobname = '%s-%s' % (namespace, subcloud.name) + pod_basename = '%s-' % jobname + all_pods = kube.get_pods_by_namespace(namespace) + desired_pod = next((s for s in all_pods if pod_basename in s), None) + if desired_pod: + kube.kube_delete_job(jobname, namespace) + kube.kube_delete_pod(desired_pod, namespace) + while kube.pod_exists(desired_pod, namespace): + time.sleep(2) + shutdown_subcloud.send_shutdown_signal() + except Exception as ex: + LOG.error("Subcloud deploy abort failed for subcloud %s" % subcloud.name) + utils.update_abort_status(context, subcloud.id, subcloud.deploy_status, + abort_failed=True) + # exception is logged above + raise ex + LOG.info("Successfully aborted deployment of %s" % subcloud.name) + utils.update_abort_status(context, subcloud.id, subcloud.deploy_status) + def subcloud_deploy_create(self, context, subcloud_id, payload): """Create subcloud and notify orchestrators. @@ -942,13 +994,9 @@ class SubcloudManager(manager.Manager): install_command = self._deploy_install_prep( subcloud, payload, ansible_subcloud_inventory_file) + self.run_deploy_commands(subcloud, payload, context, + install_command=install_command) - apply_thread = threading.Thread( - target=self.run_deploy_commands, - args=(subcloud, payload, context), - kwargs={'install_command': install_command}) - apply_thread.start() - return db_api.subcloud_db_model_to_dict(subcloud) except Exception: LOG.exception("Failed to install subcloud %s" % subcloud.name) # If we failed to install the subcloud, @@ -1011,7 +1059,7 @@ class SubcloudManager(manager.Manager): del payload['sysadmin_password'] # Update the ansible overrides file - overrides_file = os.path.join(consts.ANSIBLE_OVERRIDES_PATH, + overrides_file = os.path.join(dccommon_consts.ANSIBLE_OVERRIDES_PATH, subcloud.name + '.yml') utils.update_values_on_yaml_file(overrides_file, payload) @@ -1059,13 +1107,9 @@ class SubcloudManager(manager.Manager): ansible_subcloud_inventory_file, payload) - del payload['sysadmin_password'] - apply_thread = threading.Thread( - target=self.run_deploy_commands, - args=(subcloud, payload, context), - kwargs={'deploy_command': deploy_command}) - apply_thread.start() - return db_api.subcloud_db_model_to_dict(subcloud) + self.run_deploy_commands(subcloud, payload, context, + deploy_command=deploy_command) + except Exception: LOG.exception("Failed to configure %s" % subcloud.name) db_api.subcloud_update( @@ -1431,7 +1475,7 @@ class SubcloudManager(manager.Manager): def _create_backup_overrides_file(self, payload, subcloud_name, filename_suffix): backup_overrides_file = os.path.join( - consts.ANSIBLE_OVERRIDES_PATH, subcloud_name + '_' + + dccommon_consts.ANSIBLE_OVERRIDES_PATH, subcloud_name + '_' + filename_suffix + '.yml') with open(backup_overrides_file, 'w') as f_out: @@ -1722,7 +1766,8 @@ class SubcloudManager(manager.Manager): if install_command: install_success = self._run_subcloud_install( context, subcloud, install_command, - log_file, payload['install_values']) + log_file, payload['install_values'], + abortable=True) if not install_success: return db_api.subcloud_update( @@ -1731,8 +1776,10 @@ class SubcloudManager(manager.Manager): error_description=consts.ERROR_DESC_EMPTY) if apply_command: - self._run_subcloud_bootstrap(context, subcloud, - apply_command, log_file) + bootstrap_success = self._run_subcloud_bootstrap( + context, subcloud, apply_command, log_file) + if not bootstrap_success: + return if deploy_command: self._run_subcloud_config(subcloud, context, @@ -1742,9 +1789,39 @@ class SubcloudManager(manager.Manager): LOG.exception("run_deploy failed") raise ex + def _run_subcloud_config(self, subcloud, context, + deploy_command, log_file): + # Run the custom deploy playbook + LOG.info("Starting deploy of %s" % subcloud.name) + db_api.subcloud_update( + context, subcloud.id, + deploy_status=consts.DEPLOY_STATE_CONFIGURING, + error_description=consts.ERROR_DESC_EMPTY) + + try: + run_ansible = RunAnsible() + aborted = run_ansible.exec_playbook( + log_file, deploy_command, subcloud.name) + except PlaybookExecutionFailed: + msg = utils.find_ansible_error_msg( + subcloud.name, log_file, consts.DEPLOY_STATE_CONFIGURING) + LOG.error(msg) + db_api.subcloud_update( + context, subcloud.id, + deploy_status=consts.DEPLOY_STATE_CONFIG_FAILED, + error_description=msg[0:consts.ERROR_DESCRIPTION_LENGTH]) + return False + if aborted: + return False + LOG.info("Successfully deployed %s" % subcloud.name) + db_api.subcloud_update( + context, subcloud.id, + deploy_status=consts.DEPLOY_STATE_DONE, + error_description=consts.ERROR_DESC_EMPTY) + @staticmethod - def _run_subcloud_install( - context, subcloud, install_command, log_file, payload): + def _run_subcloud_install(context, subcloud, install_command, + log_file, payload, abortable=False): software_version = str(payload['software_version']) LOG.info("Preparing remote install of %s, version: %s", subcloud.name, software_version) @@ -1754,7 +1831,7 @@ class SubcloudManager(manager.Manager): software_version=software_version) try: install = SubcloudInstall(context, subcloud.name) - install.prep(consts.ANSIBLE_OVERRIDES_PATH, payload) + install.prep(dccommon_consts.ANSIBLE_OVERRIDES_PATH, payload) except Exception as e: LOG.exception(e) db_api.subcloud_update( @@ -1771,7 +1848,8 @@ class SubcloudManager(manager.Manager): deploy_status=consts.DEPLOY_STATE_INSTALLING, error_description=consts.ERROR_DESC_EMPTY) try: - install.install(consts.DC_ANSIBLE_LOG_DIR, install_command) + aborted = install.install( + consts.DC_ANSIBLE_LOG_DIR, install_command, abortable=abortable) except Exception as e: msg = utils.find_ansible_error_msg( subcloud.name, log_file, consts.DEPLOY_STATE_INSTALLING) @@ -1784,6 +1862,8 @@ class SubcloudManager(manager.Manager): install.cleanup(software_version) return False install.cleanup(software_version) + if aborted: + return False LOG.info("Successfully installed %s" % subcloud.name) return True @@ -1798,7 +1878,9 @@ class SubcloudManager(manager.Manager): # Run the ansible subcloud boostrap playbook LOG.info("Starting bootstrap of %s" % subcloud.name) try: - run_playbook(log_file, apply_command) + run_ansible = RunAnsible() + aborted = run_ansible.exec_playbook( + log_file, apply_command, subcloud.name) except PlaybookExecutionFailed: msg = utils.find_ansible_error_msg( subcloud.name, log_file, consts.DEPLOY_STATE_BOOTSTRAPPING) @@ -1807,7 +1889,10 @@ class SubcloudManager(manager.Manager): context, subcloud.id, deploy_status=consts.DEPLOY_STATE_BOOTSTRAP_FAILED, error_description=msg[0:consts.ERROR_DESCRIPTION_LENGTH]) - return + return False + + if aborted: + return False db_api.subcloud_update( context, subcloud.id, @@ -1815,32 +1900,7 @@ class SubcloudManager(manager.Manager): error_description=consts.ERROR_DESC_EMPTY) LOG.info("Successfully bootstrapped %s" % subcloud.name) - - def _run_subcloud_config(self, subcloud, context, - deploy_command, log_file): - # Run the custom deploy playbook - LOG.info("Starting deploy of %s" % subcloud.name) - db_api.subcloud_update( - context, subcloud.id, - deploy_status=consts.DEPLOY_STATE_CONFIGURING, - error_description=consts.ERROR_DESC_EMPTY) - - try: - run_playbook(log_file, deploy_command) - except PlaybookExecutionFailed: - msg = utils.find_ansible_error_msg( - subcloud.name, log_file, consts.DEPLOY_STATE_CONFIGURING) - LOG.error(msg) - db_api.subcloud_update( - context, subcloud.id, - deploy_status=consts.DEPLOY_STATE_CONFIG_FAILED, - error_description=msg[0:consts.ERROR_DESCRIPTION_LENGTH]) - return - LOG.info("Successfully deployed %s" % subcloud.name) - db_api.subcloud_update( - context, subcloud.id, - deploy_status=consts.DEPLOY_STATE_DONE, - error_description=consts.ERROR_DESC_EMPTY) + return True def _create_addn_hosts_dc(self, context): """Generate the addn_hosts_dc file for hostname/ip translation""" @@ -1868,7 +1928,7 @@ class SubcloudManager(manager.Manager): def _write_subcloud_ansible_config(self, cached_regionone_data, payload): """Create the override file for usage with the specified subcloud""" - overrides_file = os.path.join(consts.ANSIBLE_OVERRIDES_PATH, + overrides_file = os.path.join(dccommon_consts.ANSIBLE_OVERRIDES_PATH, payload['name'] + '.yml') mgmt_pool = cached_regionone_data['mgmt_pool'] @@ -1901,7 +1961,7 @@ class SubcloudManager(manager.Manager): """Create the deploy value files for the subcloud""" deploy_values_file = os.path.join( - consts.ANSIBLE_OVERRIDES_PATH, subcloud_name + + dccommon_consts.ANSIBLE_OVERRIDES_PATH, subcloud_name + '_deploy_values.yml') with open(deploy_values_file, 'w') as f_out_deploy_values_file: @@ -2362,7 +2422,7 @@ class SubcloudManager(manager.Manager): def _create_subcloud_update_overrides_file( self, payload, subcloud_name, filename_suffix): update_overrides_file = os.path.join( - consts.ANSIBLE_OVERRIDES_PATH, subcloud_name + '_' + + dccommon_consts.ANSIBLE_OVERRIDES_PATH, subcloud_name + '_' + filename_suffix + '.yml') self._update_override_values(payload) diff --git a/distributedcloud/dcmanager/orchestrator/states/upgrade/migrating_data.py b/distributedcloud/dcmanager/orchestrator/states/upgrade/migrating_data.py index 28d0ef506..7494ac7ba 100644 --- a/distributedcloud/dcmanager/orchestrator/states/upgrade/migrating_data.py +++ b/distributedcloud/dcmanager/orchestrator/states/upgrade/migrating_data.py @@ -6,6 +6,7 @@ import os import time +from dccommon import consts as dccommon_consts from dccommon.exceptions import PlaybookExecutionFailed from dccommon.utils import run_playbook from dcmanager.common import consts @@ -139,7 +140,7 @@ class MigratingDataState(BaseState): deploy_status=consts.DEPLOY_STATE_MIGRATING_DATA) ansible_subcloud_inventory_file = os.path.join( - consts.ANSIBLE_OVERRIDES_PATH, + dccommon_consts.ANSIBLE_OVERRIDES_PATH, strategy_step.subcloud.name + consts.INVENTORY_FILE_POSTFIX) log_file = os.path.join(consts.DC_ANSIBLE_LOG_DIR, subcloud.name) + \ '_playbook_output.log' diff --git a/distributedcloud/dcmanager/orchestrator/states/upgrade/upgrading_simplex.py b/distributedcloud/dcmanager/orchestrator/states/upgrade/upgrading_simplex.py index 36b16c989..69dd1f7ea 100644 --- a/distributedcloud/dcmanager/orchestrator/states/upgrade/upgrading_simplex.py +++ b/distributedcloud/dcmanager/orchestrator/states/upgrade/upgrading_simplex.py @@ -336,7 +336,7 @@ class UpgradingSimplexState(BaseState): try: install = SubcloudInstall( self.context, strategy_step.subcloud.name) - install.prep(consts.ANSIBLE_OVERRIDES_PATH, + install.prep(dccommon_consts.ANSIBLE_OVERRIDES_PATH, install_values) except Exception as e: db_api.subcloud_update( @@ -349,7 +349,7 @@ class UpgradingSimplexState(BaseState): raise ansible_subcloud_inventory_file = os.path.join( - consts.ANSIBLE_OVERRIDES_PATH, + dccommon_consts.ANSIBLE_OVERRIDES_PATH, strategy_step.subcloud.name + consts.INVENTORY_FILE_POSTFIX) # Create the ansible inventory for the upgrade subcloud @@ -360,7 +360,7 @@ class UpgradingSimplexState(BaseState): install_command = [ "ansible-playbook", dccommon_consts.ANSIBLE_SUBCLOUD_INSTALL_PLAYBOOK, "-i", ansible_subcloud_inventory_file, - "-e", "@%s" % consts.ANSIBLE_OVERRIDES_PATH + "/" + + "-e", "@%s" % dccommon_consts.ANSIBLE_OVERRIDES_PATH + "/" + strategy_step.subcloud.name + '/' + "install_values.yml" ] diff --git a/distributedcloud/dcmanager/rpc/client.py b/distributedcloud/dcmanager/rpc/client.py index ff420b5cb..a4fdfa8fd 100644 --- a/distributedcloud/dcmanager/rpc/client.py +++ b/distributedcloud/dcmanager/rpc/client.py @@ -203,10 +203,15 @@ class ManagerClient(RPCClient): payload=payload)) def subcloud_deploy_config(self, ctxt, subcloud_id, payload): - return self.call(ctxt, self.make_msg('subcloud_deploy_config', + return self.cast(ctxt, self.make_msg('subcloud_deploy_config', subcloud_id=subcloud_id, payload=payload)) + def subcloud_deploy_abort(self, ctxt, subcloud_id, deploy_status): + return self.cast(ctxt, self.make_msg('subcloud_deploy_abort', + subcloud_id=subcloud_id, + deploy_status=deploy_status)) + class DCManagerNotifications(RPCClient): """DC Manager Notification interface to broadcast subcloud state changed diff --git a/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_phased_subcloud_deploy.py b/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_phased_subcloud_deploy.py index 2825ba0eb..0f4dd0ec5 100644 --- a/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_phased_subcloud_deploy.py +++ b/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_phased_subcloud_deploy.py @@ -463,3 +463,40 @@ class TestSubcloudDeployInstall(testroot.DCManagerApiTest): self.assertEqual(consts.DEPLOY_STATE_PRE_INSTALL, response.json['deploy-status']) self.assertEqual(SW_VERSION, response.json['software-version']) + + +class TestSubcloudDeployAbort(testroot.DCManagerApiTest): + def setUp(self): + super(TestSubcloudDeployAbort, self).setUp() + self.ctx = utils.dummy_context() + + p = mock.patch.object(rpc_client, 'ManagerClient') + self.mock_rpc_client = p.start() + self.addCleanup(p.stop) + + def test_abort_subcloud(self): + subcloud = fake_subcloud.create_fake_subcloud( + self.ctx, + deploy_status=consts.DEPLOY_STATE_INSTALLING) + + self.mock_rpc_client().subcloud_deploy_abort.return_value = True + + response = self.app.patch_json(FAKE_URL + '/' + str(subcloud.id) + + '/abort', + headers=FAKE_HEADERS) + self.mock_rpc_client().subcloud_deploy_abort.assert_called_once_with( + mock.ANY, + subcloud.id, + subcloud.deploy_status) + self.assertEqual(response.status_int, 200) + + def test_abort_subcloud_invalid_deploy_status(self): + subcloud = fake_subcloud.create_fake_subcloud( + self.ctx, + deploy_status=consts.DEPLOY_STATE_INSTALLED) + self.mock_rpc_client().subcloud_deploy_config.return_value = True + + six.assertRaisesRegex(self, webtest.app.AppError, "400 *", + self.app.patch_json, FAKE_URL + '/' + + str(subcloud.id) + '/abort', + headers=FAKE_HEADERS) diff --git a/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_subcloud_deploy.py b/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_subcloud_deploy.py index 765a5949b..ef616d93c 100644 --- a/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_subcloud_deploy.py +++ b/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_subcloud_deploy.py @@ -19,6 +19,7 @@ import mock from six.moves import http_client import webtest +from dccommon import consts as dccommon_consts from dcmanager.api.controllers.v1 import subcloud_deploy from dcmanager.common import consts from dcmanager.common import phased_subcloud_deploy as psd_common @@ -254,8 +255,8 @@ class TestSubcloudDeploy(testroot.DCManagerApiTest): deploy_config = psd_common.get_config_file_path("subcloud1", consts.DEPLOY_CONFIG) self.assertEqual(bootstrap_file, - f'{consts.ANSIBLE_OVERRIDES_PATH}/subcloud1.yml') + f'{dccommon_consts.ANSIBLE_OVERRIDES_PATH}/subcloud1.yml') self.assertEqual(install_values, - f'{consts.ANSIBLE_OVERRIDES_PATH}/subcloud1/install_values.yml') + f'{dccommon_consts.ANSIBLE_OVERRIDES_PATH}/subcloud1/install_values.yml') self.assertEqual(deploy_config, - f'{consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_deploy_config.yml') + f'{dccommon_consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_deploy_config.yml') diff --git a/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_subclouds.py b/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_subclouds.py index 28e158a3c..27b2e3982 100644 --- a/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_subclouds.py +++ b/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_subclouds.py @@ -1591,11 +1591,11 @@ class TestSubcloudAPIOther(testroot.DCManagerApiTest): install_values = sc._get_config_file_path("subcloud1", "install_values") deploy_config = sc._get_config_file_path("subcloud1", consts.DEPLOY_CONFIG) self.assertEqual(bootstrap_file, - f'{consts.ANSIBLE_OVERRIDES_PATH}/subcloud1.yml') + f'{dccommon_consts.ANSIBLE_OVERRIDES_PATH}/subcloud1.yml') self.assertEqual(install_values, - f'{consts.ANSIBLE_OVERRIDES_PATH}/subcloud1/install_values.yml') + f'{dccommon_consts.ANSIBLE_OVERRIDES_PATH}/subcloud1/install_values.yml') self.assertEqual(deploy_config, - f'{consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_deploy_config.yml') + f'{dccommon_consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_deploy_config.yml') @mock.patch.object(rpc_client, 'ManagerClient') def test_format_ip_address(self, mock_rpc_client): diff --git a/distributedcloud/dcmanager/tests/unit/manager/test_subcloud_manager.py b/distributedcloud/dcmanager/tests/unit/manager/test_subcloud_manager.py index f55e2327c..a64c86647 100644 --- a/distributedcloud/dcmanager/tests/unit/manager/test_subcloud_manager.py +++ b/distributedcloud/dcmanager/tests/unit/manager/test_subcloud_manager.py @@ -28,6 +28,7 @@ sys.modules['fm_core'] = mock.Mock() import threading from dccommon import consts as dccommon_consts +from dccommon.utils import RunAnsible from dcmanager.common import consts from dcmanager.common import exceptions from dcmanager.common import prestage @@ -427,9 +428,7 @@ class TestSubcloudManager(base.DCManagerTestCase): @mock.patch.object( subcloud_manager.SubcloudManager, 'compose_install_command') - @mock.patch.object(threading.Thread, 'start') def test_deploy_install_subcloud(self, - mock_thread_start, mock_compose_install_command): subcloud_name = 'subcloud1' @@ -453,7 +452,6 @@ class TestSubcloudManager(base.DCManagerTestCase): subcloud_name, sm._get_ansible_filename(subcloud_name, consts.INVENTORY_FILE_POSTFIX), FAKE_PREVIOUS_SW_VERSION) - mock_thread_start.assert_called_once() @mock.patch.object(subcloud_manager.SubcloudManager, '_create_intermediate_ca_cert') @@ -538,12 +536,13 @@ class TestSubcloudManager(base.DCManagerTestCase): @mock.patch.object(subcloud_manager, 'keyring') @mock.patch.object(cutils, 'get_playbook_for_software_version') @mock.patch.object(cutils, 'update_values_on_yaml_file') - @mock.patch.object(subcloud_manager, 'run_playbook') - def test_subcloud_deploy_bootstrap(self, mock_run_playbook, mock_update_yml, + @mock.patch.object(RunAnsible, 'exec_playbook') + def test_subcloud_deploy_bootstrap(self, mock_exec_playbook, mock_update_yml, mock_get_playbook_for_software_version, mock_keyring, create_subcloud_inventory): mock_get_playbook_for_software_version.return_value = "22.12" mock_keyring.get_password.return_value = "testpass" + mock_exec_playbook.return_value = False subcloud = fake_subcloud.create_fake_subcloud( self.ctx, @@ -557,7 +556,7 @@ class TestSubcloudManager(base.DCManagerTestCase): sm = subcloud_manager.SubcloudManager() sm.subcloud_deploy_bootstrap(self.ctx, subcloud.id, payload) - mock_run_playbook.assert_called_once() + mock_exec_playbook.assert_called_once() # Verify subcloud was updated with correct values updated_subcloud = db_api.subcloud_get_by_name(self.ctx, @@ -589,10 +588,7 @@ class TestSubcloudManager(base.DCManagerTestCase): @mock.patch.object(subcloud_manager.SubcloudManager, '_prepare_for_deployment') - @mock.patch.object(threading.Thread, - 'start') - def test_configure_subcloud(self, mock_thread_start, - mock_prepare_for_deployment): + def test_configure_subcloud(self, mock_prepare_for_deployment): subcloud = self.create_subcloud_static( self.ctx, name='subcloud1', @@ -607,7 +603,6 @@ class TestSubcloudManager(base.DCManagerTestCase): sm.subcloud_deploy_config(self.ctx, subcloud.id, payload=fake_payload) - mock_thread_start.assert_called_once() mock_prepare_for_deployment.assert_called_once() @mock.patch.object(subcloud_manager.SubcloudManager, @@ -1581,22 +1576,22 @@ class TestSubcloudManager(base.DCManagerTestCase): filename = sm._get_ansible_filename('subcloud1', consts.INVENTORY_FILE_POSTFIX) self.assertEqual(filename, - f'{consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_inventory.yml') + f'{dccommon_consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_inventory.yml') def test_compose_install_command(self): sm = subcloud_manager.SubcloudManager() install_command = sm.compose_install_command( 'subcloud1', - f'{consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_inventory.yml', + f'{dccommon_consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_inventory.yml', FAKE_PREVIOUS_SW_VERSION) self.assertEqual( install_command, [ 'ansible-playbook', dccommon_consts.ANSIBLE_SUBCLOUD_INSTALL_PLAYBOOK, - '-i', f'{consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_inventory.yml', + '-i', f'{dccommon_consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_inventory.yml', '--limit', 'subcloud1', - '-e', f"@{consts.ANSIBLE_OVERRIDES_PATH}/subcloud1/install_values.yml", + '-e', f"@{dccommon_consts.ANSIBLE_OVERRIDES_PATH}/subcloud1/install_values.yml", '-e', "install_release_version=%s" % FAKE_PREVIOUS_SW_VERSION ] ) @@ -1607,7 +1602,7 @@ class TestSubcloudManager(base.DCManagerTestCase): sm = subcloud_manager.SubcloudManager() apply_command = sm.compose_apply_command( 'subcloud1', - f'{consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_inventory.yml', + f'{dccommon_consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_inventory.yml', FAKE_PREVIOUS_SW_VERSION) self.assertEqual( apply_command, @@ -1616,9 +1611,9 @@ class TestSubcloudManager(base.DCManagerTestCase): cutils.get_playbook_for_software_version( subcloud_manager.ANSIBLE_SUBCLOUD_PLAYBOOK, FAKE_PREVIOUS_SW_VERSION), - '-i', f'{consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_inventory.yml', + '-i', f'{dccommon_consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_inventory.yml', '--limit', 'subcloud1', '-e', - f"override_files_dir='{consts.ANSIBLE_OVERRIDES_PATH}' region_name=subcloud1", + f"override_files_dir='{dccommon_consts.ANSIBLE_OVERRIDES_PATH}' region_name=subcloud1", '-e', "install_release_version=%s" % FAKE_PREVIOUS_SW_VERSION ] ) @@ -1632,14 +1627,14 @@ class TestSubcloudManager(base.DCManagerTestCase): "deploy_config": "subcloud1.yaml"} deploy_command = sm.compose_deploy_command( 'subcloud1', - f'{consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_inventory.yml', + f'{dccommon_consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_inventory.yml', fake_payload) self.assertEqual( deploy_command, [ 'ansible-playbook', 'test_playbook.yaml', '-e', - f'@{consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_deploy_values.yml', '-i', - f'{consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_inventory.yml', + f'@{dccommon_consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_deploy_values.yml', '-i', + f'{dccommon_consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_inventory.yml', '--limit', 'subcloud1' ] ) @@ -1650,7 +1645,7 @@ class TestSubcloudManager(base.DCManagerTestCase): sm = subcloud_manager.SubcloudManager() rehome_command = sm.compose_rehome_command( 'subcloud1', - f'{consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_inventory.yml', + f'{dccommon_consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_inventory.yml', FAKE_PREVIOUS_SW_VERSION) self.assertEqual( rehome_command, @@ -1659,11 +1654,11 @@ class TestSubcloudManager(base.DCManagerTestCase): cutils.get_playbook_for_software_version( subcloud_manager.ANSIBLE_SUBCLOUD_REHOME_PLAYBOOK, FAKE_PREVIOUS_SW_VERSION), - '-i', f'{consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_inventory.yml', + '-i', f'{dccommon_consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_inventory.yml', '--limit', 'subcloud1', '--timeout', subcloud_manager.REHOME_PLAYBOOK_TIMEOUT, '-e', - f"override_files_dir='{consts.ANSIBLE_OVERRIDES_PATH}' region_name=subcloud1" + f"override_files_dir='{dccommon_consts.ANSIBLE_OVERRIDES_PATH}' region_name=subcloud1" ] ) @@ -2102,7 +2097,7 @@ class TestSubcloudManager(base.DCManagerTestCase): values = copy.copy(FAKE_BACKUP_CREATE_LOAD_1) override_file = os_path.join( - consts.ANSIBLE_OVERRIDES_PATH, subcloud.name + "_backup_create_values.yml" + dccommon_consts.ANSIBLE_OVERRIDES_PATH, subcloud.name + "_backup_create_values.yml" ) mock_create_backup_file.return_value = override_file @@ -2175,7 +2170,7 @@ class TestSubcloudManager(base.DCManagerTestCase): RELEASE_VERSION = '22.12' override_file = os_path.join( - consts.ANSIBLE_OVERRIDES_PATH, subcloud.name + "_backup_delete_values.yml" + dccommon_consts.ANSIBLE_OVERRIDES_PATH, subcloud.name + "_backup_delete_values.yml" ) mock_create_subcloud_inventory_file.return_value = override_file