Add subcloud deploy abort option to dcmanager
This commit adds the command "subcloud deploy abort" to dcmanager. It allows the user to abort the on-going execution of a playbook against the subcloud. Any task will be aborted immediately if the unabortable flag is not set by the playbook [1]. If current operation is install, a shutdown command will also be issued. Test Plan: Success cases: - PASS: Trigger an abort during installation and verify that the playbook execution was aborted immediately, the subcloud was shut dows after it and the RVMC pod and job were terminated. - PASS: Trigger an abort during config and verify that the playbook execution was aborted immediately. - PASS: Trigger an abort during bootstrap without the presence of unabortable flag and verify that the playbook execution was aborted immidiately. - PASS: Trigger an abort during bootstrap with the presence of unabortable flag and verify that the playbook execution was aborted only after the flag was deleted. - PASS: Trigger an abort directly calling the API (using CURL instead of using the CLI. Failure cases: - PASS: Verify that the abort request is rejected if deploy state is not 'installing', 'bootstrapping' or 'configuring'. - PASS: Abort when an unabortable task is running and then force an external error during this task restarting dcmanager-manager service, verify that deploy state is set to a failed state e.g. bootstrap-failed. - PASS: Abort when an unabortable task is running and then force an internal error during this task using ansible.builtin.fail module, verify that deploy state is set to a failed state e.g. bootstrap-failed. - PASS: Abort when an unabortable task is running and then force the playbook to halt the execution during this task using ansible.builtin.pause module, verify that deploy state is set to a failed state after 10 minutes e.g. bootstrap-failed. Story: 2010756 Task: 48102 Co-Authored-By: Gustavo Herzmann <gustavo.herzmann@windriver.com> Signed-off-by: Victor Romano <victor.gluzromano@windriver.com> Change-Id: Ic5311324a76bf7ce1215692e934d5577ff82868e
This commit is contained in:
parent
0cd5c53933
commit
f081453b3c
|
@ -2053,4 +2053,71 @@ Response Example
|
||||||
----------------
|
----------------
|
||||||
|
|
||||||
.. literalinclude:: samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-configure-response.json
|
.. literalinclude:: samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-configure-response.json
|
||||||
|
:language: json
|
||||||
|
|
||||||
|
|
||||||
|
**********************************
|
||||||
|
Abort subcloud deployment
|
||||||
|
**********************************
|
||||||
|
|
||||||
|
.. rest_method:: PATCH /v1.0/phased-subcloud-deploy/{subcloud}/abort
|
||||||
|
|
||||||
|
**Normal response codes**
|
||||||
|
|
||||||
|
200
|
||||||
|
|
||||||
|
**Error response codes**
|
||||||
|
|
||||||
|
badRequest (400), unauthorized (401), forbidden (403), badMethod (405),
|
||||||
|
HTTPUnprocessableEntity (422), internalServerError (500),
|
||||||
|
serviceUnavailable (503)
|
||||||
|
|
||||||
|
**Request parameters**
|
||||||
|
|
||||||
|
.. rest_parameters:: parameters.yaml
|
||||||
|
|
||||||
|
- subcloud: subcloud_uri
|
||||||
|
|
||||||
|
Accepts Content-Type multipart/form-data
|
||||||
|
|
||||||
|
Request Example
|
||||||
|
----------------
|
||||||
|
|
||||||
|
.. literalinclude:: samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-abort-request.json
|
||||||
|
:language: json
|
||||||
|
|
||||||
|
**Response parameters**
|
||||||
|
|
||||||
|
.. rest_parameters:: parameters.yaml
|
||||||
|
|
||||||
|
- id: subcloud_id
|
||||||
|
- group_id: group_id
|
||||||
|
- name: subcloud_name
|
||||||
|
- description: subcloud_description
|
||||||
|
- location: subcloud_location
|
||||||
|
- software-version: software_version
|
||||||
|
- availability-status: availability_status
|
||||||
|
- error-description: error_description
|
||||||
|
- deploy-status: deploy_status
|
||||||
|
- backup-status: backup_status
|
||||||
|
- backup-datetime: backup_datetime
|
||||||
|
- openstack-installed: openstack_installed
|
||||||
|
- management-state: management_state
|
||||||
|
- systemcontroller-gateway-ip: systemcontroller_gateway_ip
|
||||||
|
- management-start-ip: management_start_ip
|
||||||
|
- management-end-ip: management_end_ip
|
||||||
|
- management-subnet: management_subnet
|
||||||
|
- management-gateway-ip: management_gateway_ip
|
||||||
|
- created-at: created_at
|
||||||
|
- updated-at: updated_at
|
||||||
|
- data_install: data_install
|
||||||
|
- data_upgrade: data_upgrade
|
||||||
|
- endpoint_sync_status: endpoint_sync_status
|
||||||
|
- sync_status: sync_status
|
||||||
|
- endpoint_type: sync_status_type
|
||||||
|
|
||||||
|
Response Example
|
||||||
|
----------------
|
||||||
|
|
||||||
|
.. literalinclude:: samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-abort-response.json
|
||||||
:language: json
|
:language: json
|
|
@ -0,0 +1,3 @@
|
||||||
|
{
|
||||||
|
"subcloud": "subcloud1"
|
||||||
|
}
|
|
@ -0,0 +1,23 @@
|
||||||
|
{
|
||||||
|
"id": 1,
|
||||||
|
"name": "subcloud1",
|
||||||
|
"created-at": "2023-01-02T03:04:05.678987",
|
||||||
|
"updated-at": "2023-04-08T15:16:23.424851",
|
||||||
|
"availability-status": "offline",
|
||||||
|
"data_install": null,
|
||||||
|
"data_upgrade": null,
|
||||||
|
"deploy-status": "aborting-install",
|
||||||
|
"backup-status": null,
|
||||||
|
"backup-datetime": null,
|
||||||
|
"description": "Ottawa Site",
|
||||||
|
"group_id": 1,
|
||||||
|
"location": "YOW",
|
||||||
|
"management-end-ip": "192.168.101.50",
|
||||||
|
"management-gateway-ip": "192.168.101.1",
|
||||||
|
"management-start-ip": "192.168.101.2",
|
||||||
|
"management-state": "unmanaged",
|
||||||
|
"management-subnet": "192.168.101.0/24",
|
||||||
|
"openstack-installed": false,
|
||||||
|
"software-version": "22.12",
|
||||||
|
"systemcontroller-gateway-ip": "192.168.204.101"
|
||||||
|
}
|
|
@ -28,6 +28,7 @@ CLOUD_0 = "RegionOne"
|
||||||
VIRTUAL_MASTER_CLOUD = "SystemController"
|
VIRTUAL_MASTER_CLOUD = "SystemController"
|
||||||
|
|
||||||
SW_UPDATE_DEFAULT_TITLE = "all clouds default"
|
SW_UPDATE_DEFAULT_TITLE = "all clouds default"
|
||||||
|
ANSIBLE_OVERRIDES_PATH = '/opt/dc-vault/ansible'
|
||||||
LOAD_VAULT_DIR = '/opt/dc-vault/loads'
|
LOAD_VAULT_DIR = '/opt/dc-vault/loads'
|
||||||
DEPLOY_DIR = '/opt/platform/deploy'
|
DEPLOY_DIR = '/opt/platform/deploy'
|
||||||
|
|
||||||
|
@ -140,6 +141,10 @@ CERT_CA_FILE_CENTOS = "ca-cert.pem"
|
||||||
CERT_CA_FILE_DEBIAN = "ca-cert.crt"
|
CERT_CA_FILE_DEBIAN = "ca-cert.crt"
|
||||||
SSL_CERT_CA_DIR = "/etc/pki/ca-trust/source/anchors/"
|
SSL_CERT_CA_DIR = "/etc/pki/ca-trust/source/anchors/"
|
||||||
|
|
||||||
|
# RVMC
|
||||||
|
RVMC_NAME_PREFIX = 'rvmc'
|
||||||
|
RVMC_CONFIG_FILE_NAME = 'rvmc-config.yaml'
|
||||||
|
|
||||||
# Subcloud installation values
|
# Subcloud installation values
|
||||||
BMC_INSTALL_VALUES = [
|
BMC_INSTALL_VALUES = [
|
||||||
'bmc_username',
|
'bmc_username',
|
||||||
|
|
|
@ -119,3 +119,7 @@ class ImageNotInLocalRegistry(NotFound):
|
||||||
message = _("Image %(image_name)s:%(image_tag)s not found in the local registry. "
|
message = _("Image %(image_name)s:%(image_tag)s not found in the local registry. "
|
||||||
"Please check with command: system registry-image-list or "
|
"Please check with command: system registry-image-list or "
|
||||||
"system registry-image-tags %(image_name)s")
|
"system registry-image-tags %(image_name)s")
|
||||||
|
|
||||||
|
|
||||||
|
class SubcloudShutdownError(PlaybookExecutionFailed):
|
||||||
|
message = _("Subcloud %(subcloud_name)s could not be shut down.")
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
#
|
#
|
||||||
# Copyright (c) 2020-2022 Wind River Systems, Inc.
|
# Copyright (c) 2020-2023 Wind River Systems, Inc.
|
||||||
#
|
#
|
||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
#
|
#
|
||||||
|
@ -149,3 +149,58 @@ class KubeOperator(object):
|
||||||
if e.status != httplib.NOT_FOUND:
|
if e.status != httplib.NOT_FOUND:
|
||||||
LOG.error("Fail to delete %s:%s. %s" % (namespace, name, e))
|
LOG.error("Fail to delete %s:%s. %s" % (namespace, name, e))
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
def get_pods_by_namespace(self, namespace):
|
||||||
|
c = self._get_kubernetesclient_core()
|
||||||
|
try:
|
||||||
|
pods = c.list_namespaced_pod(namespace)
|
||||||
|
return [pod.metadata.name for pod in pods.items]
|
||||||
|
except ApiException as e:
|
||||||
|
if e.status == httplib.NOT_FOUND:
|
||||||
|
return []
|
||||||
|
LOG.error("Failed to get pod name under "
|
||||||
|
"Namespace %s." % (namespace))
|
||||||
|
raise
|
||||||
|
except Exception as e:
|
||||||
|
LOG.error("Kubernetes exception in get_pods_by_namespace: %s" % e)
|
||||||
|
raise
|
||||||
|
|
||||||
|
def pod_exists(self, pod, namespace):
|
||||||
|
pods = self.get_pods_by_namespace(namespace)
|
||||||
|
if pod in pods:
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
def kube_delete_job(self, name, namespace, **kwargs):
|
||||||
|
body = {}
|
||||||
|
if kwargs:
|
||||||
|
body.update(kwargs)
|
||||||
|
|
||||||
|
b = self._get_kubernetesclient_batch()
|
||||||
|
try:
|
||||||
|
b.delete_namespaced_job(name, namespace, body=body)
|
||||||
|
except ApiException as e:
|
||||||
|
if e.status != httplib.NOT_FOUND:
|
||||||
|
LOG.error("Failed to delete job %s under "
|
||||||
|
"Namespace %s: %s" % (name, namespace, e.body))
|
||||||
|
raise
|
||||||
|
except Exception as e:
|
||||||
|
LOG.error("Kubernetes exception in kube_delete_job: %s" % e)
|
||||||
|
raise
|
||||||
|
|
||||||
|
def kube_delete_pod(self, name, namespace, **kwargs):
|
||||||
|
body = {}
|
||||||
|
if kwargs:
|
||||||
|
body.update(kwargs)
|
||||||
|
|
||||||
|
c = self._get_kubernetesclient_core()
|
||||||
|
try:
|
||||||
|
c.delete_namespaced_pod(name, namespace, body=body)
|
||||||
|
except ApiException as e:
|
||||||
|
if e.status != httplib.NOT_FOUND:
|
||||||
|
LOG.error("Failed to delete pod %s under "
|
||||||
|
"Namespace %s: %s" % (name, namespace, e.body))
|
||||||
|
raise
|
||||||
|
except Exception as e:
|
||||||
|
LOG.error("Kubernetes exception in kube_delete_pod: %s" % e)
|
||||||
|
raise
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
# Copyright (c) 2021-2022 Wind River Systems, Inc.
|
# Copyright (c) 2021-2023 Wind River Systems, Inc.
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
# you may not use this file except in compliance with the License.
|
# you may not use this file except in compliance with the License.
|
||||||
# You may obtain a copy of the License at
|
# You may obtain a copy of the License at
|
||||||
|
@ -13,10 +13,13 @@
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
#
|
#
|
||||||
|
|
||||||
|
import json
|
||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
import socket
|
import socket
|
||||||
|
import ssl
|
||||||
import tempfile
|
import tempfile
|
||||||
|
import time
|
||||||
|
|
||||||
from eventlet.green import subprocess
|
from eventlet.green import subprocess
|
||||||
import netaddr
|
import netaddr
|
||||||
|
@ -24,13 +27,14 @@ from oslo_log import log as logging
|
||||||
from six.moves.urllib import error as urllib_error
|
from six.moves.urllib import error as urllib_error
|
||||||
from six.moves.urllib import parse
|
from six.moves.urllib import parse
|
||||||
from six.moves.urllib import request
|
from six.moves.urllib import request
|
||||||
|
import yaml
|
||||||
|
|
||||||
from dccommon import consts
|
from dccommon import consts
|
||||||
from dccommon.drivers.openstack.keystone_v3 import KeystoneClient
|
from dccommon.drivers.openstack.keystone_v3 import KeystoneClient
|
||||||
from dccommon.drivers.openstack.sysinv_v1 import SysinvClient
|
from dccommon.drivers.openstack.sysinv_v1 import SysinvClient
|
||||||
from dccommon import exceptions
|
from dccommon import exceptions
|
||||||
from dccommon import utils as dccommon_utils
|
from dccommon import utils as dccommon_utils
|
||||||
from dcmanager.common import consts as common_consts
|
from dcmanager.common import consts as dcmanager_consts
|
||||||
from dcmanager.common import utils
|
from dcmanager.common import utils
|
||||||
|
|
||||||
LOG = logging.getLogger(__name__)
|
LOG = logging.getLogger(__name__)
|
||||||
|
@ -39,7 +43,6 @@ BOOT_MENU_TIMEOUT = '5'
|
||||||
|
|
||||||
# The RVMC_IMAGE_NAME:RVMC_IMAGE_TAG must align with the one specified
|
# The RVMC_IMAGE_NAME:RVMC_IMAGE_TAG must align with the one specified
|
||||||
# in system images in the ansible install/upgrade playbook
|
# in system images in the ansible install/upgrade playbook
|
||||||
RVMC_NAME_PREFIX = 'rvmc'
|
|
||||||
RVMC_IMAGE_NAME = 'docker.io/starlingx/rvmc'
|
RVMC_IMAGE_NAME = 'docker.io/starlingx/rvmc'
|
||||||
RVMC_IMAGE_TAG = 'stx.8.0-v1.0.1'
|
RVMC_IMAGE_TAG = 'stx.8.0-v1.0.1'
|
||||||
|
|
||||||
|
@ -55,6 +58,99 @@ NETWORK_INTERFACE_PREFIX = 'ifcfg'
|
||||||
NETWORK_ROUTE_PREFIX = 'route'
|
NETWORK_ROUTE_PREFIX = 'route'
|
||||||
LOCAL_REGISTRY_PREFIX = 'registry.local:9001/'
|
LOCAL_REGISTRY_PREFIX = 'registry.local:9001/'
|
||||||
|
|
||||||
|
# Redfish constants
|
||||||
|
ACTION_URL = '/Actions/ComputerSystem.Reset'
|
||||||
|
POWER_OFF_PAYLOAD = {'Action': 'Reset', 'ResetType': 'ForceOff'}
|
||||||
|
REDFISH_HEADER = {'Content-Type': 'application/json',
|
||||||
|
'Accept': 'application/json'}
|
||||||
|
REDFISH_SYSTEMS_URL = '/redfish/v1/Systems'
|
||||||
|
SUCCESSFUL_STATUS_CODES = [200, 202, 204]
|
||||||
|
|
||||||
|
|
||||||
|
class SubcloudShutdown(object):
|
||||||
|
"""Sends a shutdown signal to a Redfish controlled subcloud
|
||||||
|
|
||||||
|
Approach:
|
||||||
|
|
||||||
|
To shutdown a Redfish controlled subcloud, it's needed to first
|
||||||
|
send a GET request to find the @odata.id of the member, and then
|
||||||
|
send a POST request with the shutdown signal. Since this is
|
||||||
|
intended as a way to turn off the subcloud during the deploy abort
|
||||||
|
process, only the ForceOff option is considered.
|
||||||
|
"""
|
||||||
|
def __init__(self, subcloud_name):
|
||||||
|
self.target = subcloud_name
|
||||||
|
self.rvmc_data = self._get_subcloud_data()
|
||||||
|
|
||||||
|
def _get_subcloud_data(self):
|
||||||
|
rvmc_config_file_path = os.path.join(consts.ANSIBLE_OVERRIDES_PATH,
|
||||||
|
self.target, consts.RVMC_CONFIG_FILE_NAME)
|
||||||
|
if not os.path.isfile(rvmc_config_file_path):
|
||||||
|
raise Exception('Missing rvmc files for %s' % self.target)
|
||||||
|
with open(os.path.abspath(rvmc_config_file_path), 'r') as f:
|
||||||
|
rvmc_data = f.read()
|
||||||
|
rvmc_config_values = yaml.load(rvmc_data, Loader=yaml.SafeLoader)
|
||||||
|
base_url = "https://" + rvmc_config_values['bmc_address']
|
||||||
|
bmc_username = rvmc_config_values['bmc_username']
|
||||||
|
bmc_password = rvmc_config_values['bmc_password']
|
||||||
|
credentials = ("%s:%s" % (bmc_username.rstrip(), bmc_password)).encode("utf-8")
|
||||||
|
return {'base_url': base_url, 'credentials': credentials}
|
||||||
|
|
||||||
|
def _make_request(self, url, credentials, method, retry=5):
|
||||||
|
if method == 'get':
|
||||||
|
payload = None
|
||||||
|
else:
|
||||||
|
payload = json.dumps(POWER_OFF_PAYLOAD).encode('utf-8')
|
||||||
|
|
||||||
|
try:
|
||||||
|
context = ssl._create_unverified_context()
|
||||||
|
req = request.Request(url, headers=REDFISH_HEADER, method=method)
|
||||||
|
req.add_header('Authorization', 'Basic %s' % credentials)
|
||||||
|
response = request.urlopen(req, data=payload, context=context)
|
||||||
|
status_code = response.getcode()
|
||||||
|
|
||||||
|
if status_code not in SUCCESSFUL_STATUS_CODES:
|
||||||
|
if retry <= 0:
|
||||||
|
raise exceptions.SubcloudShutdownError(
|
||||||
|
subcloud_name=self.target)
|
||||||
|
retry -= retry
|
||||||
|
time.sleep(2)
|
||||||
|
self._make_request(url, credentials, method, retry=retry)
|
||||||
|
except urllib_error.URLError:
|
||||||
|
# This occurs when the BMC is not available anymore,
|
||||||
|
# so we just ignore it.
|
||||||
|
return None
|
||||||
|
except Exception as ex:
|
||||||
|
raise ex
|
||||||
|
|
||||||
|
return response
|
||||||
|
|
||||||
|
def _get_data_id(self):
|
||||||
|
base_url = self.rvmc_data['base_url']
|
||||||
|
credentials = self.rvmc_data['credentials']
|
||||||
|
url = base_url + REDFISH_SYSTEMS_URL
|
||||||
|
response = self._make_request(url, credentials, method='GET')
|
||||||
|
if not response:
|
||||||
|
return None
|
||||||
|
r = json.loads(response.read().decode())
|
||||||
|
|
||||||
|
for member in r['Members']:
|
||||||
|
if member.get('@odata.id'):
|
||||||
|
url_with_id = member['@odata.id']
|
||||||
|
break
|
||||||
|
|
||||||
|
return url_with_id
|
||||||
|
|
||||||
|
def send_shutdown_signal(self):
|
||||||
|
base_url = self.rvmc_data['base_url']
|
||||||
|
credentials = self.rvmc_data['credentials']
|
||||||
|
url_with_id = self._get_data_id()
|
||||||
|
if not url_with_id:
|
||||||
|
return None
|
||||||
|
url = base_url + url_with_id + ACTION_URL
|
||||||
|
response = self._make_request(url, credentials, method='POST')
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
class SubcloudInstall(object):
|
class SubcloudInstall(object):
|
||||||
"""Class to encapsulate the subcloud install operations"""
|
"""Class to encapsulate the subcloud install operations"""
|
||||||
|
@ -169,7 +265,7 @@ class SubcloudInstall(object):
|
||||||
|
|
||||||
LOG.debug("create rvmc config file, path: %s, payload: %s",
|
LOG.debug("create rvmc config file, path: %s, payload: %s",
|
||||||
override_path, payload)
|
override_path, payload)
|
||||||
rvmc_config_file = os.path.join(override_path, 'rvmc-config.yaml')
|
rvmc_config_file = os.path.join(override_path, consts.RVMC_CONFIG_FILE_NAME)
|
||||||
|
|
||||||
with open(rvmc_config_file, 'w') as f_out_rvmc_config_file:
|
with open(rvmc_config_file, 'w') as f_out_rvmc_config_file:
|
||||||
for k, v in payload.items():
|
for k, v in payload.items():
|
||||||
|
@ -184,7 +280,7 @@ class SubcloudInstall(object):
|
||||||
RVMC_IMAGE_TAG
|
RVMC_IMAGE_TAG
|
||||||
install_override_file = os.path.join(override_path,
|
install_override_file = os.path.join(override_path,
|
||||||
'install_values.yml')
|
'install_values.yml')
|
||||||
rvmc_name = "%s-%s" % (RVMC_NAME_PREFIX, self.name)
|
rvmc_name = "%s-%s" % (consts.RVMC_NAME_PREFIX, self.name)
|
||||||
host_name = socket.gethostname()
|
host_name = socket.gethostname()
|
||||||
|
|
||||||
with open(install_override_file, 'w') as f_out_override_file:
|
with open(install_override_file, 'w') as f_out_override_file:
|
||||||
|
@ -600,16 +696,26 @@ class SubcloudInstall(object):
|
||||||
# create the install override file
|
# create the install override file
|
||||||
self.create_install_override_file(override_path, payload)
|
self.create_install_override_file(override_path, payload)
|
||||||
|
|
||||||
def install(self, log_file_dir, install_command):
|
def install(self, log_file_dir, install_command, abortable=False):
|
||||||
LOG.info("Start remote install %s", self.name)
|
LOG.info("Start remote install %s", self.name)
|
||||||
log_file = os.path.join(log_file_dir, self.name) + '_playbook_output.log'
|
log_file = os.path.join(log_file_dir, self.name) + '_playbook_output.log'
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Since this is a long-running task we want to register
|
# Since this is a long-running task we want to register
|
||||||
# for cleanup on process restart/SWACT.
|
# for cleanup on process restart/SWACT.
|
||||||
dccommon_utils.run_playbook(log_file, install_command)
|
if abortable:
|
||||||
|
# Install phase of subcloud deployment
|
||||||
|
run_ansible = dccommon_utils.RunAnsible()
|
||||||
|
aborted = run_ansible.exec_playbook(log_file, install_command, self.name)
|
||||||
|
# Returns True if the playbook was aborted and False otherwise
|
||||||
|
return aborted
|
||||||
|
else:
|
||||||
|
dccommon_utils.run_playbook(log_file, install_command)
|
||||||
|
# Always return false because this playbook execution
|
||||||
|
# method cannot be aborted
|
||||||
|
return False
|
||||||
except exceptions.PlaybookExecutionFailed:
|
except exceptions.PlaybookExecutionFailed:
|
||||||
msg = ("Failed to install %s, check individual "
|
msg = ("Failed to install %s, check individual "
|
||||||
"log at %s or run %s for details"
|
"log at %s or run %s for details"
|
||||||
% (self.name, log_file, common_consts.ERROR_DESC_CMD))
|
% (self.name, log_file, dcmanager_consts.ERROR_DESC_CMD))
|
||||||
raise Exception(msg)
|
raise Exception(msg)
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
# Copyright (c) 2020-2022 Wind River Systems, Inc.
|
# Copyright (c) 2020-2023 Wind River Systems, Inc.
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
# you may not use this file except in compliance with the License.
|
# you may not use this file except in compliance with the License.
|
||||||
# You may obtain a copy of the License at
|
# You may obtain a copy of the License at
|
||||||
|
@ -19,6 +19,8 @@ import functools
|
||||||
import os
|
import os
|
||||||
import random
|
import random
|
||||||
import re
|
import re
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
|
||||||
from eventlet.green import subprocess
|
from eventlet.green import subprocess
|
||||||
from oslo_log import log as logging
|
from oslo_log import log as logging
|
||||||
|
@ -28,6 +30,7 @@ from dccommon import consts
|
||||||
from dccommon import exceptions
|
from dccommon import exceptions
|
||||||
from dccommon.exceptions import PlaybookExecutionFailed
|
from dccommon.exceptions import PlaybookExecutionFailed
|
||||||
from dccommon.exceptions import PlaybookExecutionTimeout
|
from dccommon.exceptions import PlaybookExecutionTimeout
|
||||||
|
from dccommon.subprocess_cleanup import kill_subprocess_group
|
||||||
from dccommon.subprocess_cleanup import SubprocessCleanup
|
from dccommon.subprocess_cleanup import SubprocessCleanup
|
||||||
from dcorch.common.i18n import _
|
from dcorch.common.i18n import _
|
||||||
|
|
||||||
|
@ -86,6 +89,153 @@ class memoized(object):
|
||||||
return functools.partial(self.__call__, obj)
|
return functools.partial(self.__call__, obj)
|
||||||
|
|
||||||
|
|
||||||
|
class RunAnsible(object):
|
||||||
|
"""Class to run Ansible playbooks with the abort option
|
||||||
|
|
||||||
|
Approach:
|
||||||
|
|
||||||
|
At the start of the playbook execution, the abort status
|
||||||
|
(default value is False) and PID of the subprocess for the
|
||||||
|
specified subcloud are set on the class variable dict (abort_status).
|
||||||
|
When the user sends the abort command, the subcloud_manager changes
|
||||||
|
the abort status to True and the subprocess is killed.
|
||||||
|
|
||||||
|
If Ansible is currently executing a task that cannot be interrupted,
|
||||||
|
a deploy_not_abortable flag is created in the overrides folder by the
|
||||||
|
playbook itself, and the abort process will wait for said flag to be
|
||||||
|
deleted before killing the subprocess. If the task fails while abort
|
||||||
|
is waiting, the playbook_failed flag will indicate to the
|
||||||
|
original process to raise PlaybookExecutionFailed.
|
||||||
|
"""
|
||||||
|
abort_status = {}
|
||||||
|
lock = threading.Lock()
|
||||||
|
|
||||||
|
def _unregister_subcloud(self, subcloud_name):
|
||||||
|
with RunAnsible.lock:
|
||||||
|
if RunAnsible.abort_status.get(subcloud_name):
|
||||||
|
del RunAnsible.abort_status[subcloud_name]
|
||||||
|
|
||||||
|
def run_abort(self, subcloud_name, timeout=600):
|
||||||
|
"""Set abort status for a subcloud.
|
||||||
|
|
||||||
|
:param subcloud_name: Name of the subcloud
|
||||||
|
param timeout: Timeout in seconds.
|
||||||
|
"""
|
||||||
|
with RunAnsible.lock:
|
||||||
|
RunAnsible.abort_status[subcloud_name]['abort'] = True
|
||||||
|
unabortable_flag = os.path.join(consts.ANSIBLE_OVERRIDES_PATH,
|
||||||
|
'.%s_deploy_not_abortable' % subcloud_name)
|
||||||
|
subp = RunAnsible.abort_status[subcloud_name]['subp']
|
||||||
|
while os.path.exists(unabortable_flag) and timeout > 0:
|
||||||
|
time.sleep(1)
|
||||||
|
timeout -= 1
|
||||||
|
kill_subprocess_group(subp)
|
||||||
|
return True
|
||||||
|
|
||||||
|
def exec_playbook(self, log_file, playbook_command, subcloud_name,
|
||||||
|
timeout=None, register_cleanup=True):
|
||||||
|
"""Run ansible playbook via subprocess.
|
||||||
|
|
||||||
|
:param log_file: Logs output to file
|
||||||
|
:param timeout: Timeout in seconds. Raises PlaybookExecutionTimeout
|
||||||
|
on timeout
|
||||||
|
:param register_cleanup: Register the subprocess group for cleanup on
|
||||||
|
shutdown, if the underlying service supports cleanup.
|
||||||
|
"""
|
||||||
|
exec_env = os.environ.copy()
|
||||||
|
exec_env["ANSIBLE_LOG_PATH"] = "/dev/null"
|
||||||
|
|
||||||
|
aborted = False
|
||||||
|
|
||||||
|
if timeout:
|
||||||
|
timeout_log_str = " (timeout: %ss)" % timeout
|
||||||
|
else:
|
||||||
|
timeout_log_str = ''
|
||||||
|
|
||||||
|
with open(log_file, "a+") as f_out_log:
|
||||||
|
try:
|
||||||
|
logged_playbook_command = \
|
||||||
|
_strip_password_from_command(playbook_command)
|
||||||
|
txt = "%s Executing playbook command%s: %s\n" \
|
||||||
|
% (datetime.today().strftime('%Y-%m-%d-%H:%M:%S'),
|
||||||
|
timeout_log_str,
|
||||||
|
logged_playbook_command)
|
||||||
|
f_out_log.write(txt)
|
||||||
|
f_out_log.flush()
|
||||||
|
|
||||||
|
# Remove unabortable flag created by the playbook
|
||||||
|
# if present from previous executions
|
||||||
|
unabortable_flag = os.path.join(consts.ANSIBLE_OVERRIDES_PATH,
|
||||||
|
'.%s_deploy_not_abortable' % subcloud_name)
|
||||||
|
if os.path.exists(unabortable_flag):
|
||||||
|
os.remove(unabortable_flag)
|
||||||
|
|
||||||
|
subp = subprocess.Popen(playbook_command,
|
||||||
|
stdout=f_out_log,
|
||||||
|
stderr=f_out_log,
|
||||||
|
env=exec_env,
|
||||||
|
start_new_session=register_cleanup)
|
||||||
|
try:
|
||||||
|
if register_cleanup:
|
||||||
|
SubprocessCleanup.register_subprocess_group(subp)
|
||||||
|
with RunAnsible.lock:
|
||||||
|
RunAnsible.abort_status[subcloud_name] = {
|
||||||
|
'abort': False,
|
||||||
|
'subp': subp}
|
||||||
|
|
||||||
|
subp.wait(timeout)
|
||||||
|
subp_rc = subp.poll()
|
||||||
|
|
||||||
|
# There are 5 possible outcomes of the subprocess execution:
|
||||||
|
# 1: Playbook completed (process exited)
|
||||||
|
# - playbook_failure is False with subp_rc == 0,
|
||||||
|
# aborted is False, unabortable_flag_exists is False
|
||||||
|
# 2: Playbook was aborted (process killed)
|
||||||
|
# - playbook_failure is False with subp_rc != 0,
|
||||||
|
# aborted is True, unabortable_flag_exists is False
|
||||||
|
# 3: Playbook failed (process exited)
|
||||||
|
# - playbook_failure is True with subp_rc != 0,
|
||||||
|
# aborted is False, unabortable_flag_exists is False
|
||||||
|
# 4: Playbook failed during unabortable task (process exited)
|
||||||
|
# - playbook_failure is True with subp_rc != 0,
|
||||||
|
# aborted is False, unabortable_flag_exists is True
|
||||||
|
# 5: Playbook failed while waiting to be aborted (process exited)
|
||||||
|
# - playbook_failure is True with subp_rc != 0,
|
||||||
|
# aborted is True, unabortable_flag_exists is False
|
||||||
|
with RunAnsible.lock:
|
||||||
|
aborted = RunAnsible.abort_status[subcloud_name]['abort']
|
||||||
|
unabortable_flag_exists = os.path.exists(unabortable_flag)
|
||||||
|
playbook_failure = (subp_rc != 0 and
|
||||||
|
(not aborted or unabortable_flag_exists))
|
||||||
|
|
||||||
|
# Raise PlaybookExecutionFailed if the playbook fails when
|
||||||
|
# on normal conditions (no abort issued) or fails while
|
||||||
|
# waiting for the unabortable flag to be cleared.
|
||||||
|
if playbook_failure:
|
||||||
|
raise PlaybookExecutionFailed(playbook_cmd=playbook_command)
|
||||||
|
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
kill_subprocess_group(subp)
|
||||||
|
f_out_log.write(
|
||||||
|
"%s TIMEOUT (%ss) - playbook is terminated\n" %
|
||||||
|
(datetime.today().strftime('%Y-%m-%d-%H:%M:%S'), timeout)
|
||||||
|
)
|
||||||
|
raise PlaybookExecutionTimeout(playbook_cmd=playbook_command,
|
||||||
|
timeout=timeout)
|
||||||
|
finally:
|
||||||
|
f_out_log.flush()
|
||||||
|
if register_cleanup:
|
||||||
|
SubprocessCleanup.unregister_subprocess_group(subp)
|
||||||
|
self._unregister_subcloud(subcloud_name)
|
||||||
|
|
||||||
|
except PlaybookExecutionFailed:
|
||||||
|
raise
|
||||||
|
except Exception as ex:
|
||||||
|
LOG.error(str(ex))
|
||||||
|
raise
|
||||||
|
return aborted
|
||||||
|
|
||||||
|
|
||||||
def _strip_password_from_command(script_command):
|
def _strip_password_from_command(script_command):
|
||||||
"""Strip out any known password arguments from given command"""
|
"""Strip out any known password arguments from given command"""
|
||||||
logged_command = list()
|
logged_command = list()
|
||||||
|
@ -105,6 +255,8 @@ def _strip_password_from_command(script_command):
|
||||||
return logged_command
|
return logged_command
|
||||||
|
|
||||||
|
|
||||||
|
# TODO(vgluzrom): remove this function and replace all calls
|
||||||
|
# with RunAnsible class
|
||||||
def run_playbook(log_file, playbook_command,
|
def run_playbook(log_file, playbook_command,
|
||||||
timeout=None, register_cleanup=True):
|
timeout=None, register_cleanup=True):
|
||||||
"""Run ansible playbook via subprocess.
|
"""Run ansible playbook via subprocess.
|
||||||
|
|
|
@ -61,11 +61,13 @@ VALID_STATES_FOR_DEPLOY_INSTALL = (
|
||||||
consts.DEPLOY_STATE_CREATED,
|
consts.DEPLOY_STATE_CREATED,
|
||||||
consts.DEPLOY_STATE_PRE_INSTALL_FAILED,
|
consts.DEPLOY_STATE_PRE_INSTALL_FAILED,
|
||||||
consts.DEPLOY_STATE_INSTALL_FAILED,
|
consts.DEPLOY_STATE_INSTALL_FAILED,
|
||||||
consts.DEPLOY_STATE_INSTALLED
|
consts.DEPLOY_STATE_INSTALLED,
|
||||||
|
consts.DEPLOY_STATE_INSTALL_ABORTED
|
||||||
)
|
)
|
||||||
|
|
||||||
VALID_STATES_FOR_DEPLOY_BOOTSTRAP = [
|
VALID_STATES_FOR_DEPLOY_BOOTSTRAP = [
|
||||||
consts.DEPLOY_STATE_INSTALLED,
|
consts.DEPLOY_STATE_INSTALLED,
|
||||||
|
consts.DEPLOY_STATE_PRE_BOOTSTRAP_FAILED,
|
||||||
consts.DEPLOY_STATE_BOOTSTRAP_FAILED,
|
consts.DEPLOY_STATE_BOOTSTRAP_FAILED,
|
||||||
consts.DEPLOY_STATE_BOOTSTRAP_ABORTED,
|
consts.DEPLOY_STATE_BOOTSTRAP_ABORTED,
|
||||||
consts.DEPLOY_STATE_BOOTSTRAPPED,
|
consts.DEPLOY_STATE_BOOTSTRAPPED,
|
||||||
|
@ -81,7 +83,14 @@ VALID_STATES_FOR_DEPLOY_CONFIG = (
|
||||||
consts.DEPLOY_STATE_PRE_CONFIG_FAILED,
|
consts.DEPLOY_STATE_PRE_CONFIG_FAILED,
|
||||||
consts.DEPLOY_STATE_CONFIG_FAILED,
|
consts.DEPLOY_STATE_CONFIG_FAILED,
|
||||||
consts.DEPLOY_STATE_DEPLOY_FAILED,
|
consts.DEPLOY_STATE_DEPLOY_FAILED,
|
||||||
consts.DEPLOY_STATE_BOOTSTRAPPED
|
consts.DEPLOY_STATE_BOOTSTRAPPED,
|
||||||
|
consts.DEPLOY_STATE_CONFIG_ABORTED
|
||||||
|
)
|
||||||
|
|
||||||
|
VALID_STATES_FOR_DEPLOY_ABORT = (
|
||||||
|
consts.DEPLOY_STATE_INSTALLING,
|
||||||
|
consts.DEPLOY_STATE_BOOTSTRAPPING,
|
||||||
|
consts.DEPLOY_STATE_CONFIGURING
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -288,15 +297,38 @@ class PhasedSubcloudDeployController(object):
|
||||||
psd_common.validate_sysadmin_password(payload)
|
psd_common.validate_sysadmin_password(payload)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
subcloud = self.dcmanager_rpc_client.subcloud_deploy_config(
|
self.dcmanager_rpc_client.subcloud_deploy_config(
|
||||||
context, subcloud.id, payload)
|
context, subcloud.id, payload)
|
||||||
return subcloud
|
subcloud_dict = db_api.subcloud_db_model_to_dict(subcloud)
|
||||||
|
subcloud_dict['deploy-status'] = consts.DEPLOY_STATE_PRE_CONFIG
|
||||||
|
return subcloud_dict
|
||||||
except RemoteError as e:
|
except RemoteError as e:
|
||||||
pecan.abort(422, e.value)
|
pecan.abort(422, e.value)
|
||||||
except Exception:
|
except Exception:
|
||||||
LOG.exception("Unable to configure subcloud %s" % subcloud.name)
|
LOG.exception("Unable to configure subcloud %s" % subcloud.name)
|
||||||
pecan.abort(500, _('Unable to configure subcloud'))
|
pecan.abort(500, _('Unable to configure subcloud'))
|
||||||
|
|
||||||
|
def _deploy_abort(self, context, subcloud):
|
||||||
|
|
||||||
|
if subcloud.deploy_status not in VALID_STATES_FOR_DEPLOY_ABORT:
|
||||||
|
allowed_states_str = ', '.join(VALID_STATES_FOR_DEPLOY_ABORT)
|
||||||
|
pecan.abort(400, _('Subcloud deploy status must be in one '
|
||||||
|
'of the following states: %s')
|
||||||
|
% allowed_states_str)
|
||||||
|
|
||||||
|
try:
|
||||||
|
self.dcmanager_rpc_client.subcloud_deploy_abort(
|
||||||
|
context, subcloud.id, subcloud.deploy_status)
|
||||||
|
subcloud_dict = db_api.subcloud_db_model_to_dict(subcloud)
|
||||||
|
subcloud_dict['deploy-status'] = \
|
||||||
|
utils.ABORT_UPDATE_STATUS[subcloud.deploy_status]
|
||||||
|
return subcloud_dict
|
||||||
|
except RemoteError as e:
|
||||||
|
pecan.abort(422, e.value)
|
||||||
|
except Exception:
|
||||||
|
LOG.exception("Unable to abort subcloud %s deployment" % subcloud.name)
|
||||||
|
pecan.abort(500, _('Unable to abort subcloud deploy'))
|
||||||
|
|
||||||
@pecan.expose(generic=True, template='json')
|
@pecan.expose(generic=True, template='json')
|
||||||
def index(self):
|
def index(self):
|
||||||
# Route the request to specific methods with parameters
|
# Route the request to specific methods with parameters
|
||||||
|
@ -334,7 +366,9 @@ class PhasedSubcloudDeployController(object):
|
||||||
except (exceptions.SubcloudNotFound, exceptions.SubcloudNameNotFound):
|
except (exceptions.SubcloudNotFound, exceptions.SubcloudNameNotFound):
|
||||||
pecan.abort(404, _('Subcloud not found'))
|
pecan.abort(404, _('Subcloud not found'))
|
||||||
|
|
||||||
if verb == 'install':
|
if verb == 'abort':
|
||||||
|
subcloud = self._deploy_abort(context, subcloud)
|
||||||
|
elif verb == 'install':
|
||||||
subcloud = self._deploy_install(context, pecan.request, subcloud)
|
subcloud = self._deploy_install(context, pecan.request, subcloud)
|
||||||
elif verb == 'bootstrap':
|
elif verb == 'bootstrap':
|
||||||
subcloud = self._deploy_bootstrap(context, pecan.request, subcloud)
|
subcloud = self._deploy_bootstrap(context, pecan.request, subcloud)
|
||||||
|
|
|
@ -281,17 +281,17 @@ class SubcloudsController(object):
|
||||||
def _get_config_file_path(self, subcloud_name, config_file_type=None):
|
def _get_config_file_path(self, subcloud_name, config_file_type=None):
|
||||||
if config_file_type == consts.DEPLOY_CONFIG:
|
if config_file_type == consts.DEPLOY_CONFIG:
|
||||||
file_path = os.path.join(
|
file_path = os.path.join(
|
||||||
consts.ANSIBLE_OVERRIDES_PATH,
|
dccommon_consts.ANSIBLE_OVERRIDES_PATH,
|
||||||
subcloud_name + '_' + config_file_type + '.yml'
|
subcloud_name + '_' + config_file_type + '.yml'
|
||||||
)
|
)
|
||||||
elif config_file_type == INSTALL_VALUES:
|
elif config_file_type == INSTALL_VALUES:
|
||||||
file_path = os.path.join(
|
file_path = os.path.join(
|
||||||
consts.ANSIBLE_OVERRIDES_PATH + '/' + subcloud_name,
|
dccommon_consts.ANSIBLE_OVERRIDES_PATH + '/' + subcloud_name,
|
||||||
config_file_type + '.yml'
|
config_file_type + '.yml'
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
file_path = os.path.join(
|
file_path = os.path.join(
|
||||||
consts.ANSIBLE_OVERRIDES_PATH,
|
dccommon_consts.ANSIBLE_OVERRIDES_PATH,
|
||||||
subcloud_name + '.yml'
|
subcloud_name + '.yml'
|
||||||
)
|
)
|
||||||
return file_path
|
return file_path
|
||||||
|
|
|
@ -27,6 +27,10 @@ phased_subcloud_deploy_rules = [
|
||||||
check_str='rule:' + base.ADMIN_IN_SYSTEM_PROJECTS,
|
check_str='rule:' + base.ADMIN_IN_SYSTEM_PROJECTS,
|
||||||
description="Modify the subcloud deployment.",
|
description="Modify the subcloud deployment.",
|
||||||
operations=[
|
operations=[
|
||||||
|
{
|
||||||
|
'method': 'PATCH',
|
||||||
|
'path': '/v1.0/phased-subcloud-deploy/{subcloud}/abort'
|
||||||
|
},
|
||||||
{
|
{
|
||||||
'method': 'PATCH',
|
'method': 'PATCH',
|
||||||
'path': '/v1.0/phased-subcloud-deploy/{subcloud}/install'
|
'path': '/v1.0/phased-subcloud-deploy/{subcloud}/install'
|
||||||
|
|
|
@ -185,6 +185,12 @@ DEPLOY_STATE_CONFIGURING = 'configuring'
|
||||||
DEPLOY_STATE_CONFIG_FAILED = 'config-failed'
|
DEPLOY_STATE_CONFIG_FAILED = 'config-failed'
|
||||||
DEPLOY_STATE_DEPLOYING = 'deploying'
|
DEPLOY_STATE_DEPLOYING = 'deploying'
|
||||||
DEPLOY_STATE_DEPLOY_FAILED = 'deploy-failed'
|
DEPLOY_STATE_DEPLOY_FAILED = 'deploy-failed'
|
||||||
|
DEPLOY_STATE_ABORTING_INSTALL = 'aborting-install'
|
||||||
|
DEPLOY_STATE_INSTALL_ABORTED = 'install-aborted'
|
||||||
|
DEPLOY_STATE_ABORTING_BOOTSTRAP = 'aborting-bootstrap'
|
||||||
|
DEPLOY_STATE_BOOTSTRAP_ABORTED = 'bootstrap-aborted'
|
||||||
|
DEPLOY_STATE_ABORTING_CONFIG = 'aborting-config'
|
||||||
|
DEPLOY_STATE_CONFIG_ABORTED = 'config-aborted'
|
||||||
DEPLOY_STATE_MIGRATING_DATA = 'migrating-data'
|
DEPLOY_STATE_MIGRATING_DATA = 'migrating-data'
|
||||||
DEPLOY_STATE_DATA_MIGRATION_FAILED = 'data-migration-failed'
|
DEPLOY_STATE_DATA_MIGRATION_FAILED = 'data-migration-failed'
|
||||||
DEPLOY_STATE_MIGRATED = 'migrated'
|
DEPLOY_STATE_MIGRATED = 'migrated'
|
||||||
|
@ -281,8 +287,6 @@ ALARM_OK_STATUS = "OK"
|
||||||
ALARM_DEGRADED_STATUS = "degraded"
|
ALARM_DEGRADED_STATUS = "degraded"
|
||||||
ALARM_CRITICAL_STATUS = "critical"
|
ALARM_CRITICAL_STATUS = "critical"
|
||||||
|
|
||||||
# subcloud deploy file options
|
|
||||||
ANSIBLE_OVERRIDES_PATH = '/opt/dc-vault/ansible'
|
|
||||||
DEPLOY_PLAYBOOK = "deploy_playbook"
|
DEPLOY_PLAYBOOK = "deploy_playbook"
|
||||||
DEPLOY_OVERRIDES = "deploy_overrides"
|
DEPLOY_OVERRIDES = "deploy_overrides"
|
||||||
DEPLOY_CHART = "deploy_chart"
|
DEPLOY_CHART = "deploy_chart"
|
||||||
|
|
|
@ -692,7 +692,7 @@ def upload_deploy_config_file(request, payload):
|
||||||
|
|
||||||
|
|
||||||
def get_config_file_path(subcloud_name, config_file_type=None):
|
def get_config_file_path(subcloud_name, config_file_type=None):
|
||||||
basepath = consts.ANSIBLE_OVERRIDES_PATH
|
basepath = dccommon_consts.ANSIBLE_OVERRIDES_PATH
|
||||||
if config_file_type == consts.DEPLOY_CONFIG:
|
if config_file_type == consts.DEPLOY_CONFIG:
|
||||||
filename = f"{subcloud_name}_{config_file_type}.yml"
|
filename = f"{subcloud_name}_{config_file_type}.yml"
|
||||||
elif config_file_type == consts.INSTALL_VALUES:
|
elif config_file_type == consts.INSTALL_VALUES:
|
||||||
|
|
|
@ -54,6 +54,21 @@ DC_MANAGER_GRPNAME = "root"
|
||||||
# Max lines output msg from logs
|
# Max lines output msg from logs
|
||||||
MAX_LINES_MSG = 10
|
MAX_LINES_MSG = 10
|
||||||
|
|
||||||
|
ABORT_UPDATE_STATUS = {
|
||||||
|
consts.DEPLOY_STATE_INSTALLING: consts.DEPLOY_STATE_ABORTING_INSTALL,
|
||||||
|
consts.DEPLOY_STATE_BOOTSTRAPPING: consts.DEPLOY_STATE_ABORTING_BOOTSTRAP,
|
||||||
|
consts.DEPLOY_STATE_CONFIGURING: consts.DEPLOY_STATE_ABORTING_CONFIG,
|
||||||
|
consts.DEPLOY_STATE_ABORTING_INSTALL: consts.DEPLOY_STATE_INSTALL_ABORTED,
|
||||||
|
consts.DEPLOY_STATE_ABORTING_BOOTSTRAP: consts.DEPLOY_STATE_BOOTSTRAP_ABORTED,
|
||||||
|
consts.DEPLOY_STATE_ABORTING_CONFIG: consts.DEPLOY_STATE_CONFIG_ABORTED
|
||||||
|
}
|
||||||
|
|
||||||
|
ABORT_UPDATE_FAIL_STATUS = {
|
||||||
|
consts.DEPLOY_STATE_ABORTING_INSTALL: consts.DEPLOY_STATE_INSTALL_FAILED,
|
||||||
|
consts.DEPLOY_STATE_ABORTING_BOOTSTRAP: consts.DEPLOY_STATE_BOOTSTRAP_FAILED,
|
||||||
|
consts.DEPLOY_STATE_ABORTING_CONFIG: consts.DEPLOY_STATE_CONFIG_FAILED
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def get_import_path(cls):
|
def get_import_path(cls):
|
||||||
return cls.__module__ + "." + cls.__name__
|
return cls.__module__ + "." + cls.__name__
|
||||||
|
@ -526,7 +541,7 @@ def get_oam_addresses(subcloud_name, sc_ks_client):
|
||||||
|
|
||||||
def get_ansible_filename(subcloud_name, postfix='.yml'):
|
def get_ansible_filename(subcloud_name, postfix='.yml'):
|
||||||
"""Build ansible filename using subcloud and given postfix"""
|
"""Build ansible filename using subcloud and given postfix"""
|
||||||
ansible_filename = os.path.join(consts.ANSIBLE_OVERRIDES_PATH,
|
ansible_filename = os.path.join(dccommon_consts.ANSIBLE_OVERRIDES_PATH,
|
||||||
subcloud_name + postfix)
|
subcloud_name + postfix)
|
||||||
return ansible_filename
|
return ansible_filename
|
||||||
|
|
||||||
|
@ -1025,3 +1040,21 @@ def get_failure_msg(subcloud_name):
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
LOG.exception("{}: {}".format(subcloud_name, e))
|
LOG.exception("{}: {}".format(subcloud_name, e))
|
||||||
return consts.ERROR_DESC_FAILED
|
return consts.ERROR_DESC_FAILED
|
||||||
|
|
||||||
|
|
||||||
|
def update_abort_status(context, subcloud_id, deploy_status, abort_failed=False):
|
||||||
|
"""Update the subcloud deploy status during deploy abort operation.
|
||||||
|
|
||||||
|
:param context: request context object
|
||||||
|
:param subcloud_id: subcloud id from db
|
||||||
|
:param deploy_status: subcloud deploy status from db
|
||||||
|
:param abort_failed: if abort process fails (default False)
|
||||||
|
"""
|
||||||
|
if abort_failed:
|
||||||
|
abort_status_dict = ABORT_UPDATE_FAIL_STATUS
|
||||||
|
else:
|
||||||
|
abort_status_dict = ABORT_UPDATE_STATUS
|
||||||
|
new_deploy_status = abort_status_dict[deploy_status]
|
||||||
|
updated_subcloud = db_api.subcloud_update(context, subcloud_id,
|
||||||
|
deploy_status=new_deploy_status)
|
||||||
|
return updated_subcloud
|
||||||
|
|
|
@ -22,6 +22,7 @@ import oslo_messaging
|
||||||
from oslo_service import service
|
from oslo_service import service
|
||||||
from oslo_utils import uuidutils
|
from oslo_utils import uuidutils
|
||||||
|
|
||||||
|
from dccommon import consts as dccommon_consts
|
||||||
from dccommon.subprocess_cleanup import SubprocessCleanup
|
from dccommon.subprocess_cleanup import SubprocessCleanup
|
||||||
from dcmanager.audit import rpcapi as dcmanager_audit_rpc_client
|
from dcmanager.audit import rpcapi as dcmanager_audit_rpc_client
|
||||||
from dcmanager.common import consts
|
from dcmanager.common import consts
|
||||||
|
@ -92,7 +93,7 @@ class DCManagerService(service.Service):
|
||||||
if not os.path.isdir(consts.DC_ANSIBLE_LOG_DIR):
|
if not os.path.isdir(consts.DC_ANSIBLE_LOG_DIR):
|
||||||
os.mkdir(consts.DC_ANSIBLE_LOG_DIR, 0o755)
|
os.mkdir(consts.DC_ANSIBLE_LOG_DIR, 0o755)
|
||||||
|
|
||||||
os.makedirs(consts.ANSIBLE_OVERRIDES_PATH, 0o600, exist_ok=True)
|
os.makedirs(dccommon_consts.ANSIBLE_OVERRIDES_PATH, 0o600, exist_ok=True)
|
||||||
|
|
||||||
self.subcloud_manager.handle_subcloud_operations_in_progress()
|
self.subcloud_manager.handle_subcloud_operations_in_progress()
|
||||||
super(DCManagerService, self).start()
|
super(DCManagerService, self).start()
|
||||||
|
@ -224,6 +225,14 @@ class DCManagerService(service.Service):
|
||||||
subcloud_id,
|
subcloud_id,
|
||||||
payload)
|
payload)
|
||||||
|
|
||||||
|
@request_context
|
||||||
|
def subcloud_deploy_abort(self, context, subcloud_id, deploy_status):
|
||||||
|
# Abort the subcloud deployment
|
||||||
|
LOG.info("Handling subcloud_deploy_abort request for: %s" % subcloud_id)
|
||||||
|
return self.subcloud_manager.subcloud_deploy_abort(context,
|
||||||
|
subcloud_id,
|
||||||
|
deploy_status)
|
||||||
|
|
||||||
def _stop_rpc_server(self):
|
def _stop_rpc_server(self):
|
||||||
# Stop RPC connection to prevent new requests
|
# Stop RPC connection to prevent new requests
|
||||||
LOG.debug(_("Attempting to stop RPC service..."))
|
LOG.debug(_("Attempting to stop RPC service..."))
|
||||||
|
|
|
@ -44,7 +44,9 @@ from dccommon.drivers.openstack.sysinv_v1 import SysinvClient
|
||||||
from dccommon.exceptions import PlaybookExecutionFailed
|
from dccommon.exceptions import PlaybookExecutionFailed
|
||||||
from dccommon import kubeoperator
|
from dccommon import kubeoperator
|
||||||
from dccommon.subcloud_install import SubcloudInstall
|
from dccommon.subcloud_install import SubcloudInstall
|
||||||
|
from dccommon.subcloud_install import SubcloudShutdown
|
||||||
from dccommon.utils import run_playbook
|
from dccommon.utils import run_playbook
|
||||||
|
from dccommon.utils import RunAnsible
|
||||||
from dcmanager.audit import rpcapi as dcmanager_audit_rpc_client
|
from dcmanager.audit import rpcapi as dcmanager_audit_rpc_client
|
||||||
from dcmanager.common import consts
|
from dcmanager.common import consts
|
||||||
from dcmanager.common.consts import INVENTORY_FILE_POSTFIX
|
from dcmanager.common.consts import INVENTORY_FILE_POSTFIX
|
||||||
|
@ -110,6 +112,9 @@ TRANSITORY_STATES = {
|
||||||
consts.DEPLOY_STATE_PRE_CONFIG: consts.DEPLOY_STATE_PRE_CONFIG_FAILED,
|
consts.DEPLOY_STATE_PRE_CONFIG: consts.DEPLOY_STATE_PRE_CONFIG_FAILED,
|
||||||
consts.DEPLOY_STATE_CONFIGURING: consts.DEPLOY_STATE_CONFIG_FAILED,
|
consts.DEPLOY_STATE_CONFIGURING: consts.DEPLOY_STATE_CONFIG_FAILED,
|
||||||
consts.DEPLOY_STATE_DEPLOYING: consts.DEPLOY_STATE_DEPLOY_FAILED,
|
consts.DEPLOY_STATE_DEPLOYING: consts.DEPLOY_STATE_DEPLOY_FAILED,
|
||||||
|
consts.DEPLOY_STATE_ABORTING_INSTALL: consts.DEPLOY_STATE_INSTALL_FAILED,
|
||||||
|
consts.DEPLOY_STATE_ABORTING_BOOTSTRAP: consts.DEPLOY_STATE_BOOTSTRAP_FAILED,
|
||||||
|
consts.DEPLOY_STATE_ABORTING_CONFIG: consts.DEPLOY_STATE_CONFIG_FAILED,
|
||||||
consts.DEPLOY_STATE_MIGRATING_DATA: consts.DEPLOY_STATE_DATA_MIGRATION_FAILED,
|
consts.DEPLOY_STATE_MIGRATING_DATA: consts.DEPLOY_STATE_DATA_MIGRATION_FAILED,
|
||||||
consts.DEPLOY_STATE_PRE_RESTORE: consts.DEPLOY_STATE_RESTORE_PREP_FAILED,
|
consts.DEPLOY_STATE_PRE_RESTORE: consts.DEPLOY_STATE_RESTORE_PREP_FAILED,
|
||||||
consts.DEPLOY_STATE_RESTORING: consts.DEPLOY_STATE_RESTORE_FAILED,
|
consts.DEPLOY_STATE_RESTORING: consts.DEPLOY_STATE_RESTORE_FAILED,
|
||||||
|
@ -224,7 +229,7 @@ class SubcloudManager(manager.Manager):
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _get_ansible_filename(subcloud_name, postfix='.yml'):
|
def _get_ansible_filename(subcloud_name, postfix='.yml'):
|
||||||
ansible_filename = os.path.join(
|
ansible_filename = os.path.join(
|
||||||
consts.ANSIBLE_OVERRIDES_PATH,
|
dccommon_consts.ANSIBLE_OVERRIDES_PATH,
|
||||||
subcloud_name + postfix)
|
subcloud_name + postfix)
|
||||||
return ansible_filename
|
return ansible_filename
|
||||||
|
|
||||||
|
@ -235,7 +240,7 @@ class SubcloudManager(manager.Manager):
|
||||||
"ansible-playbook", dccommon_consts.ANSIBLE_SUBCLOUD_INSTALL_PLAYBOOK,
|
"ansible-playbook", dccommon_consts.ANSIBLE_SUBCLOUD_INSTALL_PLAYBOOK,
|
||||||
"-i", ansible_subcloud_inventory_file,
|
"-i", ansible_subcloud_inventory_file,
|
||||||
"--limit", subcloud_name,
|
"--limit", subcloud_name,
|
||||||
"-e", "@%s" % consts.ANSIBLE_OVERRIDES_PATH + "/" +
|
"-e", "@%s" % dccommon_consts.ANSIBLE_OVERRIDES_PATH + "/" +
|
||||||
subcloud_name + '/' + "install_values.yml",
|
subcloud_name + '/' + "install_values.yml",
|
||||||
"-e", "install_release_version=%s" %
|
"-e", "install_release_version=%s" %
|
||||||
software_version if software_version else SW_VERSION]
|
software_version if software_version else SW_VERSION]
|
||||||
|
@ -256,7 +261,7 @@ class SubcloudManager(manager.Manager):
|
||||||
# which overrides to load
|
# which overrides to load
|
||||||
apply_command += [
|
apply_command += [
|
||||||
"-e", str("override_files_dir='%s' region_name=%s") % (
|
"-e", str("override_files_dir='%s' region_name=%s") % (
|
||||||
consts.ANSIBLE_OVERRIDES_PATH, subcloud_name),
|
dccommon_consts.ANSIBLE_OVERRIDES_PATH, subcloud_name),
|
||||||
"-e", "install_release_version=%s" %
|
"-e", "install_release_version=%s" %
|
||||||
software_version if software_version else SW_VERSION]
|
software_version if software_version else SW_VERSION]
|
||||||
return apply_command
|
return apply_command
|
||||||
|
@ -265,7 +270,7 @@ class SubcloudManager(manager.Manager):
|
||||||
def compose_deploy_command(self, subcloud_name, ansible_subcloud_inventory_file, payload):
|
def compose_deploy_command(self, subcloud_name, ansible_subcloud_inventory_file, payload):
|
||||||
deploy_command = [
|
deploy_command = [
|
||||||
"ansible-playbook", payload[consts.DEPLOY_PLAYBOOK],
|
"ansible-playbook", payload[consts.DEPLOY_PLAYBOOK],
|
||||||
"-e", "@%s" % consts.ANSIBLE_OVERRIDES_PATH + "/" +
|
"-e", "@%s" % dccommon_consts.ANSIBLE_OVERRIDES_PATH + "/" +
|
||||||
subcloud_name + '_deploy_values.yml',
|
subcloud_name + '_deploy_values.yml',
|
||||||
"-i", ansible_subcloud_inventory_file,
|
"-i", ansible_subcloud_inventory_file,
|
||||||
"--limit", subcloud_name
|
"--limit", subcloud_name
|
||||||
|
@ -277,7 +282,7 @@ class SubcloudManager(manager.Manager):
|
||||||
"ansible-playbook", ANSIBLE_SUBCLOUD_BACKUP_CREATE_PLAYBOOK,
|
"ansible-playbook", ANSIBLE_SUBCLOUD_BACKUP_CREATE_PLAYBOOK,
|
||||||
"-i", ansible_subcloud_inventory_file,
|
"-i", ansible_subcloud_inventory_file,
|
||||||
"--limit", subcloud_name,
|
"--limit", subcloud_name,
|
||||||
"-e", "subcloud_bnr_overrides=%s" % consts.ANSIBLE_OVERRIDES_PATH + "/" +
|
"-e", "subcloud_bnr_overrides=%s" % dccommon_consts.ANSIBLE_OVERRIDES_PATH + "/" +
|
||||||
subcloud_name + "_backup_create_values.yml"]
|
subcloud_name + "_backup_create_values.yml"]
|
||||||
|
|
||||||
return backup_command
|
return backup_command
|
||||||
|
@ -288,7 +293,7 @@ class SubcloudManager(manager.Manager):
|
||||||
"ansible-playbook", ANSIBLE_SUBCLOUD_BACKUP_DELETE_PLAYBOOK,
|
"ansible-playbook", ANSIBLE_SUBCLOUD_BACKUP_DELETE_PLAYBOOK,
|
||||||
"-i", ansible_subcloud_inventory_file,
|
"-i", ansible_subcloud_inventory_file,
|
||||||
"--limit", subcloud_name,
|
"--limit", subcloud_name,
|
||||||
"-e", "subcloud_bnr_overrides=%s" % consts.ANSIBLE_OVERRIDES_PATH + "/" +
|
"-e", "subcloud_bnr_overrides=%s" % dccommon_consts.ANSIBLE_OVERRIDES_PATH + "/" +
|
||||||
subcloud_name + "_backup_delete_values.yml"]
|
subcloud_name + "_backup_delete_values.yml"]
|
||||||
|
|
||||||
return backup_command
|
return backup_command
|
||||||
|
@ -298,7 +303,7 @@ class SubcloudManager(manager.Manager):
|
||||||
"ansible-playbook", ANSIBLE_SUBCLOUD_BACKUP_RESTORE_PLAYBOOK,
|
"ansible-playbook", ANSIBLE_SUBCLOUD_BACKUP_RESTORE_PLAYBOOK,
|
||||||
"-i", ansible_subcloud_inventory_file,
|
"-i", ansible_subcloud_inventory_file,
|
||||||
"--limit", subcloud_name,
|
"--limit", subcloud_name,
|
||||||
"-e", "subcloud_bnr_overrides=%s" % consts.ANSIBLE_OVERRIDES_PATH + "/" +
|
"-e", "subcloud_bnr_overrides=%s" % dccommon_consts.ANSIBLE_OVERRIDES_PATH + "/" +
|
||||||
subcloud_name + "_backup_restore_values.yml"]
|
subcloud_name + "_backup_restore_values.yml"]
|
||||||
|
|
||||||
return backup_command
|
return backup_command
|
||||||
|
@ -309,7 +314,7 @@ class SubcloudManager(manager.Manager):
|
||||||
"-i", ansible_subcloud_inventory_file,
|
"-i", ansible_subcloud_inventory_file,
|
||||||
"--limit", subcloud_name,
|
"--limit", subcloud_name,
|
||||||
"--timeout", UPDATE_PLAYBOOK_TIMEOUT,
|
"--timeout", UPDATE_PLAYBOOK_TIMEOUT,
|
||||||
"-e", "subcloud_update_overrides=%s" % consts.ANSIBLE_OVERRIDES_PATH + "/" +
|
"-e", "subcloud_update_overrides=%s" % dccommon_consts.ANSIBLE_OVERRIDES_PATH + "/" +
|
||||||
subcloud_name + "_update_values.yml"]
|
subcloud_name + "_update_values.yml"]
|
||||||
return subcloud_update_command
|
return subcloud_update_command
|
||||||
|
|
||||||
|
@ -324,7 +329,7 @@ class SubcloudManager(manager.Manager):
|
||||||
"--limit", subcloud_name,
|
"--limit", subcloud_name,
|
||||||
"--timeout", REHOME_PLAYBOOK_TIMEOUT,
|
"--timeout", REHOME_PLAYBOOK_TIMEOUT,
|
||||||
"-e", str("override_files_dir='%s' region_name=%s") % (
|
"-e", str("override_files_dir='%s' region_name=%s") % (
|
||||||
consts.ANSIBLE_OVERRIDES_PATH, subcloud_name)]
|
dccommon_consts.ANSIBLE_OVERRIDES_PATH, subcloud_name)]
|
||||||
return rehome_command
|
return rehome_command
|
||||||
|
|
||||||
def add_subcloud(self, context, payload):
|
def add_subcloud(self, context, payload):
|
||||||
|
@ -784,6 +789,53 @@ class SubcloudManager(manager.Manager):
|
||||||
|
|
||||||
return install_command
|
return install_command
|
||||||
|
|
||||||
|
def subcloud_deploy_abort(self, context, subcloud_id, deploy_status):
|
||||||
|
"""Abort the subcloud deploy
|
||||||
|
|
||||||
|
:param context: request context object
|
||||||
|
:param subcloud_id: subcloud id from db
|
||||||
|
:param deploy_status: subcloud deploy status from db
|
||||||
|
"""
|
||||||
|
|
||||||
|
LOG.info("Aborting deployment of subcloud %s." % subcloud_id)
|
||||||
|
subcloud = utils.update_abort_status(context, subcloud_id, deploy_status)
|
||||||
|
|
||||||
|
try:
|
||||||
|
run_ansible = RunAnsible()
|
||||||
|
aborted = run_ansible.run_abort(subcloud.name)
|
||||||
|
if not aborted:
|
||||||
|
LOG.warning("Ansible deploy phase subprocess of %s "
|
||||||
|
"was terminated before it could be aborted"
|
||||||
|
% subcloud.name)
|
||||||
|
# let the main phase thread handle the state update
|
||||||
|
return
|
||||||
|
|
||||||
|
if subcloud.deploy_status == consts.DEPLOY_STATE_ABORTING_INSTALL:
|
||||||
|
# First delete the k8s job and pod, stopping the current
|
||||||
|
# installation attempt if exists
|
||||||
|
# Then send shutdown signal to subcloud
|
||||||
|
kube = kubeoperator.KubeOperator()
|
||||||
|
shutdown_subcloud = SubcloudShutdown(subcloud.name)
|
||||||
|
namespace = dccommon_consts.RVMC_NAME_PREFIX
|
||||||
|
jobname = '%s-%s' % (namespace, subcloud.name)
|
||||||
|
pod_basename = '%s-' % jobname
|
||||||
|
all_pods = kube.get_pods_by_namespace(namespace)
|
||||||
|
desired_pod = next((s for s in all_pods if pod_basename in s), None)
|
||||||
|
if desired_pod:
|
||||||
|
kube.kube_delete_job(jobname, namespace)
|
||||||
|
kube.kube_delete_pod(desired_pod, namespace)
|
||||||
|
while kube.pod_exists(desired_pod, namespace):
|
||||||
|
time.sleep(2)
|
||||||
|
shutdown_subcloud.send_shutdown_signal()
|
||||||
|
except Exception as ex:
|
||||||
|
LOG.error("Subcloud deploy abort failed for subcloud %s" % subcloud.name)
|
||||||
|
utils.update_abort_status(context, subcloud.id, subcloud.deploy_status,
|
||||||
|
abort_failed=True)
|
||||||
|
# exception is logged above
|
||||||
|
raise ex
|
||||||
|
LOG.info("Successfully aborted deployment of %s" % subcloud.name)
|
||||||
|
utils.update_abort_status(context, subcloud.id, subcloud.deploy_status)
|
||||||
|
|
||||||
def subcloud_deploy_create(self, context, subcloud_id, payload):
|
def subcloud_deploy_create(self, context, subcloud_id, payload):
|
||||||
"""Create subcloud and notify orchestrators.
|
"""Create subcloud and notify orchestrators.
|
||||||
|
|
||||||
|
@ -942,13 +994,9 @@ class SubcloudManager(manager.Manager):
|
||||||
|
|
||||||
install_command = self._deploy_install_prep(
|
install_command = self._deploy_install_prep(
|
||||||
subcloud, payload, ansible_subcloud_inventory_file)
|
subcloud, payload, ansible_subcloud_inventory_file)
|
||||||
|
self.run_deploy_commands(subcloud, payload, context,
|
||||||
|
install_command=install_command)
|
||||||
|
|
||||||
apply_thread = threading.Thread(
|
|
||||||
target=self.run_deploy_commands,
|
|
||||||
args=(subcloud, payload, context),
|
|
||||||
kwargs={'install_command': install_command})
|
|
||||||
apply_thread.start()
|
|
||||||
return db_api.subcloud_db_model_to_dict(subcloud)
|
|
||||||
except Exception:
|
except Exception:
|
||||||
LOG.exception("Failed to install subcloud %s" % subcloud.name)
|
LOG.exception("Failed to install subcloud %s" % subcloud.name)
|
||||||
# If we failed to install the subcloud,
|
# If we failed to install the subcloud,
|
||||||
|
@ -1011,7 +1059,7 @@ class SubcloudManager(manager.Manager):
|
||||||
del payload['sysadmin_password']
|
del payload['sysadmin_password']
|
||||||
|
|
||||||
# Update the ansible overrides file
|
# Update the ansible overrides file
|
||||||
overrides_file = os.path.join(consts.ANSIBLE_OVERRIDES_PATH,
|
overrides_file = os.path.join(dccommon_consts.ANSIBLE_OVERRIDES_PATH,
|
||||||
subcloud.name + '.yml')
|
subcloud.name + '.yml')
|
||||||
utils.update_values_on_yaml_file(overrides_file, payload)
|
utils.update_values_on_yaml_file(overrides_file, payload)
|
||||||
|
|
||||||
|
@ -1059,13 +1107,9 @@ class SubcloudManager(manager.Manager):
|
||||||
ansible_subcloud_inventory_file,
|
ansible_subcloud_inventory_file,
|
||||||
payload)
|
payload)
|
||||||
|
|
||||||
del payload['sysadmin_password']
|
self.run_deploy_commands(subcloud, payload, context,
|
||||||
apply_thread = threading.Thread(
|
deploy_command=deploy_command)
|
||||||
target=self.run_deploy_commands,
|
|
||||||
args=(subcloud, payload, context),
|
|
||||||
kwargs={'deploy_command': deploy_command})
|
|
||||||
apply_thread.start()
|
|
||||||
return db_api.subcloud_db_model_to_dict(subcloud)
|
|
||||||
except Exception:
|
except Exception:
|
||||||
LOG.exception("Failed to configure %s" % subcloud.name)
|
LOG.exception("Failed to configure %s" % subcloud.name)
|
||||||
db_api.subcloud_update(
|
db_api.subcloud_update(
|
||||||
|
@ -1431,7 +1475,7 @@ class SubcloudManager(manager.Manager):
|
||||||
|
|
||||||
def _create_backup_overrides_file(self, payload, subcloud_name, filename_suffix):
|
def _create_backup_overrides_file(self, payload, subcloud_name, filename_suffix):
|
||||||
backup_overrides_file = os.path.join(
|
backup_overrides_file = os.path.join(
|
||||||
consts.ANSIBLE_OVERRIDES_PATH, subcloud_name + '_' +
|
dccommon_consts.ANSIBLE_OVERRIDES_PATH, subcloud_name + '_' +
|
||||||
filename_suffix + '.yml')
|
filename_suffix + '.yml')
|
||||||
|
|
||||||
with open(backup_overrides_file, 'w') as f_out:
|
with open(backup_overrides_file, 'w') as f_out:
|
||||||
|
@ -1722,7 +1766,8 @@ class SubcloudManager(manager.Manager):
|
||||||
if install_command:
|
if install_command:
|
||||||
install_success = self._run_subcloud_install(
|
install_success = self._run_subcloud_install(
|
||||||
context, subcloud, install_command,
|
context, subcloud, install_command,
|
||||||
log_file, payload['install_values'])
|
log_file, payload['install_values'],
|
||||||
|
abortable=True)
|
||||||
if not install_success:
|
if not install_success:
|
||||||
return
|
return
|
||||||
db_api.subcloud_update(
|
db_api.subcloud_update(
|
||||||
|
@ -1731,8 +1776,10 @@ class SubcloudManager(manager.Manager):
|
||||||
error_description=consts.ERROR_DESC_EMPTY)
|
error_description=consts.ERROR_DESC_EMPTY)
|
||||||
|
|
||||||
if apply_command:
|
if apply_command:
|
||||||
self._run_subcloud_bootstrap(context, subcloud,
|
bootstrap_success = self._run_subcloud_bootstrap(
|
||||||
apply_command, log_file)
|
context, subcloud, apply_command, log_file)
|
||||||
|
if not bootstrap_success:
|
||||||
|
return
|
||||||
|
|
||||||
if deploy_command:
|
if deploy_command:
|
||||||
self._run_subcloud_config(subcloud, context,
|
self._run_subcloud_config(subcloud, context,
|
||||||
|
@ -1742,9 +1789,39 @@ class SubcloudManager(manager.Manager):
|
||||||
LOG.exception("run_deploy failed")
|
LOG.exception("run_deploy failed")
|
||||||
raise ex
|
raise ex
|
||||||
|
|
||||||
|
def _run_subcloud_config(self, subcloud, context,
|
||||||
|
deploy_command, log_file):
|
||||||
|
# Run the custom deploy playbook
|
||||||
|
LOG.info("Starting deploy of %s" % subcloud.name)
|
||||||
|
db_api.subcloud_update(
|
||||||
|
context, subcloud.id,
|
||||||
|
deploy_status=consts.DEPLOY_STATE_CONFIGURING,
|
||||||
|
error_description=consts.ERROR_DESC_EMPTY)
|
||||||
|
|
||||||
|
try:
|
||||||
|
run_ansible = RunAnsible()
|
||||||
|
aborted = run_ansible.exec_playbook(
|
||||||
|
log_file, deploy_command, subcloud.name)
|
||||||
|
except PlaybookExecutionFailed:
|
||||||
|
msg = utils.find_ansible_error_msg(
|
||||||
|
subcloud.name, log_file, consts.DEPLOY_STATE_CONFIGURING)
|
||||||
|
LOG.error(msg)
|
||||||
|
db_api.subcloud_update(
|
||||||
|
context, subcloud.id,
|
||||||
|
deploy_status=consts.DEPLOY_STATE_CONFIG_FAILED,
|
||||||
|
error_description=msg[0:consts.ERROR_DESCRIPTION_LENGTH])
|
||||||
|
return False
|
||||||
|
if aborted:
|
||||||
|
return False
|
||||||
|
LOG.info("Successfully deployed %s" % subcloud.name)
|
||||||
|
db_api.subcloud_update(
|
||||||
|
context, subcloud.id,
|
||||||
|
deploy_status=consts.DEPLOY_STATE_DONE,
|
||||||
|
error_description=consts.ERROR_DESC_EMPTY)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _run_subcloud_install(
|
def _run_subcloud_install(context, subcloud, install_command,
|
||||||
context, subcloud, install_command, log_file, payload):
|
log_file, payload, abortable=False):
|
||||||
software_version = str(payload['software_version'])
|
software_version = str(payload['software_version'])
|
||||||
LOG.info("Preparing remote install of %s, version: %s",
|
LOG.info("Preparing remote install of %s, version: %s",
|
||||||
subcloud.name, software_version)
|
subcloud.name, software_version)
|
||||||
|
@ -1754,7 +1831,7 @@ class SubcloudManager(manager.Manager):
|
||||||
software_version=software_version)
|
software_version=software_version)
|
||||||
try:
|
try:
|
||||||
install = SubcloudInstall(context, subcloud.name)
|
install = SubcloudInstall(context, subcloud.name)
|
||||||
install.prep(consts.ANSIBLE_OVERRIDES_PATH, payload)
|
install.prep(dccommon_consts.ANSIBLE_OVERRIDES_PATH, payload)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
LOG.exception(e)
|
LOG.exception(e)
|
||||||
db_api.subcloud_update(
|
db_api.subcloud_update(
|
||||||
|
@ -1771,7 +1848,8 @@ class SubcloudManager(manager.Manager):
|
||||||
deploy_status=consts.DEPLOY_STATE_INSTALLING,
|
deploy_status=consts.DEPLOY_STATE_INSTALLING,
|
||||||
error_description=consts.ERROR_DESC_EMPTY)
|
error_description=consts.ERROR_DESC_EMPTY)
|
||||||
try:
|
try:
|
||||||
install.install(consts.DC_ANSIBLE_LOG_DIR, install_command)
|
aborted = install.install(
|
||||||
|
consts.DC_ANSIBLE_LOG_DIR, install_command, abortable=abortable)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
msg = utils.find_ansible_error_msg(
|
msg = utils.find_ansible_error_msg(
|
||||||
subcloud.name, log_file, consts.DEPLOY_STATE_INSTALLING)
|
subcloud.name, log_file, consts.DEPLOY_STATE_INSTALLING)
|
||||||
|
@ -1784,6 +1862,8 @@ class SubcloudManager(manager.Manager):
|
||||||
install.cleanup(software_version)
|
install.cleanup(software_version)
|
||||||
return False
|
return False
|
||||||
install.cleanup(software_version)
|
install.cleanup(software_version)
|
||||||
|
if aborted:
|
||||||
|
return False
|
||||||
LOG.info("Successfully installed %s" % subcloud.name)
|
LOG.info("Successfully installed %s" % subcloud.name)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
@ -1798,7 +1878,9 @@ class SubcloudManager(manager.Manager):
|
||||||
# Run the ansible subcloud boostrap playbook
|
# Run the ansible subcloud boostrap playbook
|
||||||
LOG.info("Starting bootstrap of %s" % subcloud.name)
|
LOG.info("Starting bootstrap of %s" % subcloud.name)
|
||||||
try:
|
try:
|
||||||
run_playbook(log_file, apply_command)
|
run_ansible = RunAnsible()
|
||||||
|
aborted = run_ansible.exec_playbook(
|
||||||
|
log_file, apply_command, subcloud.name)
|
||||||
except PlaybookExecutionFailed:
|
except PlaybookExecutionFailed:
|
||||||
msg = utils.find_ansible_error_msg(
|
msg = utils.find_ansible_error_msg(
|
||||||
subcloud.name, log_file, consts.DEPLOY_STATE_BOOTSTRAPPING)
|
subcloud.name, log_file, consts.DEPLOY_STATE_BOOTSTRAPPING)
|
||||||
|
@ -1807,7 +1889,10 @@ class SubcloudManager(manager.Manager):
|
||||||
context, subcloud.id,
|
context, subcloud.id,
|
||||||
deploy_status=consts.DEPLOY_STATE_BOOTSTRAP_FAILED,
|
deploy_status=consts.DEPLOY_STATE_BOOTSTRAP_FAILED,
|
||||||
error_description=msg[0:consts.ERROR_DESCRIPTION_LENGTH])
|
error_description=msg[0:consts.ERROR_DESCRIPTION_LENGTH])
|
||||||
return
|
return False
|
||||||
|
|
||||||
|
if aborted:
|
||||||
|
return False
|
||||||
|
|
||||||
db_api.subcloud_update(
|
db_api.subcloud_update(
|
||||||
context, subcloud.id,
|
context, subcloud.id,
|
||||||
|
@ -1815,32 +1900,7 @@ class SubcloudManager(manager.Manager):
|
||||||
error_description=consts.ERROR_DESC_EMPTY)
|
error_description=consts.ERROR_DESC_EMPTY)
|
||||||
|
|
||||||
LOG.info("Successfully bootstrapped %s" % subcloud.name)
|
LOG.info("Successfully bootstrapped %s" % subcloud.name)
|
||||||
|
return True
|
||||||
def _run_subcloud_config(self, subcloud, context,
|
|
||||||
deploy_command, log_file):
|
|
||||||
# Run the custom deploy playbook
|
|
||||||
LOG.info("Starting deploy of %s" % subcloud.name)
|
|
||||||
db_api.subcloud_update(
|
|
||||||
context, subcloud.id,
|
|
||||||
deploy_status=consts.DEPLOY_STATE_CONFIGURING,
|
|
||||||
error_description=consts.ERROR_DESC_EMPTY)
|
|
||||||
|
|
||||||
try:
|
|
||||||
run_playbook(log_file, deploy_command)
|
|
||||||
except PlaybookExecutionFailed:
|
|
||||||
msg = utils.find_ansible_error_msg(
|
|
||||||
subcloud.name, log_file, consts.DEPLOY_STATE_CONFIGURING)
|
|
||||||
LOG.error(msg)
|
|
||||||
db_api.subcloud_update(
|
|
||||||
context, subcloud.id,
|
|
||||||
deploy_status=consts.DEPLOY_STATE_CONFIG_FAILED,
|
|
||||||
error_description=msg[0:consts.ERROR_DESCRIPTION_LENGTH])
|
|
||||||
return
|
|
||||||
LOG.info("Successfully deployed %s" % subcloud.name)
|
|
||||||
db_api.subcloud_update(
|
|
||||||
context, subcloud.id,
|
|
||||||
deploy_status=consts.DEPLOY_STATE_DONE,
|
|
||||||
error_description=consts.ERROR_DESC_EMPTY)
|
|
||||||
|
|
||||||
def _create_addn_hosts_dc(self, context):
|
def _create_addn_hosts_dc(self, context):
|
||||||
"""Generate the addn_hosts_dc file for hostname/ip translation"""
|
"""Generate the addn_hosts_dc file for hostname/ip translation"""
|
||||||
|
@ -1868,7 +1928,7 @@ class SubcloudManager(manager.Manager):
|
||||||
def _write_subcloud_ansible_config(self, cached_regionone_data, payload):
|
def _write_subcloud_ansible_config(self, cached_regionone_data, payload):
|
||||||
"""Create the override file for usage with the specified subcloud"""
|
"""Create the override file for usage with the specified subcloud"""
|
||||||
|
|
||||||
overrides_file = os.path.join(consts.ANSIBLE_OVERRIDES_PATH,
|
overrides_file = os.path.join(dccommon_consts.ANSIBLE_OVERRIDES_PATH,
|
||||||
payload['name'] + '.yml')
|
payload['name'] + '.yml')
|
||||||
|
|
||||||
mgmt_pool = cached_regionone_data['mgmt_pool']
|
mgmt_pool = cached_regionone_data['mgmt_pool']
|
||||||
|
@ -1901,7 +1961,7 @@ class SubcloudManager(manager.Manager):
|
||||||
"""Create the deploy value files for the subcloud"""
|
"""Create the deploy value files for the subcloud"""
|
||||||
|
|
||||||
deploy_values_file = os.path.join(
|
deploy_values_file = os.path.join(
|
||||||
consts.ANSIBLE_OVERRIDES_PATH, subcloud_name +
|
dccommon_consts.ANSIBLE_OVERRIDES_PATH, subcloud_name +
|
||||||
'_deploy_values.yml')
|
'_deploy_values.yml')
|
||||||
|
|
||||||
with open(deploy_values_file, 'w') as f_out_deploy_values_file:
|
with open(deploy_values_file, 'w') as f_out_deploy_values_file:
|
||||||
|
@ -2362,7 +2422,7 @@ class SubcloudManager(manager.Manager):
|
||||||
def _create_subcloud_update_overrides_file(
|
def _create_subcloud_update_overrides_file(
|
||||||
self, payload, subcloud_name, filename_suffix):
|
self, payload, subcloud_name, filename_suffix):
|
||||||
update_overrides_file = os.path.join(
|
update_overrides_file = os.path.join(
|
||||||
consts.ANSIBLE_OVERRIDES_PATH, subcloud_name + '_' +
|
dccommon_consts.ANSIBLE_OVERRIDES_PATH, subcloud_name + '_' +
|
||||||
filename_suffix + '.yml')
|
filename_suffix + '.yml')
|
||||||
|
|
||||||
self._update_override_values(payload)
|
self._update_override_values(payload)
|
||||||
|
|
|
@ -6,6 +6,7 @@
|
||||||
import os
|
import os
|
||||||
import time
|
import time
|
||||||
|
|
||||||
|
from dccommon import consts as dccommon_consts
|
||||||
from dccommon.exceptions import PlaybookExecutionFailed
|
from dccommon.exceptions import PlaybookExecutionFailed
|
||||||
from dccommon.utils import run_playbook
|
from dccommon.utils import run_playbook
|
||||||
from dcmanager.common import consts
|
from dcmanager.common import consts
|
||||||
|
@ -139,7 +140,7 @@ class MigratingDataState(BaseState):
|
||||||
deploy_status=consts.DEPLOY_STATE_MIGRATING_DATA)
|
deploy_status=consts.DEPLOY_STATE_MIGRATING_DATA)
|
||||||
|
|
||||||
ansible_subcloud_inventory_file = os.path.join(
|
ansible_subcloud_inventory_file = os.path.join(
|
||||||
consts.ANSIBLE_OVERRIDES_PATH,
|
dccommon_consts.ANSIBLE_OVERRIDES_PATH,
|
||||||
strategy_step.subcloud.name + consts.INVENTORY_FILE_POSTFIX)
|
strategy_step.subcloud.name + consts.INVENTORY_FILE_POSTFIX)
|
||||||
log_file = os.path.join(consts.DC_ANSIBLE_LOG_DIR, subcloud.name) + \
|
log_file = os.path.join(consts.DC_ANSIBLE_LOG_DIR, subcloud.name) + \
|
||||||
'_playbook_output.log'
|
'_playbook_output.log'
|
||||||
|
|
|
@ -336,7 +336,7 @@ class UpgradingSimplexState(BaseState):
|
||||||
try:
|
try:
|
||||||
install = SubcloudInstall(
|
install = SubcloudInstall(
|
||||||
self.context, strategy_step.subcloud.name)
|
self.context, strategy_step.subcloud.name)
|
||||||
install.prep(consts.ANSIBLE_OVERRIDES_PATH,
|
install.prep(dccommon_consts.ANSIBLE_OVERRIDES_PATH,
|
||||||
install_values)
|
install_values)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
db_api.subcloud_update(
|
db_api.subcloud_update(
|
||||||
|
@ -349,7 +349,7 @@ class UpgradingSimplexState(BaseState):
|
||||||
raise
|
raise
|
||||||
|
|
||||||
ansible_subcloud_inventory_file = os.path.join(
|
ansible_subcloud_inventory_file = os.path.join(
|
||||||
consts.ANSIBLE_OVERRIDES_PATH,
|
dccommon_consts.ANSIBLE_OVERRIDES_PATH,
|
||||||
strategy_step.subcloud.name + consts.INVENTORY_FILE_POSTFIX)
|
strategy_step.subcloud.name + consts.INVENTORY_FILE_POSTFIX)
|
||||||
|
|
||||||
# Create the ansible inventory for the upgrade subcloud
|
# Create the ansible inventory for the upgrade subcloud
|
||||||
|
@ -360,7 +360,7 @@ class UpgradingSimplexState(BaseState):
|
||||||
install_command = [
|
install_command = [
|
||||||
"ansible-playbook", dccommon_consts.ANSIBLE_SUBCLOUD_INSTALL_PLAYBOOK,
|
"ansible-playbook", dccommon_consts.ANSIBLE_SUBCLOUD_INSTALL_PLAYBOOK,
|
||||||
"-i", ansible_subcloud_inventory_file,
|
"-i", ansible_subcloud_inventory_file,
|
||||||
"-e", "@%s" % consts.ANSIBLE_OVERRIDES_PATH + "/" +
|
"-e", "@%s" % dccommon_consts.ANSIBLE_OVERRIDES_PATH + "/" +
|
||||||
strategy_step.subcloud.name + '/' + "install_values.yml"
|
strategy_step.subcloud.name + '/' + "install_values.yml"
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
|
@ -203,10 +203,15 @@ class ManagerClient(RPCClient):
|
||||||
payload=payload))
|
payload=payload))
|
||||||
|
|
||||||
def subcloud_deploy_config(self, ctxt, subcloud_id, payload):
|
def subcloud_deploy_config(self, ctxt, subcloud_id, payload):
|
||||||
return self.call(ctxt, self.make_msg('subcloud_deploy_config',
|
return self.cast(ctxt, self.make_msg('subcloud_deploy_config',
|
||||||
subcloud_id=subcloud_id,
|
subcloud_id=subcloud_id,
|
||||||
payload=payload))
|
payload=payload))
|
||||||
|
|
||||||
|
def subcloud_deploy_abort(self, ctxt, subcloud_id, deploy_status):
|
||||||
|
return self.cast(ctxt, self.make_msg('subcloud_deploy_abort',
|
||||||
|
subcloud_id=subcloud_id,
|
||||||
|
deploy_status=deploy_status))
|
||||||
|
|
||||||
|
|
||||||
class DCManagerNotifications(RPCClient):
|
class DCManagerNotifications(RPCClient):
|
||||||
"""DC Manager Notification interface to broadcast subcloud state changed
|
"""DC Manager Notification interface to broadcast subcloud state changed
|
||||||
|
|
|
@ -463,3 +463,40 @@ class TestSubcloudDeployInstall(testroot.DCManagerApiTest):
|
||||||
self.assertEqual(consts.DEPLOY_STATE_PRE_INSTALL,
|
self.assertEqual(consts.DEPLOY_STATE_PRE_INSTALL,
|
||||||
response.json['deploy-status'])
|
response.json['deploy-status'])
|
||||||
self.assertEqual(SW_VERSION, response.json['software-version'])
|
self.assertEqual(SW_VERSION, response.json['software-version'])
|
||||||
|
|
||||||
|
|
||||||
|
class TestSubcloudDeployAbort(testroot.DCManagerApiTest):
|
||||||
|
def setUp(self):
|
||||||
|
super(TestSubcloudDeployAbort, self).setUp()
|
||||||
|
self.ctx = utils.dummy_context()
|
||||||
|
|
||||||
|
p = mock.patch.object(rpc_client, 'ManagerClient')
|
||||||
|
self.mock_rpc_client = p.start()
|
||||||
|
self.addCleanup(p.stop)
|
||||||
|
|
||||||
|
def test_abort_subcloud(self):
|
||||||
|
subcloud = fake_subcloud.create_fake_subcloud(
|
||||||
|
self.ctx,
|
||||||
|
deploy_status=consts.DEPLOY_STATE_INSTALLING)
|
||||||
|
|
||||||
|
self.mock_rpc_client().subcloud_deploy_abort.return_value = True
|
||||||
|
|
||||||
|
response = self.app.patch_json(FAKE_URL + '/' + str(subcloud.id) +
|
||||||
|
'/abort',
|
||||||
|
headers=FAKE_HEADERS)
|
||||||
|
self.mock_rpc_client().subcloud_deploy_abort.assert_called_once_with(
|
||||||
|
mock.ANY,
|
||||||
|
subcloud.id,
|
||||||
|
subcloud.deploy_status)
|
||||||
|
self.assertEqual(response.status_int, 200)
|
||||||
|
|
||||||
|
def test_abort_subcloud_invalid_deploy_status(self):
|
||||||
|
subcloud = fake_subcloud.create_fake_subcloud(
|
||||||
|
self.ctx,
|
||||||
|
deploy_status=consts.DEPLOY_STATE_INSTALLED)
|
||||||
|
self.mock_rpc_client().subcloud_deploy_config.return_value = True
|
||||||
|
|
||||||
|
six.assertRaisesRegex(self, webtest.app.AppError, "400 *",
|
||||||
|
self.app.patch_json, FAKE_URL + '/' +
|
||||||
|
str(subcloud.id) + '/abort',
|
||||||
|
headers=FAKE_HEADERS)
|
||||||
|
|
|
@ -19,6 +19,7 @@ import mock
|
||||||
from six.moves import http_client
|
from six.moves import http_client
|
||||||
import webtest
|
import webtest
|
||||||
|
|
||||||
|
from dccommon import consts as dccommon_consts
|
||||||
from dcmanager.api.controllers.v1 import subcloud_deploy
|
from dcmanager.api.controllers.v1 import subcloud_deploy
|
||||||
from dcmanager.common import consts
|
from dcmanager.common import consts
|
||||||
from dcmanager.common import phased_subcloud_deploy as psd_common
|
from dcmanager.common import phased_subcloud_deploy as psd_common
|
||||||
|
@ -254,8 +255,8 @@ class TestSubcloudDeploy(testroot.DCManagerApiTest):
|
||||||
deploy_config = psd_common.get_config_file_path("subcloud1",
|
deploy_config = psd_common.get_config_file_path("subcloud1",
|
||||||
consts.DEPLOY_CONFIG)
|
consts.DEPLOY_CONFIG)
|
||||||
self.assertEqual(bootstrap_file,
|
self.assertEqual(bootstrap_file,
|
||||||
f'{consts.ANSIBLE_OVERRIDES_PATH}/subcloud1.yml')
|
f'{dccommon_consts.ANSIBLE_OVERRIDES_PATH}/subcloud1.yml')
|
||||||
self.assertEqual(install_values,
|
self.assertEqual(install_values,
|
||||||
f'{consts.ANSIBLE_OVERRIDES_PATH}/subcloud1/install_values.yml')
|
f'{dccommon_consts.ANSIBLE_OVERRIDES_PATH}/subcloud1/install_values.yml')
|
||||||
self.assertEqual(deploy_config,
|
self.assertEqual(deploy_config,
|
||||||
f'{consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_deploy_config.yml')
|
f'{dccommon_consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_deploy_config.yml')
|
||||||
|
|
|
@ -1591,11 +1591,11 @@ class TestSubcloudAPIOther(testroot.DCManagerApiTest):
|
||||||
install_values = sc._get_config_file_path("subcloud1", "install_values")
|
install_values = sc._get_config_file_path("subcloud1", "install_values")
|
||||||
deploy_config = sc._get_config_file_path("subcloud1", consts.DEPLOY_CONFIG)
|
deploy_config = sc._get_config_file_path("subcloud1", consts.DEPLOY_CONFIG)
|
||||||
self.assertEqual(bootstrap_file,
|
self.assertEqual(bootstrap_file,
|
||||||
f'{consts.ANSIBLE_OVERRIDES_PATH}/subcloud1.yml')
|
f'{dccommon_consts.ANSIBLE_OVERRIDES_PATH}/subcloud1.yml')
|
||||||
self.assertEqual(install_values,
|
self.assertEqual(install_values,
|
||||||
f'{consts.ANSIBLE_OVERRIDES_PATH}/subcloud1/install_values.yml')
|
f'{dccommon_consts.ANSIBLE_OVERRIDES_PATH}/subcloud1/install_values.yml')
|
||||||
self.assertEqual(deploy_config,
|
self.assertEqual(deploy_config,
|
||||||
f'{consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_deploy_config.yml')
|
f'{dccommon_consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_deploy_config.yml')
|
||||||
|
|
||||||
@mock.patch.object(rpc_client, 'ManagerClient')
|
@mock.patch.object(rpc_client, 'ManagerClient')
|
||||||
def test_format_ip_address(self, mock_rpc_client):
|
def test_format_ip_address(self, mock_rpc_client):
|
||||||
|
|
|
@ -28,6 +28,7 @@ sys.modules['fm_core'] = mock.Mock()
|
||||||
import threading
|
import threading
|
||||||
|
|
||||||
from dccommon import consts as dccommon_consts
|
from dccommon import consts as dccommon_consts
|
||||||
|
from dccommon.utils import RunAnsible
|
||||||
from dcmanager.common import consts
|
from dcmanager.common import consts
|
||||||
from dcmanager.common import exceptions
|
from dcmanager.common import exceptions
|
||||||
from dcmanager.common import prestage
|
from dcmanager.common import prestage
|
||||||
|
@ -427,9 +428,7 @@ class TestSubcloudManager(base.DCManagerTestCase):
|
||||||
|
|
||||||
@mock.patch.object(
|
@mock.patch.object(
|
||||||
subcloud_manager.SubcloudManager, 'compose_install_command')
|
subcloud_manager.SubcloudManager, 'compose_install_command')
|
||||||
@mock.patch.object(threading.Thread, 'start')
|
|
||||||
def test_deploy_install_subcloud(self,
|
def test_deploy_install_subcloud(self,
|
||||||
mock_thread_start,
|
|
||||||
mock_compose_install_command):
|
mock_compose_install_command):
|
||||||
|
|
||||||
subcloud_name = 'subcloud1'
|
subcloud_name = 'subcloud1'
|
||||||
|
@ -453,7 +452,6 @@ class TestSubcloudManager(base.DCManagerTestCase):
|
||||||
subcloud_name,
|
subcloud_name,
|
||||||
sm._get_ansible_filename(subcloud_name, consts.INVENTORY_FILE_POSTFIX),
|
sm._get_ansible_filename(subcloud_name, consts.INVENTORY_FILE_POSTFIX),
|
||||||
FAKE_PREVIOUS_SW_VERSION)
|
FAKE_PREVIOUS_SW_VERSION)
|
||||||
mock_thread_start.assert_called_once()
|
|
||||||
|
|
||||||
@mock.patch.object(subcloud_manager.SubcloudManager,
|
@mock.patch.object(subcloud_manager.SubcloudManager,
|
||||||
'_create_intermediate_ca_cert')
|
'_create_intermediate_ca_cert')
|
||||||
|
@ -538,12 +536,13 @@ class TestSubcloudManager(base.DCManagerTestCase):
|
||||||
@mock.patch.object(subcloud_manager, 'keyring')
|
@mock.patch.object(subcloud_manager, 'keyring')
|
||||||
@mock.patch.object(cutils, 'get_playbook_for_software_version')
|
@mock.patch.object(cutils, 'get_playbook_for_software_version')
|
||||||
@mock.patch.object(cutils, 'update_values_on_yaml_file')
|
@mock.patch.object(cutils, 'update_values_on_yaml_file')
|
||||||
@mock.patch.object(subcloud_manager, 'run_playbook')
|
@mock.patch.object(RunAnsible, 'exec_playbook')
|
||||||
def test_subcloud_deploy_bootstrap(self, mock_run_playbook, mock_update_yml,
|
def test_subcloud_deploy_bootstrap(self, mock_exec_playbook, mock_update_yml,
|
||||||
mock_get_playbook_for_software_version,
|
mock_get_playbook_for_software_version,
|
||||||
mock_keyring, create_subcloud_inventory):
|
mock_keyring, create_subcloud_inventory):
|
||||||
mock_get_playbook_for_software_version.return_value = "22.12"
|
mock_get_playbook_for_software_version.return_value = "22.12"
|
||||||
mock_keyring.get_password.return_value = "testpass"
|
mock_keyring.get_password.return_value = "testpass"
|
||||||
|
mock_exec_playbook.return_value = False
|
||||||
|
|
||||||
subcloud = fake_subcloud.create_fake_subcloud(
|
subcloud = fake_subcloud.create_fake_subcloud(
|
||||||
self.ctx,
|
self.ctx,
|
||||||
|
@ -557,7 +556,7 @@ class TestSubcloudManager(base.DCManagerTestCase):
|
||||||
sm = subcloud_manager.SubcloudManager()
|
sm = subcloud_manager.SubcloudManager()
|
||||||
sm.subcloud_deploy_bootstrap(self.ctx, subcloud.id, payload)
|
sm.subcloud_deploy_bootstrap(self.ctx, subcloud.id, payload)
|
||||||
|
|
||||||
mock_run_playbook.assert_called_once()
|
mock_exec_playbook.assert_called_once()
|
||||||
|
|
||||||
# Verify subcloud was updated with correct values
|
# Verify subcloud was updated with correct values
|
||||||
updated_subcloud = db_api.subcloud_get_by_name(self.ctx,
|
updated_subcloud = db_api.subcloud_get_by_name(self.ctx,
|
||||||
|
@ -589,10 +588,7 @@ class TestSubcloudManager(base.DCManagerTestCase):
|
||||||
|
|
||||||
@mock.patch.object(subcloud_manager.SubcloudManager,
|
@mock.patch.object(subcloud_manager.SubcloudManager,
|
||||||
'_prepare_for_deployment')
|
'_prepare_for_deployment')
|
||||||
@mock.patch.object(threading.Thread,
|
def test_configure_subcloud(self, mock_prepare_for_deployment):
|
||||||
'start')
|
|
||||||
def test_configure_subcloud(self, mock_thread_start,
|
|
||||||
mock_prepare_for_deployment):
|
|
||||||
subcloud = self.create_subcloud_static(
|
subcloud = self.create_subcloud_static(
|
||||||
self.ctx,
|
self.ctx,
|
||||||
name='subcloud1',
|
name='subcloud1',
|
||||||
|
@ -607,7 +603,6 @@ class TestSubcloudManager(base.DCManagerTestCase):
|
||||||
sm.subcloud_deploy_config(self.ctx,
|
sm.subcloud_deploy_config(self.ctx,
|
||||||
subcloud.id,
|
subcloud.id,
|
||||||
payload=fake_payload)
|
payload=fake_payload)
|
||||||
mock_thread_start.assert_called_once()
|
|
||||||
mock_prepare_for_deployment.assert_called_once()
|
mock_prepare_for_deployment.assert_called_once()
|
||||||
|
|
||||||
@mock.patch.object(subcloud_manager.SubcloudManager,
|
@mock.patch.object(subcloud_manager.SubcloudManager,
|
||||||
|
@ -1581,22 +1576,22 @@ class TestSubcloudManager(base.DCManagerTestCase):
|
||||||
filename = sm._get_ansible_filename('subcloud1',
|
filename = sm._get_ansible_filename('subcloud1',
|
||||||
consts.INVENTORY_FILE_POSTFIX)
|
consts.INVENTORY_FILE_POSTFIX)
|
||||||
self.assertEqual(filename,
|
self.assertEqual(filename,
|
||||||
f'{consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_inventory.yml')
|
f'{dccommon_consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_inventory.yml')
|
||||||
|
|
||||||
def test_compose_install_command(self):
|
def test_compose_install_command(self):
|
||||||
sm = subcloud_manager.SubcloudManager()
|
sm = subcloud_manager.SubcloudManager()
|
||||||
install_command = sm.compose_install_command(
|
install_command = sm.compose_install_command(
|
||||||
'subcloud1',
|
'subcloud1',
|
||||||
f'{consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_inventory.yml',
|
f'{dccommon_consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_inventory.yml',
|
||||||
FAKE_PREVIOUS_SW_VERSION)
|
FAKE_PREVIOUS_SW_VERSION)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
install_command,
|
install_command,
|
||||||
[
|
[
|
||||||
'ansible-playbook',
|
'ansible-playbook',
|
||||||
dccommon_consts.ANSIBLE_SUBCLOUD_INSTALL_PLAYBOOK,
|
dccommon_consts.ANSIBLE_SUBCLOUD_INSTALL_PLAYBOOK,
|
||||||
'-i', f'{consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_inventory.yml',
|
'-i', f'{dccommon_consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_inventory.yml',
|
||||||
'--limit', 'subcloud1',
|
'--limit', 'subcloud1',
|
||||||
'-e', f"@{consts.ANSIBLE_OVERRIDES_PATH}/subcloud1/install_values.yml",
|
'-e', f"@{dccommon_consts.ANSIBLE_OVERRIDES_PATH}/subcloud1/install_values.yml",
|
||||||
'-e', "install_release_version=%s" % FAKE_PREVIOUS_SW_VERSION
|
'-e', "install_release_version=%s" % FAKE_PREVIOUS_SW_VERSION
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
@ -1607,7 +1602,7 @@ class TestSubcloudManager(base.DCManagerTestCase):
|
||||||
sm = subcloud_manager.SubcloudManager()
|
sm = subcloud_manager.SubcloudManager()
|
||||||
apply_command = sm.compose_apply_command(
|
apply_command = sm.compose_apply_command(
|
||||||
'subcloud1',
|
'subcloud1',
|
||||||
f'{consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_inventory.yml',
|
f'{dccommon_consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_inventory.yml',
|
||||||
FAKE_PREVIOUS_SW_VERSION)
|
FAKE_PREVIOUS_SW_VERSION)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
apply_command,
|
apply_command,
|
||||||
|
@ -1616,9 +1611,9 @@ class TestSubcloudManager(base.DCManagerTestCase):
|
||||||
cutils.get_playbook_for_software_version(
|
cutils.get_playbook_for_software_version(
|
||||||
subcloud_manager.ANSIBLE_SUBCLOUD_PLAYBOOK,
|
subcloud_manager.ANSIBLE_SUBCLOUD_PLAYBOOK,
|
||||||
FAKE_PREVIOUS_SW_VERSION),
|
FAKE_PREVIOUS_SW_VERSION),
|
||||||
'-i', f'{consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_inventory.yml',
|
'-i', f'{dccommon_consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_inventory.yml',
|
||||||
'--limit', 'subcloud1', '-e',
|
'--limit', 'subcloud1', '-e',
|
||||||
f"override_files_dir='{consts.ANSIBLE_OVERRIDES_PATH}' region_name=subcloud1",
|
f"override_files_dir='{dccommon_consts.ANSIBLE_OVERRIDES_PATH}' region_name=subcloud1",
|
||||||
'-e', "install_release_version=%s" % FAKE_PREVIOUS_SW_VERSION
|
'-e', "install_release_version=%s" % FAKE_PREVIOUS_SW_VERSION
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
@ -1632,14 +1627,14 @@ class TestSubcloudManager(base.DCManagerTestCase):
|
||||||
"deploy_config": "subcloud1.yaml"}
|
"deploy_config": "subcloud1.yaml"}
|
||||||
deploy_command = sm.compose_deploy_command(
|
deploy_command = sm.compose_deploy_command(
|
||||||
'subcloud1',
|
'subcloud1',
|
||||||
f'{consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_inventory.yml',
|
f'{dccommon_consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_inventory.yml',
|
||||||
fake_payload)
|
fake_payload)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
deploy_command,
|
deploy_command,
|
||||||
[
|
[
|
||||||
'ansible-playbook', 'test_playbook.yaml', '-e',
|
'ansible-playbook', 'test_playbook.yaml', '-e',
|
||||||
f'@{consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_deploy_values.yml', '-i',
|
f'@{dccommon_consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_deploy_values.yml', '-i',
|
||||||
f'{consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_inventory.yml',
|
f'{dccommon_consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_inventory.yml',
|
||||||
'--limit', 'subcloud1'
|
'--limit', 'subcloud1'
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
@ -1650,7 +1645,7 @@ class TestSubcloudManager(base.DCManagerTestCase):
|
||||||
sm = subcloud_manager.SubcloudManager()
|
sm = subcloud_manager.SubcloudManager()
|
||||||
rehome_command = sm.compose_rehome_command(
|
rehome_command = sm.compose_rehome_command(
|
||||||
'subcloud1',
|
'subcloud1',
|
||||||
f'{consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_inventory.yml',
|
f'{dccommon_consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_inventory.yml',
|
||||||
FAKE_PREVIOUS_SW_VERSION)
|
FAKE_PREVIOUS_SW_VERSION)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
rehome_command,
|
rehome_command,
|
||||||
|
@ -1659,11 +1654,11 @@ class TestSubcloudManager(base.DCManagerTestCase):
|
||||||
cutils.get_playbook_for_software_version(
|
cutils.get_playbook_for_software_version(
|
||||||
subcloud_manager.ANSIBLE_SUBCLOUD_REHOME_PLAYBOOK,
|
subcloud_manager.ANSIBLE_SUBCLOUD_REHOME_PLAYBOOK,
|
||||||
FAKE_PREVIOUS_SW_VERSION),
|
FAKE_PREVIOUS_SW_VERSION),
|
||||||
'-i', f'{consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_inventory.yml',
|
'-i', f'{dccommon_consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_inventory.yml',
|
||||||
'--limit', 'subcloud1',
|
'--limit', 'subcloud1',
|
||||||
'--timeout', subcloud_manager.REHOME_PLAYBOOK_TIMEOUT,
|
'--timeout', subcloud_manager.REHOME_PLAYBOOK_TIMEOUT,
|
||||||
'-e',
|
'-e',
|
||||||
f"override_files_dir='{consts.ANSIBLE_OVERRIDES_PATH}' region_name=subcloud1"
|
f"override_files_dir='{dccommon_consts.ANSIBLE_OVERRIDES_PATH}' region_name=subcloud1"
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -2102,7 +2097,7 @@ class TestSubcloudManager(base.DCManagerTestCase):
|
||||||
values = copy.copy(FAKE_BACKUP_CREATE_LOAD_1)
|
values = copy.copy(FAKE_BACKUP_CREATE_LOAD_1)
|
||||||
|
|
||||||
override_file = os_path.join(
|
override_file = os_path.join(
|
||||||
consts.ANSIBLE_OVERRIDES_PATH, subcloud.name + "_backup_create_values.yml"
|
dccommon_consts.ANSIBLE_OVERRIDES_PATH, subcloud.name + "_backup_create_values.yml"
|
||||||
)
|
)
|
||||||
mock_create_backup_file.return_value = override_file
|
mock_create_backup_file.return_value = override_file
|
||||||
|
|
||||||
|
@ -2175,7 +2170,7 @@ class TestSubcloudManager(base.DCManagerTestCase):
|
||||||
RELEASE_VERSION = '22.12'
|
RELEASE_VERSION = '22.12'
|
||||||
|
|
||||||
override_file = os_path.join(
|
override_file = os_path.join(
|
||||||
consts.ANSIBLE_OVERRIDES_PATH, subcloud.name + "_backup_delete_values.yml"
|
dccommon_consts.ANSIBLE_OVERRIDES_PATH, subcloud.name + "_backup_delete_values.yml"
|
||||||
)
|
)
|
||||||
mock_create_subcloud_inventory_file.return_value = override_file
|
mock_create_subcloud_inventory_file.return_value = override_file
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue