diff --git a/api-ref/source/api-ref-dcmanager-v1.rst b/api-ref/source/api-ref-dcmanager-v1.rst index 4a6ae7bf1..5842e1e9c 100644 --- a/api-ref/source/api-ref-dcmanager-v1.rst +++ b/api-ref/source/api-ref-dcmanager-v1.rst @@ -1812,6 +1812,73 @@ Request Example :language: json +**Response parameters** + +.. rest_parameters:: parameters.yaml + + - id: subcloud_id + - name: subcloud_name + - description: subcloud_description + - location: subcloud_location + - software-version: software_version + - management-state: management_state + - availability-status: availability_status + - deploy-status: deploy_status + - backup-status: backup_status + - backup-datetime: backup_datetime + - error-description: error_description + - management-subnet: management_subnet + - management-start-ip: management_start_ip + - management-end-ip: management_end_ip + - management-gateway-ip: management_gateway_ip + - openstack-installed: openstack_installed + - systemcontroller-gateway-ip: systemcontroller_gateway_ip + - data_install: data_install + - data_upgrade: data_upgrade + - created-at: created_at + - updated-at: updated_at + - group_id: group_id + +Response Example +---------------- + +.. literalinclude:: samples/phased-subcloud-deploy/phased-subcloud-deploy-post-response.json + :language: json + +********************* +Bootstraps a subcloud +********************* + +.. rest_method:: PATCH /v1.0/phased-subcloud-deploy/bootstrap + +Accepts Content-Type multipart/form-data. + + +**Normal response codes** + +200 + +**Error response codes** + +badRequest (400), unauthorized (401), forbidden (403), badMethod (405), +conflict (409), HTTPUnprocessableEntity (422), internalServerError (500), +serviceUnavailable (503) + +**Request parameters** + +.. rest_parameters:: parameters.yaml + + - bootstrap-address: bootstrap_address + - bootstrap_values: bootstrap_values + - sysadmin_password: sysadmin_password + +Request Example +---------------- + +.. literalinclude:: samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-bootstrap-request.json + :language: json + + **Response parameters** .. rest_parameters:: parameters.yaml diff --git a/api-ref/source/samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-bootstrap-request.json b/api-ref/source/samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-bootstrap-request.json new file mode 100644 index 000000000..b1a53acbb --- /dev/null +++ b/api-ref/source/samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-bootstrap-request.json @@ -0,0 +1,5 @@ +{ + "bootstrap-address": "10.10.10.12", + "bootstrap_values": "content of bootstrap_values file", + "sysadmin_password": "XXXXXXX" +} \ No newline at end of file diff --git a/distributedcloud/dcmanager/api/controllers/v1/phased_subcloud_deploy.py b/distributedcloud/dcmanager/api/controllers/v1/phased_subcloud_deploy.py index 660ca7415..f509653e8 100644 --- a/distributedcloud/dcmanager/api/controllers/v1/phased_subcloud_deploy.py +++ b/distributedcloud/dcmanager/api/controllers/v1/phased_subcloud_deploy.py @@ -17,31 +17,45 @@ from dcmanager.api.controllers import restcomm from dcmanager.api.policies import phased_subcloud_deploy as \ phased_subcloud_deploy_policy from dcmanager.api import policy +from dcmanager.common import consts from dcmanager.common.context import RequestContext +from dcmanager.common import exceptions from dcmanager.common.i18n import _ from dcmanager.common import phased_subcloud_deploy as psd_common from dcmanager.common import utils +from dcmanager.db import api as db_api +from dcmanager.db.sqlalchemy import models from dcmanager.rpc import client as rpc_client LOG = logging.getLogger(__name__) LOCK_NAME = 'PhasedSubcloudDeployController' -BOOTSTRAP_ADDRESS = 'bootstrap-address' -BOOTSTRAP_VALUES = 'bootstrap_values' -INSTALL_VALUES = 'install_values' - SUBCLOUD_CREATE_REQUIRED_PARAMETERS = ( - BOOTSTRAP_VALUES, - BOOTSTRAP_ADDRESS + consts.BOOTSTRAP_VALUES, + consts.BOOTSTRAP_ADDRESS ) # The consts.DEPLOY_CONFIG is missing here because it's handled differently # by the upload_deploy_config_file() function SUBCLOUD_CREATE_GET_FILE_CONTENTS = ( - BOOTSTRAP_VALUES, - INSTALL_VALUES, + consts.BOOTSTRAP_VALUES, + consts.INSTALL_VALUES, ) +SUBCLOUD_BOOTSTRAP_GET_FILE_CONTENTS = ( + consts.BOOTSTRAP_VALUES, +) + +VALID_STATES_FOR_DEPLOY_BOOTSTRAP = [ + consts.DEPLOY_STATE_INSTALLED, + consts.DEPLOY_STATE_BOOTSTRAP_FAILED, + consts.DEPLOY_STATE_BOOTSTRAP_ABORTED, + consts.DEPLOY_STATE_BOOTSTRAPPED, + # The subcloud can be installed manually (without remote install) so we need + # to allow the bootstrap operation when the state == DEPLOY_STATE_CREATED + consts.DEPLOY_STATE_CREATED +] + def get_create_payload(request: pecan.Request) -> dict: payload = dict() @@ -51,7 +65,7 @@ def get_create_payload(request: pecan.Request) -> dict: file_item = request.POST[f] file_item.file.seek(0, os.SEEK_SET) data = yaml.safe_load(file_item.file.read().decode('utf8')) - if f == BOOTSTRAP_VALUES: + if f == consts.BOOTSTRAP_VALUES: payload.update(data) else: payload.update({f: data}) @@ -118,6 +132,73 @@ class PhasedSubcloudDeployController(object): pecan.abort(httpclient.INTERNAL_SERVER_ERROR, _('Unable to create subcloud')) + def _deploy_bootstrap(self, context: RequestContext, + request: pecan.Request, + subcloud: models.Subcloud): + if subcloud.deploy_status not in VALID_STATES_FOR_DEPLOY_BOOTSTRAP: + valid_states_str = ', '.join(VALID_STATES_FOR_DEPLOY_BOOTSTRAP) + pecan.abort(400, _('Subcloud deploy status must be either: %s') + % valid_states_str) + + has_bootstrap_values = consts.BOOTSTRAP_VALUES in request.POST + payload = {} + + # Try to load the existing override values + override_file = psd_common.get_config_file_path(subcloud.name) + if os.path.exists(override_file): + psd_common.populate_payload_with_pre_existing_data( + payload, subcloud, SUBCLOUD_BOOTSTRAP_GET_FILE_CONTENTS) + elif not has_bootstrap_values: + msg = _("Required bootstrap-values file was not provided and it was" + " not previously available at %s") % (override_file) + pecan.abort(400, msg) + + request_data = psd_common.get_request_data( + request, subcloud, SUBCLOUD_BOOTSTRAP_GET_FILE_CONTENTS) + + # Update the existing values with new ones from the request + payload.update(request_data) + + psd_common.validate_sysadmin_password(payload) + + if has_bootstrap_values: + # Need to validate the new values + playload_name = payload.get('name') + if playload_name != subcloud.name: + pecan.abort(400, _('The bootstrap-values "name" value (%s) ' + 'must match the current subcloud name (%s)' % + (playload_name, subcloud.name))) + + # Verify if payload contains all required bootstrap values + psd_common.validate_bootstrap_values(payload) + + # It's ok for the management subnet to conflict with itself since we + # are only going to update it if it was modified, conflicts with + # other subclouds are still verified. + psd_common.validate_subcloud_config(context, payload, + ignore_conflicts_with=subcloud) + psd_common.format_ip_address(payload) + + # Patch status and fresh_install_k8s_version may have been changed + # between deploy create and deploy bootstrap commands. Validate them + # again: + psd_common.validate_system_controller_patch_status("bootstrap") + psd_common.validate_k8s_version(payload) + + try: + # Ask dcmanager-manager to bootstrap the subcloud. + self.dcmanager_rpc_client.subcloud_deploy_bootstrap( + context, subcloud.id, payload) + return db_api.subcloud_db_model_to_dict(subcloud) + + except RemoteError as e: + pecan.abort(httpclient.UNPROCESSABLE_ENTITY, e.value) + except Exception: + LOG.exception("Unable to bootstrap subcloud %s" % + payload.get('name')) + pecan.abort(httpclient.INTERNAL_SERVER_ERROR, + _('Unable to bootstrap subcloud')) + @pecan.expose(generic=True, template='json') def index(self): # Route the request to specific methods with parameters @@ -128,3 +209,36 @@ class PhasedSubcloudDeployController(object): def post(self): context = restcomm.extract_context_from_environ() return self._deploy_create(context, pecan.request) + + @utils.synchronized(LOCK_NAME) + @index.when(method='PATCH', template='json') + def patch(self, subcloud_ref=None, verb=None): + """Modify the subcloud deployment. + + :param subcloud_ref: ID or name of subcloud to update + + :param verb: Specifies the patch action to be taken + or subcloud operation + """ + + policy.authorize(phased_subcloud_deploy_policy.POLICY_ROOT % "modify", {}, + restcomm.extract_credentials_for_policy()) + context = restcomm.extract_context_from_environ() + + if not subcloud_ref: + pecan.abort(400, _('Subcloud ID required')) + + try: + if subcloud_ref.isdigit(): + subcloud = db_api.subcloud_get(context, subcloud_ref) + else: + subcloud = db_api.subcloud_get_by_name(context, subcloud_ref) + except (exceptions.SubcloudNotFound, exceptions.SubcloudNameNotFound): + pecan.abort(404, _('Subcloud not found')) + + if verb == 'bootstrap': + subcloud = self._deploy_bootstrap(context, pecan.request, subcloud) + else: + pecan.abort(400, _('Invalid request')) + + return subcloud diff --git a/distributedcloud/dcmanager/api/policies/phased_subcloud_deploy.py b/distributedcloud/dcmanager/api/policies/phased_subcloud_deploy.py index b8dc452ec..1faea5a6e 100644 --- a/distributedcloud/dcmanager/api/policies/phased_subcloud_deploy.py +++ b/distributedcloud/dcmanager/api/policies/phased_subcloud_deploy.py @@ -22,6 +22,17 @@ phased_subcloud_deploy_rules = [ } ] ), + policy.DocumentedRuleDefault( + name=POLICY_ROOT % 'modify', + check_str='rule:' + base.ADMIN_IN_SYSTEM_PROJECTS, + description="Modify the subcloud deployment.", + operations=[ + { + 'method': 'PATCH', + 'path': '/v1.0/phased-subcloud-deploy/{subcloud}/bootstrap' + } + ] + ) ] diff --git a/distributedcloud/dcmanager/common/consts.py b/distributedcloud/dcmanager/common/consts.py index 6434db314..a6d5b34b4 100644 --- a/distributedcloud/dcmanager/common/consts.py +++ b/distributedcloud/dcmanager/common/consts.py @@ -26,6 +26,10 @@ CERTS_VAULT_DIR = "/opt/dc-vault/certs" LOADS_VAULT_DIR = "/opt/dc-vault/loads" PATCH_VAULT_DIR = "/opt/dc-vault/patches" +BOOTSTRAP_VALUES = 'bootstrap_values' +BOOTSTRAP_ADDRESS = 'bootstrap-address' +INSTALL_VALUES = 'install_values' + # Admin status for hosts ADMIN_LOCKED = 'locked' ADMIN_UNLOCKED = 'unlocked' @@ -168,9 +172,13 @@ DEPLOY_STATE_PRE_INSTALL = 'pre-install' DEPLOY_STATE_PRE_INSTALL_FAILED = 'pre-install-failed' DEPLOY_STATE_INSTALLING = 'installing' DEPLOY_STATE_INSTALL_FAILED = 'install-failed' -DEPLOY_STATE_INSTALLED = 'installed' +DEPLOY_STATE_INSTALLED = 'install-complete' +DEPLOY_STATE_PRE_BOOTSTRAP = 'pre-bootstrap' +DEPLOY_STATE_PRE_BOOTSTRAP_FAILED = 'pre-bootstrap-failed' DEPLOY_STATE_BOOTSTRAPPING = 'bootstrapping' DEPLOY_STATE_BOOTSTRAP_FAILED = 'bootstrap-failed' +DEPLOY_STATE_BOOTSTRAP_ABORTED = 'bootstrap-aborted' +DEPLOY_STATE_BOOTSTRAPPED = 'bootstrap-complete' DEPLOY_STATE_DEPLOYING = 'deploying' DEPLOY_STATE_DEPLOY_FAILED = 'deploy-failed' DEPLOY_STATE_MIGRATING_DATA = 'migrating-data' diff --git a/distributedcloud/dcmanager/common/phased_subcloud_deploy.py b/distributedcloud/dcmanager/common/phased_subcloud_deploy.py index e1306f434..ae989365d 100644 --- a/distributedcloud/dcmanager/common/phased_subcloud_deploy.py +++ b/distributedcloud/dcmanager/common/phased_subcloud_deploy.py @@ -7,11 +7,13 @@ import base64 import json import os +import typing import netaddr from oslo_log import log as logging import pecan import tsconfig.tsconfig as tsc +import yaml from dccommon import consts as dccommon_consts from dccommon.drivers.openstack import patching_v1 @@ -24,6 +26,7 @@ from dcmanager.common import exceptions from dcmanager.common.i18n import _ from dcmanager.common import utils from dcmanager.db import api as db_api +from dcmanager.db.sqlalchemy import models LOG = logging.getLogger(__name__) @@ -148,7 +151,8 @@ def validate_system_controller_patch_status(operation: str): % operation) -def validate_subcloud_config(context, payload, operation=None): +def validate_subcloud_config(context, payload, operation=None, + ignore_conflicts_with=None): """Check whether subcloud config is valid.""" # Validate the name @@ -173,6 +177,10 @@ def validate_subcloud_config(context, payload, operation=None): subcloud_subnets = [] subclouds = db_api.subcloud_get_all(context) for subcloud in subclouds: + # Ignore management subnet conflict with the subcloud specified by + # ignore_conflicts_with + if ignore_conflicts_with and (subcloud.id == ignore_conflicts_with.id): + continue subcloud_subnets.append(netaddr.IPNetwork(subcloud.management_subnet)) MIN_MANAGEMENT_SUBNET_SIZE = 8 @@ -775,3 +783,54 @@ def add_subcloud_to_database(context, payload): group_id, data_install=data_install) return subcloud + + +def get_request_data(request: pecan.Request, + subcloud: models.Subcloud, + subcloud_file_contents: typing.Sequence): + payload = dict() + for f in subcloud_file_contents: + if f in request.POST: + file_item = request.POST[f] + file_item.file.seek(0, os.SEEK_SET) + contents = file_item.file.read() + if subcloud.name and f == consts.DEPLOY_CONFIG: + fn = get_config_file_path(subcloud.name, f) + upload_config_file(contents, fn, f) + payload.update({f: fn}) + else: + data = yaml.safe_load(contents.decode('utf8')) + if f == consts.BOOTSTRAP_VALUES: + payload.update(data) + else: + payload.update({f: data}) + del request.POST[f] + payload.update(request.POST) + return payload + + +def populate_payload_with_pre_existing_data(payload: dict, + subcloud: models.Subcloud, + mandatory_values: typing.Sequence): + for value in mandatory_values: + if value == consts.INSTALL_VALUES: + pass + elif value == consts.BOOTSTRAP_VALUES: + filename = get_config_file_path(subcloud.name) + LOG.info("Loading existing bootstrap values from: %s" % filename) + try: + existing_values = utils.load_yaml_file(filename) + except FileNotFoundError: + msg = _("Required %s file was not provided and it was not " + "previously available.") % value + pecan.abort(400, msg) + payload.update(existing_values) + elif value == consts.DEPLOY_CONFIG: + if not payload.get(consts.DEPLOY_CONFIG): + fn = get_config_file_path(subcloud.name, value) + if not os.path.exists(fn): + msg = _("Required %s file was not provided and it was not " + "previously available.") % consts.DEPLOY_CONFIG + pecan.abort(400, msg) + payload.update({value: fn}) + get_common_deploy_files(payload, subcloud.software_version) diff --git a/distributedcloud/dcmanager/common/utils.py b/distributedcloud/dcmanager/common/utils.py index 2320c267d..eec88db80 100644 --- a/distributedcloud/dcmanager/common/utils.py +++ b/distributedcloud/dcmanager/common/utils.py @@ -18,6 +18,7 @@ import datetime import grp import itertools +import json import netaddr import os import pwd @@ -963,6 +964,39 @@ def get_value_from_yaml_file(filename, key): return value +def update_values_on_yaml_file(filename, values, yaml_dump=True): + """Update all specified key values from the given yaml file. + + :param filename: the yaml filename + :param values: dict with yaml keys and values to replace + :param yaml_dump: write file using yaml dump (default is True) + """ + update_file = False + if not os.path.isfile(filename): + return + with open(os.path.abspath(filename), 'r') as f: + data = f.read() + data = yaml.load(data, Loader=yaml.SafeLoader) + for key, value in values.items(): + if key not in data or value != data.get(key): + data.update({key: value}) + update_file = True + if update_file: + with open(os.path.abspath(filename), 'w') as f: + if yaml_dump: + yaml.dump(data, f, sort_keys=False) + else: + f.write('---\n') + for k, v in data.items(): + f.write("%s: %s\n" % (k, json.dumps(v))) + + +def load_yaml_file(filename: str): + with open(os.path.abspath(filename), 'r') as f: + data = yaml.load(f, Loader=yaml.loader.SafeLoader) + return data + + def decode_and_normalize_passwd(input_passwd): pattern = r'^[' + string.punctuation + ']' passwd = base64.decode_as_text(input_passwd) diff --git a/distributedcloud/dcmanager/db/api.py b/distributedcloud/dcmanager/db/api.py index 50d8c0644..83a60921c 100644 --- a/distributedcloud/dcmanager/db/api.py +++ b/distributedcloud/dcmanager/db/api.py @@ -24,6 +24,7 @@ from oslo_config import cfg from oslo_db import api from dccommon import consts as dccommon_consts +from dcmanager.db.sqlalchemy import models CONF = cfg.CONF @@ -151,7 +152,7 @@ def subcloud_get_with_status(context, subcloud_id): return IMPL.subcloud_get_with_status(context, subcloud_id) -def subcloud_get_by_name(context, name): +def subcloud_get_by_name(context, name) -> models.Subcloud: """Retrieve a subcloud by name or raise if it does not exist.""" return IMPL.subcloud_get_by_name(context, name) @@ -174,7 +175,9 @@ def subcloud_update(context, subcloud_id, management_state=None, deploy_status=None, backup_status=None, backup_datetime=None, error_description=None, openstack_installed=None, group_id=None, - data_install=None, data_upgrade=None, first_identity_sync_complete=None): + data_install=None, data_upgrade=None, + first_identity_sync_complete=None, + systemcontroller_gateway_ip=None): """Update a subcloud or raise if it does not exist.""" return IMPL.subcloud_update(context, subcloud_id, management_state, availability_status, software_version, @@ -182,7 +185,9 @@ def subcloud_update(context, subcloud_id, management_state=None, management_start_ip, management_end_ip, location, audit_fail_count, deploy_status, backup_status, backup_datetime, error_description, openstack_installed, - group_id, data_install, data_upgrade, first_identity_sync_complete) + group_id, data_install, data_upgrade, + first_identity_sync_complete, + systemcontroller_gateway_ip) def subcloud_bulk_update_by_ids(context, subcloud_ids, update_form): diff --git a/distributedcloud/dcmanager/db/sqlalchemy/api.py b/distributedcloud/dcmanager/db/sqlalchemy/api.py index 1b753a07e..c14e2a14e 100644 --- a/distributedcloud/dcmanager/db/sqlalchemy/api.py +++ b/distributedcloud/dcmanager/db/sqlalchemy/api.py @@ -383,7 +383,8 @@ def subcloud_update(context, subcloud_id, management_state=None, group_id=None, data_install=None, data_upgrade=None, - first_identity_sync_complete=None): + first_identity_sync_complete=None, + systemcontroller_gateway_ip=None): with write_session() as session: subcloud_ref = subcloud_get(context, subcloud_id) if management_state is not None: @@ -424,6 +425,9 @@ def subcloud_update(context, subcloud_id, management_state=None, subcloud_ref.group_id = group_id if first_identity_sync_complete is not None: subcloud_ref.first_identity_sync_complete = first_identity_sync_complete + if systemcontroller_gateway_ip is not None: + subcloud_ref.systemcontroller_gateway_ip = \ + systemcontroller_gateway_ip subcloud_ref.save(session) return subcloud_ref diff --git a/distributedcloud/dcmanager/manager/service.py b/distributedcloud/dcmanager/manager/service.py index 322d039da..feaabe1df 100644 --- a/distributedcloud/dcmanager/manager/service.py +++ b/distributedcloud/dcmanager/manager/service.py @@ -197,6 +197,15 @@ class DCManagerService(service.Service): subcloud_id, payload) + @request_context + def subcloud_deploy_bootstrap(self, context, subcloud_id, payload): + # Bootstraps a subcloud + LOG.info("Handling subcloud_deploy_bootstrap request for: %s" % + payload.get('name')) + return self.subcloud_manager.subcloud_deploy_bootstrap(context, + subcloud_id, + payload) + def _stop_rpc_server(self): # Stop RPC connection to prevent new requests LOG.debug(_("Attempting to stop RPC service...")) diff --git a/distributedcloud/dcmanager/manager/subcloud_manager.py b/distributedcloud/dcmanager/manager/subcloud_manager.py index 405a1ac4d..6ea412ceb 100644 --- a/distributedcloud/dcmanager/manager/subcloud_manager.py +++ b/distributedcloud/dcmanager/manager/subcloud_manager.py @@ -60,6 +60,7 @@ from dcmanager.db.sqlalchemy.models import Subcloud from dcmanager.rpc import client as dcmanager_rpc_client from dcorch.rpc import client as dcorch_rpc_client + LOG = logging.getLogger(__name__) # Name of our distributed cloud addn_hosts file for dnsmasq @@ -239,6 +240,7 @@ class SubcloudManager(manager.Manager): software_version if software_version else SW_VERSION] return install_command + # TODO(gherzman): rename compose_apply_command to compose_bootstrap_command def compose_apply_command(self, subcloud_name, ansible_subcloud_inventory_file, software_version=None): @@ -892,6 +894,86 @@ class SubcloudManager(manager.Manager): deploy_status=consts.DEPLOY_STATE_CREATE_FAILED) return db_api.subcloud_db_model_to_dict(subcloud) + def subcloud_deploy_bootstrap(self, context, subcloud_id, payload): + """Bootstrap subcloud + + :param context: request context object + :param subcloud_id: subcloud_id from db + :param payload: subcloud bootstrap configuration + """ + LOG.info("Bootstrapping subcloud %s." % payload['name']) + + try: + subcloud = db_api.subcloud_get(context, subcloud_id) + + management_subnet = utils.get_management_subnet(payload) + sys_controller_gw_ip = payload.get( + "systemcontroller_gateway_address") + + if (management_subnet != subcloud.management_subnet) or ( + sys_controller_gw_ip != subcloud.systemcontroller_gateway_ip): + m_ks_client = OpenStackDriver( + region_name=dccommon_consts.DEFAULT_REGION_NAME, + region_clients=None).keystone_client + # Create a new route + self._create_subcloud_route(payload, m_ks_client, + sys_controller_gw_ip) + # Delete previous route + self._delete_subcloud_routes(m_ks_client, subcloud) + # Update endpoints + self._update_services_endpoint(context, payload, subcloud.name, + m_ks_client) + + # Update subcloud + subcloud = db_api.subcloud_update( + context, + subcloud.id, + description=payload.get("description", None), + management_subnet=utils.get_management_subnet(payload), + management_gateway_ip=utils.get_management_gateway_address( + payload), + management_start_ip=utils.get_management_start_address( + payload), + management_end_ip=utils.get_management_end_address(payload), + systemcontroller_gateway_ip=payload.get( + "systemcontroller_gateway_address", None), + location=payload.get("location", None), + deploy_status=consts.DEPLOY_STATE_PRE_BOOTSTRAP) + + # Populate payload with passwords + payload['ansible_become_pass'] = payload['sysadmin_password'] + payload['ansible_ssh_pass'] = payload['sysadmin_password'] + payload['admin_password'] = str(keyring.get_password('CGCS', + 'admin')) + del payload['sysadmin_password'] + + # Update the ansible overrides file + overrides_file = os.path.join(consts.ANSIBLE_OVERRIDES_PATH, + subcloud.name + '.yml') + utils.update_values_on_yaml_file(overrides_file, payload) + + # Ansible inventory filename for the specified subcloud + ansible_subcloud_inventory_file = utils.get_ansible_filename( + subcloud.name, INVENTORY_FILE_POSTFIX) + + # Update the ansible inventory for the subcloud + utils.create_subcloud_inventory(payload, + ansible_subcloud_inventory_file) + + apply_command = self.compose_apply_command( + subcloud.name, + ansible_subcloud_inventory_file, + subcloud.software_version) + + self.run_deploy_commands(subcloud, payload, context, + apply_command=apply_command) + + except Exception: + LOG.exception("Failed to bootstrap subcloud %s" % payload['name']) + db_api.subcloud_update( + context, subcloud_id, + deploy_status=consts.DEPLOY_STATE_PRE_BOOTSTRAP_FAILED) + def _subcloud_operation_notice( self, operation, restore_subclouds, failed_subclouds, invalid_subclouds): @@ -1529,6 +1611,22 @@ class SubcloudManager(manager.Manager): deploy_status=consts.DEPLOY_STATE_DONE, error_description=consts.ERROR_DESC_EMPTY) + def run_deploy_commands(self, subcloud, payload, context, + install_command=None, apply_command=None, + deploy_command=None, rehome_command=None, + network_reconfig=None): + try: + log_file = ( + os.path.join(consts.DC_ANSIBLE_LOG_DIR, subcloud.name) + + "_playbook_output.log" + ) + if apply_command: + self._run_subcloud_bootstrap(context, subcloud, + apply_command, log_file) + except Exception as ex: + LOG.exception("run_deploy failed") + raise ex + @staticmethod def _run_subcloud_install( context, subcloud, install_command, log_file, payload): @@ -1574,6 +1672,35 @@ class SubcloudManager(manager.Manager): LOG.info("Successfully installed %s" % subcloud.name) return True + def _run_subcloud_bootstrap(self, context, subcloud, + apply_command, log_file): + # Update the subcloud deploy_status to bootstrapping + db_api.subcloud_update( + context, subcloud.id, + deploy_status=consts.DEPLOY_STATE_BOOTSTRAPPING, + error_description=consts.ERROR_DESC_EMPTY) + + # Run the ansible subcloud boostrap playbook + LOG.info("Starting bootstrap of %s" % subcloud.name) + try: + run_playbook(log_file, apply_command) + except PlaybookExecutionFailed: + msg = utils.find_ansible_error_msg( + subcloud.name, log_file, consts.DEPLOY_STATE_BOOTSTRAPPING) + LOG.error(msg) + db_api.subcloud_update( + context, subcloud.id, + deploy_status=consts.DEPLOY_STATE_BOOTSTRAP_FAILED, + error_description=msg[0:consts.ERROR_DESCRIPTION_LENGTH]) + return + + db_api.subcloud_update( + context, subcloud.id, + deploy_status=consts.DEPLOY_STATE_BOOTSTRAPPED, + error_description=consts.ERROR_DESC_EMPTY) + + LOG.info("Successfully bootstrapped %s" % subcloud.name) + def _create_addn_hosts_dc(self, context): """Generate the addn_hosts_dc file for hostname/ip translation""" @@ -2002,7 +2129,8 @@ class SubcloudManager(manager.Manager): m_ks_client = OpenStackDriver( region_name=dccommon_consts.DEFAULT_REGION_NAME, region_clients=None).keystone_client - self._create_subcloud_route(payload, m_ks_client, subcloud) + self._create_subcloud_route(payload, m_ks_client, + subcloud.systemcontroller_gateway_ip) except HTTPConflict: # The route already exists LOG.warning( @@ -2031,7 +2159,8 @@ class SubcloudManager(manager.Manager): # Delete old routes self._delete_subcloud_routes(m_ks_client, subcloud) - def _create_subcloud_route(self, payload, keystone_client, subcloud): + def _create_subcloud_route(self, payload, keystone_client, + systemcontroller_gateway_ip): subcloud_subnet = netaddr.IPNetwork(utils.get_management_subnet(payload)) endpoint = keystone_client.endpoint_cache.get_endpoint('sysinv') sysinv_client = SysinvClient(dccommon_consts.DEFAULT_REGION_NAME, @@ -2043,7 +2172,7 @@ class SubcloudManager(manager.Manager): sysinv_client.create_route(mgmt_if_uuid, str(subcloud_subnet.ip), subcloud_subnet.prefixlen, - subcloud.systemcontroller_gateway_ip, + systemcontroller_gateway_ip, 1) def _update_services_endpoint( diff --git a/distributedcloud/dcmanager/rpc/client.py b/distributedcloud/dcmanager/rpc/client.py index a430a7c43..fc0b029f2 100644 --- a/distributedcloud/dcmanager/rpc/client.py +++ b/distributedcloud/dcmanager/rpc/client.py @@ -192,6 +192,11 @@ class ManagerClient(RPCClient): subcloud_id=subcloud_id, payload=payload)) + def subcloud_deploy_bootstrap(self, ctxt, subcloud_id, payload): + return self.cast(ctxt, self.make_msg('subcloud_deploy_bootstrap', + subcloud_id=subcloud_id, + payload=payload)) + class DCManagerNotifications(RPCClient): """DC Manager Notification interface to broadcast subcloud state changed diff --git a/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_phased_subcloud_deploy.py b/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_phased_subcloud_deploy.py index 2d7a2562e..5f3574ce8 100644 --- a/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_phased_subcloud_deploy.py +++ b/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_phased_subcloud_deploy.py @@ -4,12 +4,32 @@ # SPDX-License-Identifier: Apache-2.0 # +import copy +import json import mock +from os import path as os_path +import six +import webtest +from dcmanager.common import consts from dcmanager.common import phased_subcloud_deploy as psd_common +from dcmanager.common import utils as dutils from dcmanager.db import api as db_api +from dcmanager.rpc import client as rpc_client +from dcmanager.tests.unit.api import test_root_controller as testroot +from dcmanager.tests.unit.api.v1.controllers.test_subclouds import \ + FakeAddressPool from dcmanager.tests.unit.api.v1.controllers.test_subclouds import \ TestSubcloudPost +from dcmanager.tests.unit.common import fake_subcloud +from dcmanager.tests import utils + +FAKE_URL = '/v1.0/phased-subcloud-deploy' + +FAKE_TENANT = utils.UUID1 + +FAKE_HEADERS = {'X-Tenant-Id': FAKE_TENANT, 'X_ROLE': 'admin,member,reader', + 'X-Identity-Status': 'Confirmed', 'X-Project-Name': 'admin'} class FakeRPCClient(object): @@ -56,3 +76,156 @@ class TestSubcloudDeployCreate(TestSubcloudPost): headers=self.get_api_headers(), expect_errors=True) self._verify_post_failure(response, "bootstrap-address", None) + + +class TestSubcloudDeployBootstrap(testroot.DCManagerApiTest): + def setUp(self): + super().setUp() + self.ctx = utils.dummy_context() + + p = mock.patch.object(rpc_client, 'ManagerClient') + self.mock_rpc_client = p.start() + self.addCleanup(p.stop) + + self.management_address_pool = FakeAddressPool('192.168.204.0', 24, + '192.168.204.2', + '192.168.204.100') + + p = mock.patch.object(psd_common, 'get_network_address_pool') + self.mock_get_network_address_pool = p.start() + self.mock_get_network_address_pool.return_value = \ + self.management_address_pool + self.addCleanup(p.stop) + + p = mock.patch.object(psd_common, 'get_ks_client') + self.mock_get_ks_client = p.start() + self.addCleanup(p.stop) + + p = mock.patch.object(psd_common.PatchingClient, 'query') + self.mock_query = p.start() + self.addCleanup(p.stop) + + @mock.patch.object(dutils, 'load_yaml_file') + @mock.patch.object(os_path, 'exists') + def test_subcloud_bootstrap(self, mock_path_exists, mock_load_yaml): + mock_path_exists.side_effect = [False, False, False, False, True] + mock_load_yaml.return_value = { + "software_version": fake_subcloud.FAKE_SOFTWARE_VERSION} + + subcloud = fake_subcloud.create_fake_subcloud( + self.ctx, + name=fake_subcloud.FAKE_BOOTSTRAP_FILE_DATA["name"], + deploy_status=consts.DEPLOY_STATE_INSTALLED) + + fake_content = json.dumps( + fake_subcloud.FAKE_BOOTSTRAP_FILE_DATA).encode("utf-8") + response = self.app.patch( + FAKE_URL + '/' + str(subcloud.id) + '/bootstrap', + headers=FAKE_HEADERS, + params=fake_subcloud.FAKE_BOOTSTRAP_VALUE, + upload_files=[("bootstrap_values", + "bootstrap_fake_filename", + fake_content)]) + + self.assertEqual(response.status_int, 200) + self.mock_rpc_client.return_value.subcloud_deploy_bootstrap.\ + assert_called_once() + + expected_payload = {**fake_subcloud.FAKE_BOOTSTRAP_VALUE, + **fake_subcloud.FAKE_BOOTSTRAP_FILE_DATA} + expected_payload["sysadmin_password"] = "testpass" + expected_payload["software_version"] = \ + fake_subcloud.FAKE_SOFTWARE_VERSION + + (_, res_subcloud_id, res_payload), _ = self.mock_rpc_client.\ + return_value.subcloud_deploy_bootstrap.call_args + + self.assertDictEqual(res_payload, expected_payload) + self.assertEqual(res_subcloud_id, subcloud.id) + + def test_subcloud_bootstrap_no_body(self): + subcloud = fake_subcloud.create_fake_subcloud( + self.ctx, + name=fake_subcloud.FAKE_BOOTSTRAP_FILE_DATA["name"], + deploy_status=consts.DEPLOY_STATE_INSTALLED) + url = FAKE_URL + '/' + str(subcloud.id) + '/bootstrap' + six.assertRaisesRegex(self, webtest.app.AppError, "400 *", + self.app.patch_json, url, + headers=FAKE_HEADERS, params={}) + + def test_subcloud_bootstrap_subcloud_not_found(self): + url = FAKE_URL + '/' + "nonexistent_subcloud" + '/bootstrap' + six.assertRaisesRegex(self, webtest.app.AppError, "404 *", + self.app.patch_json, url, + headers=FAKE_HEADERS, params={}) + + @mock.patch.object(dutils, 'load_yaml_file') + @mock.patch.object(os_path, 'exists') + def test_subcloud_bootstrap_no_bootstrap_values_on_request( + self, mock_path_exists, mock_load_yaml_file): + mock_path_exists.side_effect = [False, False, False, False, True] + fake_bootstrap_values = copy.copy( + fake_subcloud.FAKE_BOOTSTRAP_FILE_DATA) + fake_bootstrap_values["software_version"] = \ + fake_subcloud.FAKE_SOFTWARE_VERSION + mock_load_yaml_file.return_value = \ + fake_bootstrap_values + + subcloud = fake_subcloud.create_fake_subcloud( + self.ctx, + name=fake_subcloud.FAKE_BOOTSTRAP_FILE_DATA["name"], + deploy_status=consts.DEPLOY_STATE_INSTALLED) + + response = self.app.patch( + FAKE_URL + '/' + str(subcloud.id) + '/bootstrap', + headers=FAKE_HEADERS, + params=fake_subcloud.FAKE_BOOTSTRAP_VALUE) + + self.assertEqual(response.status_int, 200) + self.mock_rpc_client.return_value.subcloud_deploy_bootstrap.\ + assert_called_once() + + expected_payload = {**fake_subcloud.FAKE_BOOTSTRAP_VALUE, + **fake_subcloud.FAKE_BOOTSTRAP_FILE_DATA} + expected_payload["sysadmin_password"] = "testpass" + expected_payload["software_version"] = \ + fake_subcloud.FAKE_SOFTWARE_VERSION + + (_, res_subcloud_id, res_payload), _ = self.mock_rpc_client.\ + return_value.subcloud_deploy_bootstrap.call_args + + self.assertDictEqual(res_payload, expected_payload) + self.assertEqual(res_subcloud_id, subcloud.id) + + def test_subcloud_bootstrap_management_subnet_conflict(self): + conflicting_subnet = { + "management_subnet": "192.168.102.0/24", + "management_start_ip": "192.168.102.2", + "management_end_ip": "192.168.102.50", + "management_gateway_ip": "192.168.102.1"} + + fake_subcloud.create_fake_subcloud( + self.ctx, + name="existing_subcloud", + deploy_status=consts.DEPLOY_STATE_DONE, + **conflicting_subnet + ) + + subcloud = fake_subcloud.create_fake_subcloud( + self.ctx, + name=fake_subcloud.FAKE_BOOTSTRAP_FILE_DATA["name"], + deploy_status=consts.DEPLOY_STATE_INSTALLED) + + modified_bootstrap_data = copy.copy( + fake_subcloud.FAKE_BOOTSTRAP_FILE_DATA) + modified_bootstrap_data.update(conflicting_subnet) + + fake_content = json.dumps(modified_bootstrap_data).encode("utf-8") + url = FAKE_URL + '/' + str(subcloud.id) + '/bootstrap' + six.assertRaisesRegex(self, webtest.app.AppError, "400 *", + self.app.patch, url, + headers=FAKE_HEADERS, + params=fake_subcloud.FAKE_BOOTSTRAP_VALUE, + upload_files=[("bootstrap_values", + "bootstrap_fake_filename", + fake_content)]) diff --git a/distributedcloud/dcmanager/tests/unit/common/fake_subcloud.py b/distributedcloud/dcmanager/tests/unit/common/fake_subcloud.py index 6792354a9..aac9d8cf3 100644 --- a/distributedcloud/dcmanager/tests/unit/common/fake_subcloud.py +++ b/distributedcloud/dcmanager/tests/unit/common/fake_subcloud.py @@ -61,6 +61,19 @@ FAKE_SUBCLOUD_BOOTSTRAP_PAYLOAD = { (base64.b64encode('testpass'.encode("utf-8"))).decode('ascii'), } +FAKE_BOOTSTRAP_FILE_DATA = { + "system_mode": "simplex", + "name": "fake subcloud1", + "management_subnet": "192.168.101.0/24", + "management_start_address": "192.168.101.2", + "management_end_address": "192.168.101.50", + "management_gateway_address": "192.168.101.1", + "external_oam_subnet": "10.10.10.0/24", + "external_oam_gateway_address": "10.10.10.1", + "external_oam_floating_address": "10.10.10.12", + "systemcontroller_gateway_address": "192.168.204.101", +} + FAKE_SUBCLOUD_INSTALL_VALUES = { "image": "http://192.168.101.2:8080/iso/bootimage.iso", "software_version": FAKE_SOFTWARE_VERSION, diff --git a/distributedcloud/dcmanager/tests/unit/manager/test_subcloud_manager.py b/distributedcloud/dcmanager/tests/unit/manager/test_subcloud_manager.py index 617da42c4..321241f16 100644 --- a/distributedcloud/dcmanager/tests/unit/manager/test_subcloud_manager.py +++ b/distributedcloud/dcmanager/tests/unit/manager/test_subcloud_manager.py @@ -32,6 +32,7 @@ from dcmanager.common import consts from dcmanager.common import exceptions from dcmanager.common import prestage from dcmanager.common import utils as cutils +from dcmanager.db import api as dc_db_api from dcmanager.db.sqlalchemy import api as db_api from dcmanager.manager import subcloud_manager from dcmanager.state import subcloud_state_manager @@ -503,6 +504,59 @@ class TestSubcloudManager(base.DCManagerTestCase): self.assertEqual(consts.DEPLOY_STATE_CREATE_FAILED, updated_subcloud.deploy_status) + @mock.patch.object(cutils, 'create_subcloud_inventory') + @mock.patch.object(subcloud_manager, 'keyring') + @mock.patch.object(cutils, 'get_playbook_for_software_version') + @mock.patch.object(cutils, 'update_values_on_yaml_file') + @mock.patch.object(subcloud_manager, 'run_playbook') + def test_subcloud_deploy_bootstrap(self, mock_run_playbook, mock_update_yml, + mock_get_playbook_for_software_version, + mock_keyring, create_subcloud_inventory): + mock_get_playbook_for_software_version.return_value = "22.12" + mock_keyring.get_password.return_value = "testpass" + + subcloud = fake_subcloud.create_fake_subcloud( + self.ctx, + name=fake_subcloud.FAKE_BOOTSTRAP_FILE_DATA["name"], + deploy_status=consts.DEPLOY_STATE_INSTALLED) + + payload = {**fake_subcloud.FAKE_BOOTSTRAP_VALUE, + **fake_subcloud.FAKE_BOOTSTRAP_FILE_DATA} + payload["sysadmin_password"] = "testpass" + + sm = subcloud_manager.SubcloudManager() + sm.subcloud_deploy_bootstrap(self.ctx, subcloud.id, payload) + + mock_run_playbook.assert_called_once() + + # Verify subcloud was updated with correct values + updated_subcloud = db_api.subcloud_get_by_name(self.ctx, + payload['name']) + self.assertEqual(consts.DEPLOY_STATE_BOOTSTRAPPED, + updated_subcloud.deploy_status) + + @mock.patch.object(dc_db_api, 'subcloud_get') + def test_subcloud_deploy_bootstrap_failed(self, mock_subcloud_get): + mock_subcloud_get.side_effect = FakeException('boom') + + subcloud = fake_subcloud.create_fake_subcloud( + self.ctx, + name=fake_subcloud.FAKE_BOOTSTRAP_FILE_DATA["name"], + deploy_status=consts.DEPLOY_STATE_INSTALLED) + + payload = {**fake_subcloud.FAKE_BOOTSTRAP_VALUE, + **fake_subcloud.FAKE_BOOTSTRAP_FILE_DATA} + payload["sysadmin_password"] = "testpass" + + sm = subcloud_manager.SubcloudManager() + sm.subcloud_deploy_bootstrap(self.ctx, subcloud.id, payload) + + # Verify subcloud was updated with correct values + updated_subcloud = db_api.subcloud_get_by_name(self.ctx, + payload['name']) + self.assertEqual(consts.DEPLOY_STATE_PRE_BOOTSTRAP_FAILED, + updated_subcloud.deploy_status) + @mock.patch.object(subcloud_manager.SubcloudManager, 'compose_apply_command') @mock.patch.object(subcloud_manager.SubcloudManager,