From 32f6fc5805e95d66017e97d69739bfd2f3474a35 Mon Sep 17 00:00:00 2001 From: Gustavo Herzmann Date: Fri, 26 May 2023 12:49:09 -0300 Subject: [PATCH] Add 'subcloud deploy bootstrap' command to dcmanager Adds the subcloud deploy bootstrap command to dcmanager. It only performs the bootstrap phase, where all parameters are validated and the bootstrap playbook is executed. The bootstrap values can be updated using the bootstrap-values parameter, otherwise the system will use the boostrap values from the subcloud deploy create command. Test Plan: The following tests were run twice, once using the CLI, and once using CURL to request the API directly. The subcloud was created using the follwing command: dcmanager subcloud deploy create --bootstrap-address --bootstrap-values Except for the different release test, where the --release 21.12 parameter was also used. The following commands were used to test the bootstrap (changing the parameters accordingly for each test): CLI: dcmanager subcloud deploy bootstrap --bootstrap-address --bootstrap-values --sysadmin-password CURL: curl -X PATCH -H "X-Auth-Token: ${TOKEN//[$'\t\r\n']}" "http://$MGMT_IP:8119/v1.0/phased-subcloud-deploy/ /bootstrap" -F bootstrap_values=@ -F bootstrap-address= -F sysadmin_password= Success cases: 1. PASS - Bootstrap a subcloud without the bootstrap-values and bootstrap-address parameters (omit them from the CLI/CURL command) and verify that it uses the existing values (from the deploy create command); 2. PASS - Bootstrap a subcloud with the same bootstrap-values and bootstrap-address parameters used during deploy create (include the parameters in the CLI/CURL command, using the same values used during deploy create); 3. PASS - Bootstrap a subcloud with bootstrap-values containing a new value for management_subnet, verifying that the routes and endpoints are updated accordingly; 4. PASS - Verify that if a new bootstrap-address is used, the subcloud inventory file is updated accordingly; 5. PASS - Verify subcloud bootstrap with previous release version (tested with 21.12); Failure cases: 6. PASS - Verify that it's not possible to bootstrap with a management_subnet that conflicts with the subnet of an existing subcloud; 7. PASS - Verify that it's not possible to bootstrap when the deploy status is not in one of the following states: 'install-complete', 'bootstrap-failed', 'bootstrap-aborted', 'bootstrap-complete' and 'create-complete'; 8. PASS - Verify that it's not possible to bootstrap when the 'name' parameter from bootstrap-values doesn't match the subcloud name; 9. PASS - Verify that it's not possible to bootstrap without passing the 'sysadmin-password' parameter (using CURL, since the CLI will prompt for the password if it's omited); 10. PASS - Verify that it's not possible to bootstrap with a 'sysadmin-password' that's not b64encoded (using CURL, since the CLI automatically encodes the password). Story: 2010756 Task: 48114 Change-Id: Ic987406bbcc154a8edbaa008c40ccabe1766d03d Signed-off-by: Gustavo Herzmann --- api-ref/source/api-ref-dcmanager-v1.rst | 67 +++++++ ...bcloud-deploy-patch-bootstrap-request.json | 5 + .../controllers/v1/phased_subcloud_deploy.py | 132 ++++++++++++- .../api/policies/phased_subcloud_deploy.py | 11 ++ distributedcloud/dcmanager/common/consts.py | 10 +- .../common/phased_subcloud_deploy.py | 61 +++++- distributedcloud/dcmanager/common/utils.py | 34 ++++ distributedcloud/dcmanager/db/api.py | 11 +- .../dcmanager/db/sqlalchemy/api.py | 6 +- distributedcloud/dcmanager/manager/service.py | 9 + .../dcmanager/manager/subcloud_manager.py | 135 +++++++++++++- distributedcloud/dcmanager/rpc/client.py | 5 + .../test_phased_subcloud_deploy.py | 173 ++++++++++++++++++ .../tests/unit/common/fake_subcloud.py | 13 ++ .../unit/manager/test_subcloud_manager.py | 54 ++++++ 15 files changed, 708 insertions(+), 18 deletions(-) create mode 100644 api-ref/source/samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-bootstrap-request.json diff --git a/api-ref/source/api-ref-dcmanager-v1.rst b/api-ref/source/api-ref-dcmanager-v1.rst index 4a6ae7bf1..5842e1e9c 100644 --- a/api-ref/source/api-ref-dcmanager-v1.rst +++ b/api-ref/source/api-ref-dcmanager-v1.rst @@ -1812,6 +1812,73 @@ Request Example :language: json +**Response parameters** + +.. rest_parameters:: parameters.yaml + + - id: subcloud_id + - name: subcloud_name + - description: subcloud_description + - location: subcloud_location + - software-version: software_version + - management-state: management_state + - availability-status: availability_status + - deploy-status: deploy_status + - backup-status: backup_status + - backup-datetime: backup_datetime + - error-description: error_description + - management-subnet: management_subnet + - management-start-ip: management_start_ip + - management-end-ip: management_end_ip + - management-gateway-ip: management_gateway_ip + - openstack-installed: openstack_installed + - systemcontroller-gateway-ip: systemcontroller_gateway_ip + - data_install: data_install + - data_upgrade: data_upgrade + - created-at: created_at + - updated-at: updated_at + - group_id: group_id + +Response Example +---------------- + +.. literalinclude:: samples/phased-subcloud-deploy/phased-subcloud-deploy-post-response.json + :language: json + +********************* +Bootstraps a subcloud +********************* + +.. rest_method:: PATCH /v1.0/phased-subcloud-deploy/bootstrap + +Accepts Content-Type multipart/form-data. + + +**Normal response codes** + +200 + +**Error response codes** + +badRequest (400), unauthorized (401), forbidden (403), badMethod (405), +conflict (409), HTTPUnprocessableEntity (422), internalServerError (500), +serviceUnavailable (503) + +**Request parameters** + +.. rest_parameters:: parameters.yaml + + - bootstrap-address: bootstrap_address + - bootstrap_values: bootstrap_values + - sysadmin_password: sysadmin_password + +Request Example +---------------- + +.. literalinclude:: samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-bootstrap-request.json + :language: json + + **Response parameters** .. rest_parameters:: parameters.yaml diff --git a/api-ref/source/samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-bootstrap-request.json b/api-ref/source/samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-bootstrap-request.json new file mode 100644 index 000000000..b1a53acbb --- /dev/null +++ b/api-ref/source/samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-bootstrap-request.json @@ -0,0 +1,5 @@ +{ + "bootstrap-address": "10.10.10.12", + "bootstrap_values": "content of bootstrap_values file", + "sysadmin_password": "XXXXXXX" +} \ No newline at end of file diff --git a/distributedcloud/dcmanager/api/controllers/v1/phased_subcloud_deploy.py b/distributedcloud/dcmanager/api/controllers/v1/phased_subcloud_deploy.py index 660ca7415..f509653e8 100644 --- a/distributedcloud/dcmanager/api/controllers/v1/phased_subcloud_deploy.py +++ b/distributedcloud/dcmanager/api/controllers/v1/phased_subcloud_deploy.py @@ -17,31 +17,45 @@ from dcmanager.api.controllers import restcomm from dcmanager.api.policies import phased_subcloud_deploy as \ phased_subcloud_deploy_policy from dcmanager.api import policy +from dcmanager.common import consts from dcmanager.common.context import RequestContext +from dcmanager.common import exceptions from dcmanager.common.i18n import _ from dcmanager.common import phased_subcloud_deploy as psd_common from dcmanager.common import utils +from dcmanager.db import api as db_api +from dcmanager.db.sqlalchemy import models from dcmanager.rpc import client as rpc_client LOG = logging.getLogger(__name__) LOCK_NAME = 'PhasedSubcloudDeployController' -BOOTSTRAP_ADDRESS = 'bootstrap-address' -BOOTSTRAP_VALUES = 'bootstrap_values' -INSTALL_VALUES = 'install_values' - SUBCLOUD_CREATE_REQUIRED_PARAMETERS = ( - BOOTSTRAP_VALUES, - BOOTSTRAP_ADDRESS + consts.BOOTSTRAP_VALUES, + consts.BOOTSTRAP_ADDRESS ) # The consts.DEPLOY_CONFIG is missing here because it's handled differently # by the upload_deploy_config_file() function SUBCLOUD_CREATE_GET_FILE_CONTENTS = ( - BOOTSTRAP_VALUES, - INSTALL_VALUES, + consts.BOOTSTRAP_VALUES, + consts.INSTALL_VALUES, ) +SUBCLOUD_BOOTSTRAP_GET_FILE_CONTENTS = ( + consts.BOOTSTRAP_VALUES, +) + +VALID_STATES_FOR_DEPLOY_BOOTSTRAP = [ + consts.DEPLOY_STATE_INSTALLED, + consts.DEPLOY_STATE_BOOTSTRAP_FAILED, + consts.DEPLOY_STATE_BOOTSTRAP_ABORTED, + consts.DEPLOY_STATE_BOOTSTRAPPED, + # The subcloud can be installed manually (without remote install) so we need + # to allow the bootstrap operation when the state == DEPLOY_STATE_CREATED + consts.DEPLOY_STATE_CREATED +] + def get_create_payload(request: pecan.Request) -> dict: payload = dict() @@ -51,7 +65,7 @@ def get_create_payload(request: pecan.Request) -> dict: file_item = request.POST[f] file_item.file.seek(0, os.SEEK_SET) data = yaml.safe_load(file_item.file.read().decode('utf8')) - if f == BOOTSTRAP_VALUES: + if f == consts.BOOTSTRAP_VALUES: payload.update(data) else: payload.update({f: data}) @@ -118,6 +132,73 @@ class PhasedSubcloudDeployController(object): pecan.abort(httpclient.INTERNAL_SERVER_ERROR, _('Unable to create subcloud')) + def _deploy_bootstrap(self, context: RequestContext, + request: pecan.Request, + subcloud: models.Subcloud): + if subcloud.deploy_status not in VALID_STATES_FOR_DEPLOY_BOOTSTRAP: + valid_states_str = ', '.join(VALID_STATES_FOR_DEPLOY_BOOTSTRAP) + pecan.abort(400, _('Subcloud deploy status must be either: %s') + % valid_states_str) + + has_bootstrap_values = consts.BOOTSTRAP_VALUES in request.POST + payload = {} + + # Try to load the existing override values + override_file = psd_common.get_config_file_path(subcloud.name) + if os.path.exists(override_file): + psd_common.populate_payload_with_pre_existing_data( + payload, subcloud, SUBCLOUD_BOOTSTRAP_GET_FILE_CONTENTS) + elif not has_bootstrap_values: + msg = _("Required bootstrap-values file was not provided and it was" + " not previously available at %s") % (override_file) + pecan.abort(400, msg) + + request_data = psd_common.get_request_data( + request, subcloud, SUBCLOUD_BOOTSTRAP_GET_FILE_CONTENTS) + + # Update the existing values with new ones from the request + payload.update(request_data) + + psd_common.validate_sysadmin_password(payload) + + if has_bootstrap_values: + # Need to validate the new values + playload_name = payload.get('name') + if playload_name != subcloud.name: + pecan.abort(400, _('The bootstrap-values "name" value (%s) ' + 'must match the current subcloud name (%s)' % + (playload_name, subcloud.name))) + + # Verify if payload contains all required bootstrap values + psd_common.validate_bootstrap_values(payload) + + # It's ok for the management subnet to conflict with itself since we + # are only going to update it if it was modified, conflicts with + # other subclouds are still verified. + psd_common.validate_subcloud_config(context, payload, + ignore_conflicts_with=subcloud) + psd_common.format_ip_address(payload) + + # Patch status and fresh_install_k8s_version may have been changed + # between deploy create and deploy bootstrap commands. Validate them + # again: + psd_common.validate_system_controller_patch_status("bootstrap") + psd_common.validate_k8s_version(payload) + + try: + # Ask dcmanager-manager to bootstrap the subcloud. + self.dcmanager_rpc_client.subcloud_deploy_bootstrap( + context, subcloud.id, payload) + return db_api.subcloud_db_model_to_dict(subcloud) + + except RemoteError as e: + pecan.abort(httpclient.UNPROCESSABLE_ENTITY, e.value) + except Exception: + LOG.exception("Unable to bootstrap subcloud %s" % + payload.get('name')) + pecan.abort(httpclient.INTERNAL_SERVER_ERROR, + _('Unable to bootstrap subcloud')) + @pecan.expose(generic=True, template='json') def index(self): # Route the request to specific methods with parameters @@ -128,3 +209,36 @@ class PhasedSubcloudDeployController(object): def post(self): context = restcomm.extract_context_from_environ() return self._deploy_create(context, pecan.request) + + @utils.synchronized(LOCK_NAME) + @index.when(method='PATCH', template='json') + def patch(self, subcloud_ref=None, verb=None): + """Modify the subcloud deployment. + + :param subcloud_ref: ID or name of subcloud to update + + :param verb: Specifies the patch action to be taken + or subcloud operation + """ + + policy.authorize(phased_subcloud_deploy_policy.POLICY_ROOT % "modify", {}, + restcomm.extract_credentials_for_policy()) + context = restcomm.extract_context_from_environ() + + if not subcloud_ref: + pecan.abort(400, _('Subcloud ID required')) + + try: + if subcloud_ref.isdigit(): + subcloud = db_api.subcloud_get(context, subcloud_ref) + else: + subcloud = db_api.subcloud_get_by_name(context, subcloud_ref) + except (exceptions.SubcloudNotFound, exceptions.SubcloudNameNotFound): + pecan.abort(404, _('Subcloud not found')) + + if verb == 'bootstrap': + subcloud = self._deploy_bootstrap(context, pecan.request, subcloud) + else: + pecan.abort(400, _('Invalid request')) + + return subcloud diff --git a/distributedcloud/dcmanager/api/policies/phased_subcloud_deploy.py b/distributedcloud/dcmanager/api/policies/phased_subcloud_deploy.py index b8dc452ec..1faea5a6e 100644 --- a/distributedcloud/dcmanager/api/policies/phased_subcloud_deploy.py +++ b/distributedcloud/dcmanager/api/policies/phased_subcloud_deploy.py @@ -22,6 +22,17 @@ phased_subcloud_deploy_rules = [ } ] ), + policy.DocumentedRuleDefault( + name=POLICY_ROOT % 'modify', + check_str='rule:' + base.ADMIN_IN_SYSTEM_PROJECTS, + description="Modify the subcloud deployment.", + operations=[ + { + 'method': 'PATCH', + 'path': '/v1.0/phased-subcloud-deploy/{subcloud}/bootstrap' + } + ] + ) ] diff --git a/distributedcloud/dcmanager/common/consts.py b/distributedcloud/dcmanager/common/consts.py index 6434db314..a6d5b34b4 100644 --- a/distributedcloud/dcmanager/common/consts.py +++ b/distributedcloud/dcmanager/common/consts.py @@ -26,6 +26,10 @@ CERTS_VAULT_DIR = "/opt/dc-vault/certs" LOADS_VAULT_DIR = "/opt/dc-vault/loads" PATCH_VAULT_DIR = "/opt/dc-vault/patches" +BOOTSTRAP_VALUES = 'bootstrap_values' +BOOTSTRAP_ADDRESS = 'bootstrap-address' +INSTALL_VALUES = 'install_values' + # Admin status for hosts ADMIN_LOCKED = 'locked' ADMIN_UNLOCKED = 'unlocked' @@ -168,9 +172,13 @@ DEPLOY_STATE_PRE_INSTALL = 'pre-install' DEPLOY_STATE_PRE_INSTALL_FAILED = 'pre-install-failed' DEPLOY_STATE_INSTALLING = 'installing' DEPLOY_STATE_INSTALL_FAILED = 'install-failed' -DEPLOY_STATE_INSTALLED = 'installed' +DEPLOY_STATE_INSTALLED = 'install-complete' +DEPLOY_STATE_PRE_BOOTSTRAP = 'pre-bootstrap' +DEPLOY_STATE_PRE_BOOTSTRAP_FAILED = 'pre-bootstrap-failed' DEPLOY_STATE_BOOTSTRAPPING = 'bootstrapping' DEPLOY_STATE_BOOTSTRAP_FAILED = 'bootstrap-failed' +DEPLOY_STATE_BOOTSTRAP_ABORTED = 'bootstrap-aborted' +DEPLOY_STATE_BOOTSTRAPPED = 'bootstrap-complete' DEPLOY_STATE_DEPLOYING = 'deploying' DEPLOY_STATE_DEPLOY_FAILED = 'deploy-failed' DEPLOY_STATE_MIGRATING_DATA = 'migrating-data' diff --git a/distributedcloud/dcmanager/common/phased_subcloud_deploy.py b/distributedcloud/dcmanager/common/phased_subcloud_deploy.py index e1306f434..ae989365d 100644 --- a/distributedcloud/dcmanager/common/phased_subcloud_deploy.py +++ b/distributedcloud/dcmanager/common/phased_subcloud_deploy.py @@ -7,11 +7,13 @@ import base64 import json import os +import typing import netaddr from oslo_log import log as logging import pecan import tsconfig.tsconfig as tsc +import yaml from dccommon import consts as dccommon_consts from dccommon.drivers.openstack import patching_v1 @@ -24,6 +26,7 @@ from dcmanager.common import exceptions from dcmanager.common.i18n import _ from dcmanager.common import utils from dcmanager.db import api as db_api +from dcmanager.db.sqlalchemy import models LOG = logging.getLogger(__name__) @@ -148,7 +151,8 @@ def validate_system_controller_patch_status(operation: str): % operation) -def validate_subcloud_config(context, payload, operation=None): +def validate_subcloud_config(context, payload, operation=None, + ignore_conflicts_with=None): """Check whether subcloud config is valid.""" # Validate the name @@ -173,6 +177,10 @@ def validate_subcloud_config(context, payload, operation=None): subcloud_subnets = [] subclouds = db_api.subcloud_get_all(context) for subcloud in subclouds: + # Ignore management subnet conflict with the subcloud specified by + # ignore_conflicts_with + if ignore_conflicts_with and (subcloud.id == ignore_conflicts_with.id): + continue subcloud_subnets.append(netaddr.IPNetwork(subcloud.management_subnet)) MIN_MANAGEMENT_SUBNET_SIZE = 8 @@ -775,3 +783,54 @@ def add_subcloud_to_database(context, payload): group_id, data_install=data_install) return subcloud + + +def get_request_data(request: pecan.Request, + subcloud: models.Subcloud, + subcloud_file_contents: typing.Sequence): + payload = dict() + for f in subcloud_file_contents: + if f in request.POST: + file_item = request.POST[f] + file_item.file.seek(0, os.SEEK_SET) + contents = file_item.file.read() + if subcloud.name and f == consts.DEPLOY_CONFIG: + fn = get_config_file_path(subcloud.name, f) + upload_config_file(contents, fn, f) + payload.update({f: fn}) + else: + data = yaml.safe_load(contents.decode('utf8')) + if f == consts.BOOTSTRAP_VALUES: + payload.update(data) + else: + payload.update({f: data}) + del request.POST[f] + payload.update(request.POST) + return payload + + +def populate_payload_with_pre_existing_data(payload: dict, + subcloud: models.Subcloud, + mandatory_values: typing.Sequence): + for value in mandatory_values: + if value == consts.INSTALL_VALUES: + pass + elif value == consts.BOOTSTRAP_VALUES: + filename = get_config_file_path(subcloud.name) + LOG.info("Loading existing bootstrap values from: %s" % filename) + try: + existing_values = utils.load_yaml_file(filename) + except FileNotFoundError: + msg = _("Required %s file was not provided and it was not " + "previously available.") % value + pecan.abort(400, msg) + payload.update(existing_values) + elif value == consts.DEPLOY_CONFIG: + if not payload.get(consts.DEPLOY_CONFIG): + fn = get_config_file_path(subcloud.name, value) + if not os.path.exists(fn): + msg = _("Required %s file was not provided and it was not " + "previously available.") % consts.DEPLOY_CONFIG + pecan.abort(400, msg) + payload.update({value: fn}) + get_common_deploy_files(payload, subcloud.software_version) diff --git a/distributedcloud/dcmanager/common/utils.py b/distributedcloud/dcmanager/common/utils.py index 2320c267d..eec88db80 100644 --- a/distributedcloud/dcmanager/common/utils.py +++ b/distributedcloud/dcmanager/common/utils.py @@ -18,6 +18,7 @@ import datetime import grp import itertools +import json import netaddr import os import pwd @@ -963,6 +964,39 @@ def get_value_from_yaml_file(filename, key): return value +def update_values_on_yaml_file(filename, values, yaml_dump=True): + """Update all specified key values from the given yaml file. + + :param filename: the yaml filename + :param values: dict with yaml keys and values to replace + :param yaml_dump: write file using yaml dump (default is True) + """ + update_file = False + if not os.path.isfile(filename): + return + with open(os.path.abspath(filename), 'r') as f: + data = f.read() + data = yaml.load(data, Loader=yaml.SafeLoader) + for key, value in values.items(): + if key not in data or value != data.get(key): + data.update({key: value}) + update_file = True + if update_file: + with open(os.path.abspath(filename), 'w') as f: + if yaml_dump: + yaml.dump(data, f, sort_keys=False) + else: + f.write('---\n') + for k, v in data.items(): + f.write("%s: %s\n" % (k, json.dumps(v))) + + +def load_yaml_file(filename: str): + with open(os.path.abspath(filename), 'r') as f: + data = yaml.load(f, Loader=yaml.loader.SafeLoader) + return data + + def decode_and_normalize_passwd(input_passwd): pattern = r'^[' + string.punctuation + ']' passwd = base64.decode_as_text(input_passwd) diff --git a/distributedcloud/dcmanager/db/api.py b/distributedcloud/dcmanager/db/api.py index 50d8c0644..83a60921c 100644 --- a/distributedcloud/dcmanager/db/api.py +++ b/distributedcloud/dcmanager/db/api.py @@ -24,6 +24,7 @@ from oslo_config import cfg from oslo_db import api from dccommon import consts as dccommon_consts +from dcmanager.db.sqlalchemy import models CONF = cfg.CONF @@ -151,7 +152,7 @@ def subcloud_get_with_status(context, subcloud_id): return IMPL.subcloud_get_with_status(context, subcloud_id) -def subcloud_get_by_name(context, name): +def subcloud_get_by_name(context, name) -> models.Subcloud: """Retrieve a subcloud by name or raise if it does not exist.""" return IMPL.subcloud_get_by_name(context, name) @@ -174,7 +175,9 @@ def subcloud_update(context, subcloud_id, management_state=None, deploy_status=None, backup_status=None, backup_datetime=None, error_description=None, openstack_installed=None, group_id=None, - data_install=None, data_upgrade=None, first_identity_sync_complete=None): + data_install=None, data_upgrade=None, + first_identity_sync_complete=None, + systemcontroller_gateway_ip=None): """Update a subcloud or raise if it does not exist.""" return IMPL.subcloud_update(context, subcloud_id, management_state, availability_status, software_version, @@ -182,7 +185,9 @@ def subcloud_update(context, subcloud_id, management_state=None, management_start_ip, management_end_ip, location, audit_fail_count, deploy_status, backup_status, backup_datetime, error_description, openstack_installed, - group_id, data_install, data_upgrade, first_identity_sync_complete) + group_id, data_install, data_upgrade, + first_identity_sync_complete, + systemcontroller_gateway_ip) def subcloud_bulk_update_by_ids(context, subcloud_ids, update_form): diff --git a/distributedcloud/dcmanager/db/sqlalchemy/api.py b/distributedcloud/dcmanager/db/sqlalchemy/api.py index 1b753a07e..c14e2a14e 100644 --- a/distributedcloud/dcmanager/db/sqlalchemy/api.py +++ b/distributedcloud/dcmanager/db/sqlalchemy/api.py @@ -383,7 +383,8 @@ def subcloud_update(context, subcloud_id, management_state=None, group_id=None, data_install=None, data_upgrade=None, - first_identity_sync_complete=None): + first_identity_sync_complete=None, + systemcontroller_gateway_ip=None): with write_session() as session: subcloud_ref = subcloud_get(context, subcloud_id) if management_state is not None: @@ -424,6 +425,9 @@ def subcloud_update(context, subcloud_id, management_state=None, subcloud_ref.group_id = group_id if first_identity_sync_complete is not None: subcloud_ref.first_identity_sync_complete = first_identity_sync_complete + if systemcontroller_gateway_ip is not None: + subcloud_ref.systemcontroller_gateway_ip = \ + systemcontroller_gateway_ip subcloud_ref.save(session) return subcloud_ref diff --git a/distributedcloud/dcmanager/manager/service.py b/distributedcloud/dcmanager/manager/service.py index 322d039da..feaabe1df 100644 --- a/distributedcloud/dcmanager/manager/service.py +++ b/distributedcloud/dcmanager/manager/service.py @@ -197,6 +197,15 @@ class DCManagerService(service.Service): subcloud_id, payload) + @request_context + def subcloud_deploy_bootstrap(self, context, subcloud_id, payload): + # Bootstraps a subcloud + LOG.info("Handling subcloud_deploy_bootstrap request for: %s" % + payload.get('name')) + return self.subcloud_manager.subcloud_deploy_bootstrap(context, + subcloud_id, + payload) + def _stop_rpc_server(self): # Stop RPC connection to prevent new requests LOG.debug(_("Attempting to stop RPC service...")) diff --git a/distributedcloud/dcmanager/manager/subcloud_manager.py b/distributedcloud/dcmanager/manager/subcloud_manager.py index 405a1ac4d..6ea412ceb 100644 --- a/distributedcloud/dcmanager/manager/subcloud_manager.py +++ b/distributedcloud/dcmanager/manager/subcloud_manager.py @@ -60,6 +60,7 @@ from dcmanager.db.sqlalchemy.models import Subcloud from dcmanager.rpc import client as dcmanager_rpc_client from dcorch.rpc import client as dcorch_rpc_client + LOG = logging.getLogger(__name__) # Name of our distributed cloud addn_hosts file for dnsmasq @@ -239,6 +240,7 @@ class SubcloudManager(manager.Manager): software_version if software_version else SW_VERSION] return install_command + # TODO(gherzman): rename compose_apply_command to compose_bootstrap_command def compose_apply_command(self, subcloud_name, ansible_subcloud_inventory_file, software_version=None): @@ -892,6 +894,86 @@ class SubcloudManager(manager.Manager): deploy_status=consts.DEPLOY_STATE_CREATE_FAILED) return db_api.subcloud_db_model_to_dict(subcloud) + def subcloud_deploy_bootstrap(self, context, subcloud_id, payload): + """Bootstrap subcloud + + :param context: request context object + :param subcloud_id: subcloud_id from db + :param payload: subcloud bootstrap configuration + """ + LOG.info("Bootstrapping subcloud %s." % payload['name']) + + try: + subcloud = db_api.subcloud_get(context, subcloud_id) + + management_subnet = utils.get_management_subnet(payload) + sys_controller_gw_ip = payload.get( + "systemcontroller_gateway_address") + + if (management_subnet != subcloud.management_subnet) or ( + sys_controller_gw_ip != subcloud.systemcontroller_gateway_ip): + m_ks_client = OpenStackDriver( + region_name=dccommon_consts.DEFAULT_REGION_NAME, + region_clients=None).keystone_client + # Create a new route + self._create_subcloud_route(payload, m_ks_client, + sys_controller_gw_ip) + # Delete previous route + self._delete_subcloud_routes(m_ks_client, subcloud) + # Update endpoints + self._update_services_endpoint(context, payload, subcloud.name, + m_ks_client) + + # Update subcloud + subcloud = db_api.subcloud_update( + context, + subcloud.id, + description=payload.get("description", None), + management_subnet=utils.get_management_subnet(payload), + management_gateway_ip=utils.get_management_gateway_address( + payload), + management_start_ip=utils.get_management_start_address( + payload), + management_end_ip=utils.get_management_end_address(payload), + systemcontroller_gateway_ip=payload.get( + "systemcontroller_gateway_address", None), + location=payload.get("location", None), + deploy_status=consts.DEPLOY_STATE_PRE_BOOTSTRAP) + + # Populate payload with passwords + payload['ansible_become_pass'] = payload['sysadmin_password'] + payload['ansible_ssh_pass'] = payload['sysadmin_password'] + payload['admin_password'] = str(keyring.get_password('CGCS', + 'admin')) + del payload['sysadmin_password'] + + # Update the ansible overrides file + overrides_file = os.path.join(consts.ANSIBLE_OVERRIDES_PATH, + subcloud.name + '.yml') + utils.update_values_on_yaml_file(overrides_file, payload) + + # Ansible inventory filename for the specified subcloud + ansible_subcloud_inventory_file = utils.get_ansible_filename( + subcloud.name, INVENTORY_FILE_POSTFIX) + + # Update the ansible inventory for the subcloud + utils.create_subcloud_inventory(payload, + ansible_subcloud_inventory_file) + + apply_command = self.compose_apply_command( + subcloud.name, + ansible_subcloud_inventory_file, + subcloud.software_version) + + self.run_deploy_commands(subcloud, payload, context, + apply_command=apply_command) + + except Exception: + LOG.exception("Failed to bootstrap subcloud %s" % payload['name']) + db_api.subcloud_update( + context, subcloud_id, + deploy_status=consts.DEPLOY_STATE_PRE_BOOTSTRAP_FAILED) + def _subcloud_operation_notice( self, operation, restore_subclouds, failed_subclouds, invalid_subclouds): @@ -1529,6 +1611,22 @@ class SubcloudManager(manager.Manager): deploy_status=consts.DEPLOY_STATE_DONE, error_description=consts.ERROR_DESC_EMPTY) + def run_deploy_commands(self, subcloud, payload, context, + install_command=None, apply_command=None, + deploy_command=None, rehome_command=None, + network_reconfig=None): + try: + log_file = ( + os.path.join(consts.DC_ANSIBLE_LOG_DIR, subcloud.name) + + "_playbook_output.log" + ) + if apply_command: + self._run_subcloud_bootstrap(context, subcloud, + apply_command, log_file) + except Exception as ex: + LOG.exception("run_deploy failed") + raise ex + @staticmethod def _run_subcloud_install( context, subcloud, install_command, log_file, payload): @@ -1574,6 +1672,35 @@ class SubcloudManager(manager.Manager): LOG.info("Successfully installed %s" % subcloud.name) return True + def _run_subcloud_bootstrap(self, context, subcloud, + apply_command, log_file): + # Update the subcloud deploy_status to bootstrapping + db_api.subcloud_update( + context, subcloud.id, + deploy_status=consts.DEPLOY_STATE_BOOTSTRAPPING, + error_description=consts.ERROR_DESC_EMPTY) + + # Run the ansible subcloud boostrap playbook + LOG.info("Starting bootstrap of %s" % subcloud.name) + try: + run_playbook(log_file, apply_command) + except PlaybookExecutionFailed: + msg = utils.find_ansible_error_msg( + subcloud.name, log_file, consts.DEPLOY_STATE_BOOTSTRAPPING) + LOG.error(msg) + db_api.subcloud_update( + context, subcloud.id, + deploy_status=consts.DEPLOY_STATE_BOOTSTRAP_FAILED, + error_description=msg[0:consts.ERROR_DESCRIPTION_LENGTH]) + return + + db_api.subcloud_update( + context, subcloud.id, + deploy_status=consts.DEPLOY_STATE_BOOTSTRAPPED, + error_description=consts.ERROR_DESC_EMPTY) + + LOG.info("Successfully bootstrapped %s" % subcloud.name) + def _create_addn_hosts_dc(self, context): """Generate the addn_hosts_dc file for hostname/ip translation""" @@ -2002,7 +2129,8 @@ class SubcloudManager(manager.Manager): m_ks_client = OpenStackDriver( region_name=dccommon_consts.DEFAULT_REGION_NAME, region_clients=None).keystone_client - self._create_subcloud_route(payload, m_ks_client, subcloud) + self._create_subcloud_route(payload, m_ks_client, + subcloud.systemcontroller_gateway_ip) except HTTPConflict: # The route already exists LOG.warning( @@ -2031,7 +2159,8 @@ class SubcloudManager(manager.Manager): # Delete old routes self._delete_subcloud_routes(m_ks_client, subcloud) - def _create_subcloud_route(self, payload, keystone_client, subcloud): + def _create_subcloud_route(self, payload, keystone_client, + systemcontroller_gateway_ip): subcloud_subnet = netaddr.IPNetwork(utils.get_management_subnet(payload)) endpoint = keystone_client.endpoint_cache.get_endpoint('sysinv') sysinv_client = SysinvClient(dccommon_consts.DEFAULT_REGION_NAME, @@ -2043,7 +2172,7 @@ class SubcloudManager(manager.Manager): sysinv_client.create_route(mgmt_if_uuid, str(subcloud_subnet.ip), subcloud_subnet.prefixlen, - subcloud.systemcontroller_gateway_ip, + systemcontroller_gateway_ip, 1) def _update_services_endpoint( diff --git a/distributedcloud/dcmanager/rpc/client.py b/distributedcloud/dcmanager/rpc/client.py index a430a7c43..fc0b029f2 100644 --- a/distributedcloud/dcmanager/rpc/client.py +++ b/distributedcloud/dcmanager/rpc/client.py @@ -192,6 +192,11 @@ class ManagerClient(RPCClient): subcloud_id=subcloud_id, payload=payload)) + def subcloud_deploy_bootstrap(self, ctxt, subcloud_id, payload): + return self.cast(ctxt, self.make_msg('subcloud_deploy_bootstrap', + subcloud_id=subcloud_id, + payload=payload)) + class DCManagerNotifications(RPCClient): """DC Manager Notification interface to broadcast subcloud state changed diff --git a/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_phased_subcloud_deploy.py b/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_phased_subcloud_deploy.py index 2d7a2562e..5f3574ce8 100644 --- a/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_phased_subcloud_deploy.py +++ b/distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_phased_subcloud_deploy.py @@ -4,12 +4,32 @@ # SPDX-License-Identifier: Apache-2.0 # +import copy +import json import mock +from os import path as os_path +import six +import webtest +from dcmanager.common import consts from dcmanager.common import phased_subcloud_deploy as psd_common +from dcmanager.common import utils as dutils from dcmanager.db import api as db_api +from dcmanager.rpc import client as rpc_client +from dcmanager.tests.unit.api import test_root_controller as testroot +from dcmanager.tests.unit.api.v1.controllers.test_subclouds import \ + FakeAddressPool from dcmanager.tests.unit.api.v1.controllers.test_subclouds import \ TestSubcloudPost +from dcmanager.tests.unit.common import fake_subcloud +from dcmanager.tests import utils + +FAKE_URL = '/v1.0/phased-subcloud-deploy' + +FAKE_TENANT = utils.UUID1 + +FAKE_HEADERS = {'X-Tenant-Id': FAKE_TENANT, 'X_ROLE': 'admin,member,reader', + 'X-Identity-Status': 'Confirmed', 'X-Project-Name': 'admin'} class FakeRPCClient(object): @@ -56,3 +76,156 @@ class TestSubcloudDeployCreate(TestSubcloudPost): headers=self.get_api_headers(), expect_errors=True) self._verify_post_failure(response, "bootstrap-address", None) + + +class TestSubcloudDeployBootstrap(testroot.DCManagerApiTest): + def setUp(self): + super().setUp() + self.ctx = utils.dummy_context() + + p = mock.patch.object(rpc_client, 'ManagerClient') + self.mock_rpc_client = p.start() + self.addCleanup(p.stop) + + self.management_address_pool = FakeAddressPool('192.168.204.0', 24, + '192.168.204.2', + '192.168.204.100') + + p = mock.patch.object(psd_common, 'get_network_address_pool') + self.mock_get_network_address_pool = p.start() + self.mock_get_network_address_pool.return_value = \ + self.management_address_pool + self.addCleanup(p.stop) + + p = mock.patch.object(psd_common, 'get_ks_client') + self.mock_get_ks_client = p.start() + self.addCleanup(p.stop) + + p = mock.patch.object(psd_common.PatchingClient, 'query') + self.mock_query = p.start() + self.addCleanup(p.stop) + + @mock.patch.object(dutils, 'load_yaml_file') + @mock.patch.object(os_path, 'exists') + def test_subcloud_bootstrap(self, mock_path_exists, mock_load_yaml): + mock_path_exists.side_effect = [False, False, False, False, True] + mock_load_yaml.return_value = { + "software_version": fake_subcloud.FAKE_SOFTWARE_VERSION} + + subcloud = fake_subcloud.create_fake_subcloud( + self.ctx, + name=fake_subcloud.FAKE_BOOTSTRAP_FILE_DATA["name"], + deploy_status=consts.DEPLOY_STATE_INSTALLED) + + fake_content = json.dumps( + fake_subcloud.FAKE_BOOTSTRAP_FILE_DATA).encode("utf-8") + response = self.app.patch( + FAKE_URL + '/' + str(subcloud.id) + '/bootstrap', + headers=FAKE_HEADERS, + params=fake_subcloud.FAKE_BOOTSTRAP_VALUE, + upload_files=[("bootstrap_values", + "bootstrap_fake_filename", + fake_content)]) + + self.assertEqual(response.status_int, 200) + self.mock_rpc_client.return_value.subcloud_deploy_bootstrap.\ + assert_called_once() + + expected_payload = {**fake_subcloud.FAKE_BOOTSTRAP_VALUE, + **fake_subcloud.FAKE_BOOTSTRAP_FILE_DATA} + expected_payload["sysadmin_password"] = "testpass" + expected_payload["software_version"] = \ + fake_subcloud.FAKE_SOFTWARE_VERSION + + (_, res_subcloud_id, res_payload), _ = self.mock_rpc_client.\ + return_value.subcloud_deploy_bootstrap.call_args + + self.assertDictEqual(res_payload, expected_payload) + self.assertEqual(res_subcloud_id, subcloud.id) + + def test_subcloud_bootstrap_no_body(self): + subcloud = fake_subcloud.create_fake_subcloud( + self.ctx, + name=fake_subcloud.FAKE_BOOTSTRAP_FILE_DATA["name"], + deploy_status=consts.DEPLOY_STATE_INSTALLED) + url = FAKE_URL + '/' + str(subcloud.id) + '/bootstrap' + six.assertRaisesRegex(self, webtest.app.AppError, "400 *", + self.app.patch_json, url, + headers=FAKE_HEADERS, params={}) + + def test_subcloud_bootstrap_subcloud_not_found(self): + url = FAKE_URL + '/' + "nonexistent_subcloud" + '/bootstrap' + six.assertRaisesRegex(self, webtest.app.AppError, "404 *", + self.app.patch_json, url, + headers=FAKE_HEADERS, params={}) + + @mock.patch.object(dutils, 'load_yaml_file') + @mock.patch.object(os_path, 'exists') + def test_subcloud_bootstrap_no_bootstrap_values_on_request( + self, mock_path_exists, mock_load_yaml_file): + mock_path_exists.side_effect = [False, False, False, False, True] + fake_bootstrap_values = copy.copy( + fake_subcloud.FAKE_BOOTSTRAP_FILE_DATA) + fake_bootstrap_values["software_version"] = \ + fake_subcloud.FAKE_SOFTWARE_VERSION + mock_load_yaml_file.return_value = \ + fake_bootstrap_values + + subcloud = fake_subcloud.create_fake_subcloud( + self.ctx, + name=fake_subcloud.FAKE_BOOTSTRAP_FILE_DATA["name"], + deploy_status=consts.DEPLOY_STATE_INSTALLED) + + response = self.app.patch( + FAKE_URL + '/' + str(subcloud.id) + '/bootstrap', + headers=FAKE_HEADERS, + params=fake_subcloud.FAKE_BOOTSTRAP_VALUE) + + self.assertEqual(response.status_int, 200) + self.mock_rpc_client.return_value.subcloud_deploy_bootstrap.\ + assert_called_once() + + expected_payload = {**fake_subcloud.FAKE_BOOTSTRAP_VALUE, + **fake_subcloud.FAKE_BOOTSTRAP_FILE_DATA} + expected_payload["sysadmin_password"] = "testpass" + expected_payload["software_version"] = \ + fake_subcloud.FAKE_SOFTWARE_VERSION + + (_, res_subcloud_id, res_payload), _ = self.mock_rpc_client.\ + return_value.subcloud_deploy_bootstrap.call_args + + self.assertDictEqual(res_payload, expected_payload) + self.assertEqual(res_subcloud_id, subcloud.id) + + def test_subcloud_bootstrap_management_subnet_conflict(self): + conflicting_subnet = { + "management_subnet": "192.168.102.0/24", + "management_start_ip": "192.168.102.2", + "management_end_ip": "192.168.102.50", + "management_gateway_ip": "192.168.102.1"} + + fake_subcloud.create_fake_subcloud( + self.ctx, + name="existing_subcloud", + deploy_status=consts.DEPLOY_STATE_DONE, + **conflicting_subnet + ) + + subcloud = fake_subcloud.create_fake_subcloud( + self.ctx, + name=fake_subcloud.FAKE_BOOTSTRAP_FILE_DATA["name"], + deploy_status=consts.DEPLOY_STATE_INSTALLED) + + modified_bootstrap_data = copy.copy( + fake_subcloud.FAKE_BOOTSTRAP_FILE_DATA) + modified_bootstrap_data.update(conflicting_subnet) + + fake_content = json.dumps(modified_bootstrap_data).encode("utf-8") + url = FAKE_URL + '/' + str(subcloud.id) + '/bootstrap' + six.assertRaisesRegex(self, webtest.app.AppError, "400 *", + self.app.patch, url, + headers=FAKE_HEADERS, + params=fake_subcloud.FAKE_BOOTSTRAP_VALUE, + upload_files=[("bootstrap_values", + "bootstrap_fake_filename", + fake_content)]) diff --git a/distributedcloud/dcmanager/tests/unit/common/fake_subcloud.py b/distributedcloud/dcmanager/tests/unit/common/fake_subcloud.py index 6792354a9..aac9d8cf3 100644 --- a/distributedcloud/dcmanager/tests/unit/common/fake_subcloud.py +++ b/distributedcloud/dcmanager/tests/unit/common/fake_subcloud.py @@ -61,6 +61,19 @@ FAKE_SUBCLOUD_BOOTSTRAP_PAYLOAD = { (base64.b64encode('testpass'.encode("utf-8"))).decode('ascii'), } +FAKE_BOOTSTRAP_FILE_DATA = { + "system_mode": "simplex", + "name": "fake subcloud1", + "management_subnet": "192.168.101.0/24", + "management_start_address": "192.168.101.2", + "management_end_address": "192.168.101.50", + "management_gateway_address": "192.168.101.1", + "external_oam_subnet": "10.10.10.0/24", + "external_oam_gateway_address": "10.10.10.1", + "external_oam_floating_address": "10.10.10.12", + "systemcontroller_gateway_address": "192.168.204.101", +} + FAKE_SUBCLOUD_INSTALL_VALUES = { "image": "http://192.168.101.2:8080/iso/bootimage.iso", "software_version": FAKE_SOFTWARE_VERSION, diff --git a/distributedcloud/dcmanager/tests/unit/manager/test_subcloud_manager.py b/distributedcloud/dcmanager/tests/unit/manager/test_subcloud_manager.py index 617da42c4..321241f16 100644 --- a/distributedcloud/dcmanager/tests/unit/manager/test_subcloud_manager.py +++ b/distributedcloud/dcmanager/tests/unit/manager/test_subcloud_manager.py @@ -32,6 +32,7 @@ from dcmanager.common import consts from dcmanager.common import exceptions from dcmanager.common import prestage from dcmanager.common import utils as cutils +from dcmanager.db import api as dc_db_api from dcmanager.db.sqlalchemy import api as db_api from dcmanager.manager import subcloud_manager from dcmanager.state import subcloud_state_manager @@ -503,6 +504,59 @@ class TestSubcloudManager(base.DCManagerTestCase): self.assertEqual(consts.DEPLOY_STATE_CREATE_FAILED, updated_subcloud.deploy_status) + @mock.patch.object(cutils, 'create_subcloud_inventory') + @mock.patch.object(subcloud_manager, 'keyring') + @mock.patch.object(cutils, 'get_playbook_for_software_version') + @mock.patch.object(cutils, 'update_values_on_yaml_file') + @mock.patch.object(subcloud_manager, 'run_playbook') + def test_subcloud_deploy_bootstrap(self, mock_run_playbook, mock_update_yml, + mock_get_playbook_for_software_version, + mock_keyring, create_subcloud_inventory): + mock_get_playbook_for_software_version.return_value = "22.12" + mock_keyring.get_password.return_value = "testpass" + + subcloud = fake_subcloud.create_fake_subcloud( + self.ctx, + name=fake_subcloud.FAKE_BOOTSTRAP_FILE_DATA["name"], + deploy_status=consts.DEPLOY_STATE_INSTALLED) + + payload = {**fake_subcloud.FAKE_BOOTSTRAP_VALUE, + **fake_subcloud.FAKE_BOOTSTRAP_FILE_DATA} + payload["sysadmin_password"] = "testpass" + + sm = subcloud_manager.SubcloudManager() + sm.subcloud_deploy_bootstrap(self.ctx, subcloud.id, payload) + + mock_run_playbook.assert_called_once() + + # Verify subcloud was updated with correct values + updated_subcloud = db_api.subcloud_get_by_name(self.ctx, + payload['name']) + self.assertEqual(consts.DEPLOY_STATE_BOOTSTRAPPED, + updated_subcloud.deploy_status) + + @mock.patch.object(dc_db_api, 'subcloud_get') + def test_subcloud_deploy_bootstrap_failed(self, mock_subcloud_get): + mock_subcloud_get.side_effect = FakeException('boom') + + subcloud = fake_subcloud.create_fake_subcloud( + self.ctx, + name=fake_subcloud.FAKE_BOOTSTRAP_FILE_DATA["name"], + deploy_status=consts.DEPLOY_STATE_INSTALLED) + + payload = {**fake_subcloud.FAKE_BOOTSTRAP_VALUE, + **fake_subcloud.FAKE_BOOTSTRAP_FILE_DATA} + payload["sysadmin_password"] = "testpass" + + sm = subcloud_manager.SubcloudManager() + sm.subcloud_deploy_bootstrap(self.ctx, subcloud.id, payload) + + # Verify subcloud was updated with correct values + updated_subcloud = db_api.subcloud_get_by_name(self.ctx, + payload['name']) + self.assertEqual(consts.DEPLOY_STATE_PRE_BOOTSTRAP_FAILED, + updated_subcloud.deploy_status) + @mock.patch.object(subcloud_manager.SubcloudManager, 'compose_apply_command') @mock.patch.object(subcloud_manager.SubcloudManager,