diff --git a/distributedcloud/__init__.py b/distributedcloud/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/distributedcloud/dccommon/consts.py b/distributedcloud/dccommon/consts.py index cfaa78267..87653299e 100644 --- a/distributedcloud/dccommon/consts.py +++ b/distributedcloud/dccommon/consts.py @@ -33,6 +33,7 @@ CLOUD_0 = "RegionOne" VIRTUAL_MASTER_CLOUD = "SystemController" SW_UPDATE_DEFAULT_TITLE = "all clouds default" +LOAD_VAULT_DIR = '/opt/dc-vault/loads' USER_HEADER_VALUE = "distcloud" USER_HEADER = {'User-Header': USER_HEADER_VALUE} @@ -41,3 +42,4 @@ ADMIN_USER_NAME = "admin" ADMIN_PROJECT_NAME = "admin" SYSINV_USER_NAME = "sysinv" DCMANAGER_USER_NAME = "dcmanager" +SERVICES_USER_NAME = "services" diff --git a/distributedcloud/dccommon/drivers/openstack/barbican.py b/distributedcloud/dccommon/drivers/openstack/barbican.py new file mode 100644 index 000000000..09b91820c --- /dev/null +++ b/distributedcloud/dccommon/drivers/openstack/barbican.py @@ -0,0 +1,75 @@ +# Copyright 2016 Ericsson AB + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Copyright (c) 2020 Wind River Systems, Inc. +# +# The right to copy, distribute, modify, or otherwise make use +# of this software may be licensed only pursuant to the terms +# of an applicable Wind River license agreement. +# + + +from barbicanclient import client + +from oslo_log import log + +from dccommon import consts as dccommon_consts +from dccommon.drivers import base +from dccommon import exceptions + + +LOG = log.getLogger(__name__) +API_VERSION = 'v1' + + +class BarbicanClient(base.DriverBase): + """Barbican driver. + + The session needs to be associated with synchronized 'services' project + in order for the client to get the host bmc password. + """ + + def __init__( + self, region, session, endpoint_type=dccommon_consts.KS_ENDPOINT_DEFAULT): + + try: + self.barbican_client = client.Client( + API_VERSION, + session=session, + region_name=region, + interface=endpoint_type) + + self.region_name = region + except exceptions.ServiceUnavailable: + raise + + def get_host_bmc_password(self, host_uuid): + """Get the Board Management Controller password corresponding to the host + + :param host_uuid The host uuid + """ + + secrets = self.barbican_client.secrets.list() + for secret in secrets: + if secret.name == host_uuid: + secret_ref = secret.secret_ref + break + else: + return + + secret = self.barbican_client.secrets.get(secret_ref) + + bmc_password = secret.payload + + return bmc_password diff --git a/distributedcloud/dccommon/drivers/openstack/sdk_platform.py b/distributedcloud/dccommon/drivers/openstack/sdk_platform.py index 9bbb8010c..2f53a20e2 100644 --- a/distributedcloud/dccommon/drivers/openstack/sdk_platform.py +++ b/distributedcloud/dccommon/drivers/openstack/sdk_platform.py @@ -23,6 +23,7 @@ from oslo_log import log from oslo_utils import timeutils from dccommon import consts +from dccommon.drivers.openstack.barbican import BarbicanClient from dccommon.drivers.openstack.fm import FmClient from dccommon.drivers.openstack.keystone_v3 import KeystoneClient from dccommon.drivers.openstack.sysinv_v1 import SysinvClient @@ -39,6 +40,7 @@ STALE_TOKEN_DURATION_STEP = 20 KEYSTONE_CLIENT_NAME = 'keystone' SYSINV_CLIENT_NAME = 'sysinv' FM_CLIENT_NAME = 'fm' +BARBICAN_CLIENT_NAME = 'barbican' LOG = log.getLogger(__name__) @@ -46,13 +48,15 @@ LOCK_NAME = 'dc-openstackdriver-platform' SUPPORTED_REGION_CLIENTS = [ SYSINV_CLIENT_NAME, - FM_CLIENT_NAME + FM_CLIENT_NAME, + BARBICAN_CLIENT_NAME, ] # region client type and class mappings region_client_class_map = { SYSINV_CLIENT_NAME: SysinvClient, FM_CLIENT_NAME: FmClient, + BARBICAN_CLIENT_NAME: BarbicanClient, } @@ -68,6 +72,7 @@ class OpenStackDriver(object): self.keystone_client = None self.sysinv_client = None self.fm_client = None + self.barbican_client = None if region_clients: # check if the requested clients are in the supported client list diff --git a/distributedcloud/dccommon/drivers/openstack/sysinv_v1.py b/distributedcloud/dccommon/drivers/openstack/sysinv_v1.py index 774fd4c83..1cd714775 100644 --- a/distributedcloud/dccommon/drivers/openstack/sysinv_v1.py +++ b/distributedcloud/dccommon/drivers/openstack/sysinv_v1.py @@ -130,6 +130,39 @@ class SysinvClient(base.DriverBase): action_value = 'unlock' return self._do_host_action(host_id, action_value) + def configure_bmc_host(self, + host_id, + bm_username, + bm_ip, + bm_password, + bm_type='ipmi'): + """Configure bmc of a host""" + patch = [ + {'op': 'replace', + 'path': '/bm_username', + 'value': bm_username}, + {'op': 'replace', + 'path': '/bm_ip', + 'value': bm_ip}, + {'op': 'replace', + 'path': '/bm_password', + 'value': bm_password}, + {'op': 'replace', + 'path': '/bm_type', + 'value': bm_type}, + ] + return self.sysinv_client.ihost.update(host_id, patch) + + def power_on_host(self, host_id): + """Power on a host""" + action_value = 'power-on' + return self._do_host_action(host_id, action_value) + + def power_off_host(self, host_id): + """Power off a host""" + action_value = 'power-off' + return self._do_host_action(host_id, action_value) + def get_management_interface(self, hostname): """Get the management interface for a host.""" interfaces = self.sysinv_client.iinterface.list(hostname) diff --git a/distributedcloud/dcmanager/common/install_consts.py b/distributedcloud/dccommon/install_consts.py similarity index 91% rename from distributedcloud/dcmanager/common/install_consts.py rename to distributedcloud/dccommon/install_consts.py index da6ea5f1d..ec2502856 100644 --- a/distributedcloud/dcmanager/common/install_consts.py +++ b/distributedcloud/dccommon/install_consts.py @@ -31,3 +31,6 @@ MANDATORY_INSTALL_VALUES = [ 'bmc_password', 'install_type' ] + +ANSIBLE_SUBCLOUD_INSTALL_PLAYBOOK = \ + '/usr/share/ansible/stx-ansible/playbooks/install.yml' diff --git a/distributedcloud/dcmanager/manager/subcloud_install.py b/distributedcloud/dccommon/subcloud_install.py similarity index 98% rename from distributedcloud/dcmanager/manager/subcloud_install.py rename to distributedcloud/dccommon/subcloud_install.py index 9cd154fc2..d7d170e11 100644 --- a/distributedcloud/dcmanager/manager/subcloud_install.py +++ b/distributedcloud/dccommon/subcloud_install.py @@ -24,19 +24,17 @@ from eventlet.green import subprocess import json import netaddr import os -import socket - +from oslo_log import log as logging from six.moves.urllib import error as urllib_error from six.moves.urllib import parse from six.moves.urllib import request +import socket +from dccommon import consts from dccommon.drivers.openstack.keystone_v3 import KeystoneClient from dccommon.drivers.openstack.sysinv_v1 import SysinvClient -from dcmanager.common import consts -from dcmanager.common import exceptions -from dcmanager.common import install_consts - -from oslo_log import log as logging +from dccommon import exceptions +from dccommon import install_consts LOG = logging.getLogger(__name__) @@ -88,7 +86,7 @@ class SubcloudInstall(object): ks_client = KeystoneClient() session = ks_client.endpoint_cache.get_session_from_token( context.auth_token, context.project) - self.sysinv_client = SysinvClient(consts.DEFAULT_REGION_NAME, session) + self.sysinv_client = SysinvClient(consts.CLOUD_0, session) self.name = subcloud_name self.input_iso = None self.www_root = None @@ -288,14 +286,14 @@ class SubcloudInstall(object): msg = "Error: Downloading file %s may be interrupted: %s" % ( values['image'], e) LOG.error(msg) - raise exceptions.DCManagerException( + raise exceptions.DCCommonException( resource=self.name, msg=msg) except Exception as e: msg = "Error: Could not download file %s: %s" % ( values['image'], e) LOG.error(msg) - raise exceptions.DCManagerException( + raise exceptions.DCCommonException( resource=self.name, msg=msg) diff --git a/distributedcloud/dcmanager/api/controllers/v1/subclouds.py b/distributedcloud/dcmanager/api/controllers/v1/subclouds.py index 8abf09d54..b800c67c7 100644 --- a/distributedcloud/dcmanager/api/controllers/v1/subclouds.py +++ b/distributedcloud/dcmanager/api/controllers/v1/subclouds.py @@ -21,6 +21,7 @@ from requests_toolbelt.multipart import decoder import base64 +import json import keyring from netaddr import AddrFormatError from netaddr import IPAddress @@ -39,6 +40,7 @@ from pecan import request from dccommon.drivers.openstack.keystone_v3 import KeystoneClient from dccommon.drivers.openstack.sysinv_v1 import SysinvClient from dccommon import exceptions as dccommon_exceptions +from dccommon import install_consts from keystoneauth1 import exceptions as keystone_exceptions @@ -48,7 +50,6 @@ from dcmanager.api.controllers import restcomm from dcmanager.common import consts from dcmanager.common import exceptions from dcmanager.common.i18n import _ -from dcmanager.common import install_consts from dcmanager.common import utils from dcmanager.db import api as db_api @@ -495,6 +496,10 @@ class SubcloudsController(object): # if group_id has been omitted from payload, use 'Default'. group_id = payload.get('group_id', consts.DEFAULT_SUBCLOUD_GROUP_ID) + data_install = None + if 'install_values' in payload: + data_install = json.dumps(payload['install_values']) + subcloud = db_api.subcloud_create( context, payload['name'], @@ -508,7 +513,8 @@ class SubcloudsController(object): payload['systemcontroller_gateway_address'], consts.DEPLOY_STATE_NONE, False, - group_id) + group_id, + data_install=data_install) return subcloud @index.when(method='GET', template='json') diff --git a/distributedcloud/dcmanager/common/consts.py b/distributedcloud/dcmanager/common/consts.py index d6455d7a6..6714d17ed 100644 --- a/distributedcloud/dcmanager/common/consts.py +++ b/distributedcloud/dcmanager/common/consts.py @@ -123,6 +123,7 @@ DEPLOY_STATE_PRE_INSTALL = 'pre-install' DEPLOY_STATE_PRE_INSTALL_FAILED = 'pre-install-failed' DEPLOY_STATE_INSTALLING = 'installing' DEPLOY_STATE_INSTALL_FAILED = 'install-failed' +DEPLOY_STATE_INSTALLED = 'installed' DEPLOY_STATE_BOOTSTRAPPING = 'bootstrapping' DEPLOY_STATE_BOOTSTRAP_FAILED = 'bootstrap-failed' DEPLOY_STATE_DEPLOYING = 'deploying' @@ -147,3 +148,12 @@ DEPLOY_COMMON_FILE_OPTIONS = [ DEPLOY_OVERRIDES, DEPLOY_CHART ] + + +DC_LOG_DIR = '/var/log/dcmanager/' +INVENTORY_FILE_POSTFIX = '_inventory.yml' + +# The following password is just a temporary and internal password that is used +# after a remote install as part of the upgrade. The real sysadmin password +# will be restored af the subcloud is re-managed at the end of the upgrade. +TEMP_SYSADMIN_PASSWORD = 'St8rlingX*' diff --git a/distributedcloud/dcmanager/common/utils.py b/distributedcloud/dcmanager/common/utils.py index 0175e9fb4..0ed005ca1 100644 --- a/distributedcloud/dcmanager/common/utils.py +++ b/distributedcloud/dcmanager/common/utils.py @@ -207,3 +207,30 @@ def get_filename_by_prefix(dir_path, prefix): if filename.startswith(prefix): return filename return None + + +def create_subcloud_inventory(subcloud, inventory_file): + """Create the ansible inventory file for the specified subcloud""" + + # Delete the file if it already exists + delete_subcloud_inventory(inventory_file) + + with open(inventory_file, 'w') as f_out_inventory: + f_out_inventory.write( + '---\n' + 'all:\n' + ' vars:\n' + ' ansible_ssh_user: sysadmin\n' + ' hosts:\n' + ' ' + subcloud['name'] + ':\n' + ' ansible_host: ' + + subcloud['bootstrap-address'] + '\n' + ) + + +def delete_subcloud_inventory(inventory_file): + """Delete the ansible inventory file for the specified subcloud""" + + # Delete the file if it exists + if os.path.isfile(inventory_file): + os.remove(inventory_file) diff --git a/distributedcloud/dcmanager/db/api.py b/distributedcloud/dcmanager/db/api.py index 1e204defa..cd90981a7 100644 --- a/distributedcloud/dcmanager/db/api.py +++ b/distributedcloud/dcmanager/db/api.py @@ -66,6 +66,8 @@ def subcloud_db_model_to_dict(subcloud): "openstack-installed": subcloud.openstack_installed, "systemcontroller-gateway-ip": subcloud.systemcontroller_gateway_ip, + "data_install": subcloud.data_install, + "data_upgrade": subcloud.data_upgrade, "created-at": subcloud.created_at, "updated-at": subcloud.updated_at, "group_id": subcloud.group_id} @@ -76,14 +78,14 @@ def subcloud_create(context, name, description, location, software_version, management_subnet, management_gateway_ip, management_start_ip, management_end_ip, systemcontroller_gateway_ip, deploy_status, - openstack_installed, group_id): + openstack_installed, group_id, data_install=None): """Create a subcloud.""" return IMPL.subcloud_create(context, name, description, location, software_version, management_subnet, management_gateway_ip, management_start_ip, management_end_ip, systemcontroller_gateway_ip, deploy_status, - openstack_installed, group_id) + openstack_installed, group_id, data_install) def subcloud_get(context, subcloud_id): @@ -115,12 +117,13 @@ def subcloud_update(context, subcloud_id, management_state=None, availability_status=None, software_version=None, description=None, location=None, audit_fail_count=None, deploy_status=None, openstack_installed=None, - group_id=None): + group_id=None, data_install=None, data_upgrade=None): """Update a subcloud or raise if it does not exist.""" return IMPL.subcloud_update(context, subcloud_id, management_state, availability_status, software_version, description, location, audit_fail_count, - deploy_status, openstack_installed, group_id) + deploy_status, openstack_installed, group_id, + data_install, data_upgrade) def subcloud_destroy(context, subcloud_id): diff --git a/distributedcloud/dcmanager/db/sqlalchemy/api.py b/distributedcloud/dcmanager/db/sqlalchemy/api.py index 7d604452d..851ede1aa 100644 --- a/distributedcloud/dcmanager/db/sqlalchemy/api.py +++ b/distributedcloud/dcmanager/db/sqlalchemy/api.py @@ -213,7 +213,7 @@ def subcloud_create(context, name, description, location, software_version, management_subnet, management_gateway_ip, management_start_ip, management_end_ip, systemcontroller_gateway_ip, deploy_status, - openstack_installed, group_id): + openstack_installed, group_id, data_install=None): with write_session() as session: subcloud_ref = models.Subcloud() subcloud_ref.name = name @@ -231,6 +231,8 @@ def subcloud_create(context, name, description, location, software_version, subcloud_ref.audit_fail_count = 0 subcloud_ref.openstack_installed = openstack_installed subcloud_ref.group_id = group_id + if data_install is not None: + subcloud_ref.data_install = data_install session.add(subcloud_ref) return subcloud_ref @@ -239,8 +241,11 @@ def subcloud_create(context, name, description, location, software_version, def subcloud_update(context, subcloud_id, management_state=None, availability_status=None, software_version=None, description=None, location=None, audit_fail_count=None, - deploy_status=None, openstack_installed=None, - group_id=None): + deploy_status=None, + openstack_installed=None, + group_id=None, + data_install=None, + data_upgrade=None): with write_session() as session: subcloud_ref = subcloud_get(context, subcloud_id) if management_state is not None: @@ -255,8 +260,12 @@ def subcloud_update(context, subcloud_id, management_state=None, subcloud_ref.location = location if audit_fail_count is not None: subcloud_ref.audit_fail_count = audit_fail_count + if data_install is not None: + subcloud_ref.data_install = data_install if deploy_status is not None: subcloud_ref.deploy_status = deploy_status + if data_upgrade is not None: + subcloud_ref.data_upgrade = data_upgrade if openstack_installed is not None: subcloud_ref.openstack_installed = openstack_installed if group_id is not None: diff --git a/distributedcloud/dcmanager/db/sqlalchemy/migrate_repo/versions/007_add_subcloud_install.py b/distributedcloud/dcmanager/db/sqlalchemy/migrate_repo/versions/007_add_subcloud_install.py new file mode 100644 index 000000000..23aa9d1b7 --- /dev/null +++ b/distributedcloud/dcmanager/db/sqlalchemy/migrate_repo/versions/007_add_subcloud_install.py @@ -0,0 +1,40 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Copyright (c) 2020 Wind River Systems, Inc. +# +# The right to copy, distribute, modify, or otherwise make use +# of this software may be licensed only pursuant to the terms +# of an applicable Wind River license agreement. +# + +from sqlalchemy import Column +from sqlalchemy import MetaData +from sqlalchemy import Table +from sqlalchemy import Text + + +def upgrade(migrate_engine): + meta = MetaData() + meta.bind = migrate_engine + + subclouds = Table('subclouds', meta, autoload=True) + + # Add the 'data_install' to persist data_install data + subclouds.create_column(Column('data_install', Text)) + + # Add the data_upgrade which persist over an upgrade + subclouds.create_column(Column('data_upgrade', Text)) + + +def downgrade(migrate_engine): + raise NotImplementedError('Database downgrade is unsupported.') diff --git a/distributedcloud/dcmanager/db/sqlalchemy/models.py b/distributedcloud/dcmanager/db/sqlalchemy/models.py index 76e340839..306bb9e4a 100644 --- a/distributedcloud/dcmanager/db/sqlalchemy/models.py +++ b/distributedcloud/dcmanager/db/sqlalchemy/models.py @@ -99,7 +99,9 @@ class Subcloud(BASE, DCManagerBase): software_version = Column(String(255)) management_state = Column(String(255)) availability_status = Column(String(255)) + data_install = Column(String()) deploy_status = Column(String(255)) + data_upgrade = Column(String()) management_subnet = Column(String(255)) management_gateway_ip = Column(String(255)) management_start_ip = Column(String(255), unique=True) @@ -107,6 +109,7 @@ class Subcloud(BASE, DCManagerBase): openstack_installed = Column(Boolean, nullable=False, default=False) systemcontroller_gateway_ip = Column(String(255)) audit_fail_count = Column(Integer) + # multiple subclouds can be in a particular group group_id = Column(Integer, ForeignKey('subcloud_group.id')) diff --git a/distributedcloud/dcmanager/manager/states/base.py b/distributedcloud/dcmanager/manager/states/base.py index d22d779d1..3441320d8 100644 --- a/distributedcloud/dcmanager/manager/states/base.py +++ b/distributedcloud/dcmanager/manager/states/base.py @@ -8,9 +8,11 @@ import six from oslo_log import log as logging +from dccommon.drivers.openstack.barbican import BarbicanClient from dccommon.drivers.openstack.sdk_platform import OpenStackDriver from dccommon.drivers.openstack.sysinv_v1 import SysinvClient from dcmanager.common import consts +from dcmanager.common import context LOG = logging.getLogger(__name__) @@ -20,6 +22,7 @@ class BaseState(object): def __init__(self): super(BaseState, self).__init__() + self.context = context.get_admin_context() def debug_log(self, strategy_step, details): LOG.debug("Stage: %s, State: %s, Subcloud: %s, Details: %s" @@ -35,6 +38,13 @@ class BaseState(object): self.get_region_name(strategy_step), details)) + def error_log(self, strategy_step, details): + LOG.error("Stage: %s, State: %s, Subcloud: %s, Details: %s" + % (strategy_step.stage, + strategy_step.state, + self.get_region_name(strategy_step), + details)) + @staticmethod def get_region_name(strategy_step): """Get the region name for a strategy step""" @@ -64,6 +74,13 @@ class BaseState(object): """ return SysinvClient(region_name, session) + @staticmethod + def get_barbican_client(region_name, session): + """construct a barbican client + + """ + return BarbicanClient(region_name, session) + @abc.abstractmethod def perform_state_action(self, strategy_step): """Perform the action for this state on the strategy_step""" diff --git a/distributedcloud/dcmanager/manager/states/upgrade/upgrading_simplex.py b/distributedcloud/dcmanager/manager/states/upgrade/upgrading_simplex.py index e60b8754f..8120b422b 100644 --- a/distributedcloud/dcmanager/manager/states/upgrade/upgrading_simplex.py +++ b/distributedcloud/dcmanager/manager/states/upgrade/upgrading_simplex.py @@ -3,9 +3,22 @@ # # SPDX-License-Identifier: Apache-2.0 # -from oslo_log import log as logging +import json +import keyring +import os +from dccommon.install_consts import ANSIBLE_SUBCLOUD_INSTALL_PLAYBOOK +from dccommon.subcloud_install import SubcloudInstall + +from dcmanager.common import consts +from dcmanager.common.consts import INVENTORY_FILE_POSTFIX +from dcmanager.common import utils +from dcmanager.db import api as db_api from dcmanager.manager.states.base import BaseState +from dcmanager.manager.states.upgrade import utils as upgrade_utils + +from oslo_log import log as logging +from tsconfig.tsconfig import SW_VERSION LOG = logging.getLogger(__name__) @@ -22,9 +35,349 @@ class UpgradingSimplexState(BaseState): Any exceptions raised by this method set the strategy to FAILED Returning normally from this method set the strategy to the next step """ - LOG.warning("UpgradingSimplexState has not been implemented yet.") - # When we return from this method without throwing an exception, the - # state machine can proceed to the next state - LOG.warning("Faking transition to next state") + LOG.info("Performing simplex upgrade for subcloud %s" % + strategy_step.subcloud.name) + + subcloud_sysinv_client = None + subcloud_barbican_client = None + try: + subcloud_ks_client = self.get_keystone_client(strategy_step.subcloud.name) + subcloud_sysinv_client = self.get_sysinv_client( + strategy_step.subcloud.name, + subcloud_ks_client.session) + subcloud_barbican_client = self.get_barbican_client( + strategy_step.subcloud.name, + subcloud_ks_client.session) + except Exception: + # if getting the token times out, the orchestrator may have + # restarted and subcloud may be offline; so will attempt + # to use the persisted values + message = ("Simplex upgrade perform_subcloud_install " + "subcloud %s failed to get subcloud client" % + strategy_step.subcloud.name) + self.error_log(strategy_step, message) + pass + + # Check whether subcloud is already re-installed with N+1 load + target_version = SW_VERSION + if self._check_load_already_active( + target_version, subcloud_sysinv_client): + self.info_log(strategy_step, + "Load:%s already active" % target_version) + return True + + # Check whether subcloud supports redfish, and if not, fail. + # This needs to be inferred from absence of install_values as + # there is currrently no external api to query. + install_values = self.get_subcloud_upgrade_install_values( + strategy_step, subcloud_sysinv_client, subcloud_barbican_client) + + local_ks_client = self.get_keystone_client() + + # Upgrade the subcloud to the install_values image + self.perform_subcloud_install( + strategy_step, local_ks_client.session, install_values) + + def _check_load_already_active(self, target_version, subcloud_sysinv_client): + """Check if the target_version is already active in subcloud""" + + if subcloud_sysinv_client: + current_loads = subcloud_sysinv_client.get_loads() + for load in current_loads: + if (load.software_version == target_version and + load.state == 'active'): + return True + return False + + def get_subcloud_upgrade_install_values( + self, strategy_step, + subcloud_sysinv_client, subcloud_barbican_client): + """Get the data required for the remote subcloud install. + + subcloud data_install are obtained from: + + dcmanager database: + subcloud.subcloud_install_initial::for values which are persisted at subcloud_add time + + INSTALL: (needed for upgrade install) + bootstrap_interface + bootstrap_vlan + bootstrap_address + bootstrap_address_prefix + install_type # could also be from host-show + + # This option can be set to extend the installing stage timeout value + # wait_for_timeout: 3600 + + # Set this options for https with self-signed certificate + # no_check_certificate + + # Override default filesystem device: also from host-show, but is static. + # rootfs_device: "/dev/disk/by-path/pci-0000:00:1f.2-ata-1.0" + # boot_device: "/dev/disk/by-path/pci-0000:00:1f.2-ata-1.0" + + BOOTSTRAP: (also needed for bootstrap) + # If the subcloud's bootstrap IP interface and the system controller are not on the + # same network then the customer must configure a default route or static route + # so that the Central Cloud can login bootstrap the newly installed subcloud. + # If nexthop_gateway is specified and the network_address is not specified then a + # default route will be configured. Otherwise, if a network_address is specified + then + # a static route will be configured. + nexthop_gateway: default_route_address + network_address: static_route_address + network_mask: static_route_mask + + subcloud.data_upgrade - persist for upgrade duration + for values from subcloud online sysinv host-show (persist since upgrade-start) + bmc_address # sysinv_v1 host-show + bmc_username # sysinv_v1 host-show + for values from barbican_client (as barbican user), or from upgrade-start: + bmc_password --- obtain from barbican_client as barbican user + """ + + install_values = {'name': strategy_step.subcloud.name} + + install_values.update( + self._get_subcloud_upgrade_load_info(strategy_step)) + + upgrade_data_install_values = self._get_subcloud_upgrade_data_install( + strategy_step) + install_values.update(upgrade_data_install_values) + + install_values.update( + self._get_subcloud_upgrade_data( + strategy_step, subcloud_sysinv_client, subcloud_barbican_client)) + + # Check bmc values + if not self._bmc_data_available(install_values): + if self._bmc_data_available(upgrade_data_install_values): + # It is possible the bmc data is only latched on install if it + # was not part of the deployment configuration + install_values.update({ + 'bmc_address': + upgrade_data_install_values.get('bmc_address'), + 'bmc_username': + upgrade_data_install_values.get('bmc_username'), + 'bmc_password': + upgrade_data_install_values.get('bmc_password'), + }) + else: + message = ("Failed to get bmc credentials for subcloud %s" % + strategy_step.subcloud.name) + raise Exception(message) + + self.info_log(strategy_step, + "get_subcloud_upgrade_data_install %s" % install_values) + return install_values + + @staticmethod + def _bmc_data_available(bmc_values): + if (not bmc_values.get('bmc_username') or + not bmc_values.get('bmc_address') or + not bmc_values.get('bmc_password')): + return False return True + + def _get_subcloud_upgrade_load_info(self, strategy_step): + """Get the subcloud upgrade load information""" + + # The 'software_version' is the active running load on SystemController + matching_iso, _ = upgrade_utils.get_vault_load_files(SW_VERSION) + if not os.path.isfile(matching_iso): + message = ("Failed to get upgrade load info for subcloud %s" % + strategy_step.subcloud.name) + raise Exception(message) + + load_info = {'software_version': SW_VERSION, + 'image': matching_iso} + + return load_info + + def _get_subcloud_upgrade_data_install(self, strategy_step): + """Get subcloud upgrade data_install from persisted values""" + + upgrade_data_install = {} + + subcloud = db_api.subcloud_get(self.context, strategy_step.subcloud_id) + if not subcloud.data_install: + message = ("Failed to get upgrade data from install " + "for subcloud %s." % + strategy_step.subcloud.name) + LOG.warn(message) + raise Exception(message) + + data_install = json.loads(subcloud.data_install) + + # base64 encoded sysadmin_password is default + upgrade_data_install.update({ + 'ansible_become_pass': consts.TEMP_SYSADMIN_PASSWORD, + 'ansible_ssh_pass': consts.TEMP_SYSADMIN_PASSWORD, + }) + # Get mandatory bootstrap info from data_install + # bootstrap_address is referenced in SubcloudInstall + # bootstrap-address is referenced in create_subcloud_inventory and + # subcloud manager. + # todo(jkung): refactor to just use one bootstrap address index + upgrade_data_install.update({ + 'bootstrap_interface': data_install.get('bootstrap_interface'), + 'bootstrap-address': data_install.get('bootstrap_address'), + 'bootstrap_address': data_install.get('bootstrap_address'), + 'bootstrap_address_prefix': data_install.get('bootstrap_address_prefix'), + 'bmc_username': data_install.get('bmc_username'), + 'bmc_address': data_install.get('bmc_address'), + 'bmc_password': data_install.get('bmc_password'), + }) + + # optional bootstrap parameters + optional_bootstrap_parameters = [ + 'nexthop_gateway', # default route address + 'network_address', # static route address + 'network_mask', # static route mask + 'bootstrap_vlan', + 'wait_for_timeout', + 'no_check_certificate', + ] + + for p in optional_bootstrap_parameters: + if p in data_install: + upgrade_data_install.update({p: data_install.get(p)}) + + return upgrade_data_install + + def _get_subcloud_upgrade_data( + self, strategy_step, subcloud_sysinv_client, subcloud_barbican_client): + """Get the subcloud data required for upgrades. + + In case the subcloud is no longer reachable, get upgrade_data from + persisted database values. For example, this may be required in + the scenario where the subcloud experiences an unexpected error + (e.g. loss of power) and this step needs to be rerun. + """ + + volatile_data_install = {} + + if subcloud_sysinv_client is None: + # subcloud is not reachable, use previously saved values + subcloud = db_api.subcloud_get( + self.context, strategy_step.subcloud_id) + if subcloud.data_upgrade: + return json.loads(subcloud.data_upgrade) + else: + message = ('Cannot retrieve upgrade data install ' + 'for subcloud: %s' % + strategy_step.subcloud.name) + raise Exception(message) + + subcloud_system = subcloud_sysinv_client.get_system() + + if subcloud_system.system_type != 'All-in-one': + message = ('subcloud %s install unsupported for system type: %s' % + (strategy_step.subcloud.name, + subcloud_system.system_type)) + raise Exception(message) + + host = subcloud_sysinv_client.get_host('controller-0') + + install_type = self._get_install_type(host) + + bmc_password = None + if subcloud_barbican_client: + bmc_password = subcloud_barbican_client.get_host_bmc_password(host.uuid) + + volatile_data_install.update({ + 'bmc_address': host.bm_ip, + 'bmc_username': host.bm_username, + 'bmc_password': bmc_password, + 'install_type': install_type, + 'boot_device': host.boot_device, + 'rootfs_device': host.rootfs_device, + }) + + # Persist the volatile data + db_api.subcloud_update( + self.context, strategy_step.subcloud_id, + data_upgrade=json.dumps(volatile_data_install)) + + admin_password = str(keyring.get_password('CGCS', 'admin')) + volatile_data_install.update({'admin_password': admin_password}) + + return volatile_data_install + + @staticmethod + def _get_install_type(host): + if 'lowlatency' in host.subfunctions.split(','): + lowlatency = True + else: + lowlatency = False + + if 'graphical' in host.console.split(','): # graphical console + if lowlatency: + install_type = 5 + else: + install_type = 3 + else: # serial console + if lowlatency: + install_type = 4 + else: + install_type = 2 + return install_type + + def perform_subcloud_install(self, strategy_step, session, install_values): + + db_api.subcloud_update( + self.context, strategy_step.subcloud_id, + deploy_status=consts.DEPLOY_STATE_PRE_INSTALL) + self.context.auth_token = session.get_token() + self.context.project = session.get_project_id() + try: + install = SubcloudInstall( + self.context, strategy_step.subcloud.name) + install.prep(consts.ANSIBLE_OVERRIDES_PATH, + install_values) + except Exception as e: + db_api.subcloud_update( + self.context, strategy_step.subcloud_id, + deploy_status=consts.DEPLOY_STATE_PRE_INSTALL_FAILED) + self.error_log(strategy_step, e.message) + # TODO(jkung): cleanup to be implemented within SubcloudInstall + install.cleanup() + raise + + ansible_subcloud_inventory_file = os.path.join( + consts.ANSIBLE_OVERRIDES_PATH, + strategy_step.subcloud.name + INVENTORY_FILE_POSTFIX) + + # Create the ansible inventory for the upgrade subcloud + utils.create_subcloud_inventory(install_values, + ansible_subcloud_inventory_file) + + # SubcloudInstall.prep creates data_install.yml (install overrides) + install_command = [ + "ansible-playbook", ANSIBLE_SUBCLOUD_INSTALL_PLAYBOOK, + "-i", ansible_subcloud_inventory_file, + "-e", "@%s" % consts.ANSIBLE_OVERRIDES_PATH + "/" + + strategy_step.subcloud.name + '/' + "install_values.yml" + ] + + # Run the remote install playbook + db_api.subcloud_update( + self.context, strategy_step.subcloud_id, + deploy_status=consts.DEPLOY_STATE_INSTALLING) + try: + install.install(consts.DC_LOG_DIR, install_command) + except Exception as e: + db_api.subcloud_update( + self.context, strategy_step.subcloud_id, + deploy_status=consts.DEPLOY_STATE_INSTALL_FAILED) + self.error_log(strategy_step, e.message) + install.cleanup() + raise + + db_api.subcloud_update( + self.context, strategy_step.subcloud_id, + deploy_status=consts.DEPLOY_STATE_INSTALLED) + install.cleanup() + LOG.info("Successfully installed subcloud %s" % + strategy_step.subcloud.name) diff --git a/distributedcloud/dcmanager/manager/subcloud_manager.py b/distributedcloud/dcmanager/manager/subcloud_manager.py index 599f3cabc..a66a7b534 100644 --- a/distributedcloud/dcmanager/manager/subcloud_manager.py +++ b/distributedcloud/dcmanager/manager/subcloud_manager.py @@ -39,11 +39,13 @@ from dccommon import consts as dccommon_consts from dccommon.drivers.openstack.keystone_v3 import KeystoneClient from dccommon.drivers.openstack.sysinv_v1 import SysinvClient from dccommon import kubeoperator +from dccommon.subcloud_install import SubcloudInstall from dcorch.common import consts as dcorch_consts from dcorch.rpc import client as dcorch_rpc_client from dcmanager.common import consts +from dcmanager.common.consts import INVENTORY_FILE_POSTFIX from dcmanager.common import context from dcmanager.common import exceptions from dcmanager.common.i18n import _ @@ -51,7 +53,6 @@ from dcmanager.common import manager from dcmanager.common import utils from dcmanager.db import api as db_api -from dcmanager.manager.subcloud_install import SubcloudInstall from fm_api import constants as fm_const from fm_api import fm_api @@ -63,12 +64,10 @@ LOG = logging.getLogger(__name__) ADDN_HOSTS_DC = 'dnsmasq.addn_hosts_dc' # Subcloud configuration paths -INVENTORY_FILE_POSTFIX = '_inventory.yml' ANSIBLE_SUBCLOUD_PLAYBOOK = \ '/usr/share/ansible/stx-ansible/playbooks/bootstrap.yml' ANSIBLE_SUBCLOUD_INSTALL_PLAYBOOK = \ '/usr/share/ansible/stx-ansible/playbooks/install.yml' -DC_LOG_DIR = '/var/log/dcmanager/' USERS_TO_REPLICATE = [ 'sysinv', @@ -79,8 +78,6 @@ USERS_TO_REPLICATE = [ 'barbican', 'dcmanager'] -SERVICES_USER = 'services' - SC_INTERMEDIATE_CERT_DURATION = "87600h" SC_INTERMEDIATE_CERT_RENEW_BEFORE = "720h" CERT_NAMESPACE = "dc-cert" @@ -191,7 +188,6 @@ class SubcloudManager(manager.Manager): """Add subcloud and notify orchestrators. :param context: request context object - :param name: name of subcloud to add :param payload: subcloud configuration """ LOG.info("Adding subcloud %s." % payload['name']) @@ -303,7 +299,8 @@ class SubcloudManager(manager.Manager): dccommon_consts.ADMIN_USER_NAME) admin_project = m_ks_client.get_project_by_name( dccommon_consts.ADMIN_PROJECT_NAME) - services_project = m_ks_client.get_project_by_name(SERVICES_USER) + services_project = m_ks_client.get_project_by_name( + dccommon_consts.SERVICES_USER_NAME) sysinv_user = m_ks_client.get_user_by_name( dccommon_consts.SYSINV_USER_NAME) dcmanager_user = m_ks_client.get_user_by_name( @@ -342,14 +339,14 @@ class SubcloudManager(manager.Manager): ] del payload['sysadmin_password'] - payload['users'] = dict() for user in USERS_TO_REPLICATE: payload['users'][user] = \ - str(keyring.get_password(user, SERVICES_USER)) + str(keyring.get_password( + user, dccommon_consts.SERVICES_USER_NAME)) # Create the ansible inventory for the new subcloud - self._create_subcloud_inventory(payload, + utils.create_subcloud_inventory(payload, ansible_subcloud_inventory_file) # create subcloud intermediate certificate and pass in keys @@ -467,7 +464,7 @@ class SubcloudManager(manager.Manager): context, subcloud.id, deploy_status=consts.DEPLOY_STATE_INSTALLING) try: - install.install(DC_LOG_DIR, install_command) + install.install(consts.DC_LOG_DIR, install_command) except Exception as e: db_api.subcloud_update( context, subcloud.id, @@ -490,7 +487,7 @@ class SubcloudManager(manager.Manager): # Run the ansible boostrap-subcloud playbook log_file = \ - DC_LOG_DIR + subcloud.name + '_bootstrap_' + \ + consts.DC_LOG_DIR + subcloud.name + '_bootstrap_' + \ str(datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')) \ + '.log' with open(log_file, "w") as f_out_log: @@ -519,7 +516,7 @@ class SubcloudManager(manager.Manager): context, subcloud.id, deploy_status=consts.DEPLOY_STATE_DEPLOYING) log_file = \ - DC_LOG_DIR + subcloud.name + '_deploy_' + \ + consts.DC_LOG_DIR + subcloud.name + '_deploy_' + \ str(datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')) \ + '.log' with open(log_file, "w") as f_out_log: @@ -569,35 +566,6 @@ class SubcloudManager(manager.Manager): # restart dnsmasq so it can re-read our addn_hosts file. os.system("pkill -HUP dnsmasq") - def _create_subcloud_inventory(self, - subcloud, - inventory_file): - """Create the inventory file for the specified subcloud""" - - # Delete the file if it already exists - if os.path.isfile(inventory_file): - os.remove(inventory_file) - - with open(inventory_file, 'w') as f_out_inventory: - f_out_inventory.write( - '---\n' - 'all:\n' - ' vars:\n' - ' ansible_ssh_user: sysadmin\n' - ' hosts:\n' - ' ' + subcloud['name'] + ':\n' - ' ansible_host: ' + - subcloud['bootstrap-address'] + '\n' - ) - - def _delete_subcloud_inventory(self, - inventory_file): - """Delete the inventory file for the specified subcloud""" - - # Delete the file if it exists - if os.path.isfile(inventory_file): - os.remove(inventory_file) - def _write_subcloud_ansible_config(self, context, payload): """Create the override file for usage with the specified subcloud""" @@ -736,7 +704,7 @@ class SubcloudManager(manager.Manager): raise e # Delete the ansible inventory for the new subcloud - self._delete_subcloud_inventory(ansible_subcloud_inventory_file) + utils.delete_subcloud_inventory(ansible_subcloud_inventory_file) # Delete the subcloud intermediate certificate SubcloudManager._delete_subcloud_cert(subcloud.name) diff --git a/distributedcloud/dcmanager/tests/base.py b/distributedcloud/dcmanager/tests/base.py index 1260d7165..9969c3ad6 100644 --- a/distributedcloud/dcmanager/tests/base.py +++ b/distributedcloud/dcmanager/tests/base.py @@ -20,6 +20,7 @@ # of an applicable Wind River license agreement. # +import json import sqlalchemy from oslo_config import cfg @@ -65,7 +66,8 @@ SUBCLOUD_SAMPLE_DATA_0 = [ "10.10.10.12", # external_oam_floating_address "testpass", # sysadmin_password 1, # group_id - consts.DEPLOY_STATE_DONE # deploy_status + consts.DEPLOY_STATE_DONE, # deploy_status + json.dumps({'data_install': 'test data install values'}), # data_install ] diff --git a/distributedcloud/dcmanager/tests/unit/manager/test_subcloud_manager.py b/distributedcloud/dcmanager/tests/unit/manager/test_subcloud_manager.py index 82ef95dd4..22728e775 100644 --- a/distributedcloud/dcmanager/tests/unit/manager/test_subcloud_manager.py +++ b/distributedcloud/dcmanager/tests/unit/manager/test_subcloud_manager.py @@ -30,6 +30,7 @@ import threading from dccommon import consts as dccommon_consts from dcmanager.common import consts from dcmanager.common import exceptions +from dcmanager.common import utils as cutils from dcmanager.db.sqlalchemy import api as db_api from dcmanager.manager import subcloud_manager from dcmanager.tests import base @@ -123,6 +124,7 @@ class Subcloud(object): data['external_oam_floating_address'] self.systemcontroller_gateway_ip = \ data['systemcontroller_gateway_address'] + self.data_install = data['data_install'] self.created_at = timeutils.utcnow() self.updated_at = timeutils.utcnow() @@ -159,6 +161,7 @@ class TestSubcloudManager(base.DCManagerTestCase): 'deploy_status': "not-deployed", 'openstack_installed': False, 'group_id': 1, + 'data_install': 'data from install', } values.update(kwargs) return db_api.subcloud_create(ctxt, **values) @@ -172,15 +175,13 @@ class TestSubcloudManager(base.DCManagerTestCase): @mock.patch.object(subcloud_manager.SubcloudManager, '_create_intermediate_ca_cert') - @mock.patch.object(subcloud_manager.SubcloudManager, - '_delete_subcloud_inventory') + @mock.patch.object(cutils, 'delete_subcloud_inventory') @mock.patch.object(subcloud_manager, 'KeystoneClient') @mock.patch.object(subcloud_manager, 'db_api') @mock.patch.object(subcloud_manager, 'SysinvClient') @mock.patch.object(subcloud_manager.SubcloudManager, '_create_addn_hosts_dc') - @mock.patch.object(subcloud_manager.SubcloudManager, - '_create_subcloud_inventory') + @mock.patch.object(cutils, 'create_subcloud_inventory') @mock.patch.object(subcloud_manager.SubcloudManager, '_write_subcloud_ansible_config') @mock.patch.object(subcloud_manager, diff --git a/distributedcloud/dcmanager/tests/utils.py b/distributedcloud/dcmanager/tests/utils.py index 7096f9c85..93a4d4b24 100644 --- a/distributedcloud/dcmanager/tests/utils.py +++ b/distributedcloud/dcmanager/tests/utils.py @@ -119,4 +119,5 @@ def create_subcloud_dict(data_list): 'external_oam_floating_address': data_list[21], 'sysadmin_password': data_list[22], 'group_id': data_list[23], - 'deploy_status': data_list[24]} + 'deploy_status': data_list[24], + 'data_install': data_list[25]} diff --git a/distributedcloud/requirements.txt b/distributedcloud/requirements.txt index de103237e..48e36ac89 100644 --- a/distributedcloud/requirements.txt +++ b/distributedcloud/requirements.txt @@ -41,6 +41,7 @@ oslo.utils>=3.20.0 # Apache-2.0 oslo.versionedobjects>=1.17.0 # Apache-2.0 sqlalchemy-migrate>=0.11.0 # Apache-2.0 python-openstackclient!=3.10.0,>=3.3.0 # Apache-2.0 +python-barbicanclient>=4.5.2 python-neutronclient>=6.3.0 # Apache-2.0 python-cinderclient>=2.1.0 # Apache-2.0 python-novaclient>=7.1.0 # Apache-2.0