Implement the state for Simplex Upgrade

The Simplex Upgrade step upgrades the simplex subcloud to the load
imported in dc-vault corresponding to the target software release.

Required static data is captured at subcloud add and referenced for
the simplex upgrade.

Dynamic data is retrieved from the online subcloud; including the
bmc credentials which are obtained via a synchronized service within
project allowing for access to the bmc credentials.

BMC password retrieval endpoint_cache integration
with https://review.opendev.org/#/c/736004/

Tests Performed:
- Subcloud upgrade (install) with load corresponding
  to System Controller
- Subcloud add persists install data
- Simplex upgrade persists dynamic upgrade data from start of
  upgrade simplex
- Obtain bmc password from online subcloud
- Obtain bmc password from install data, in case the
  bmc is not configured

Pending for subsequent commit:
- tox unit tests

Change-Id: Ie047139280a5780bfe47b9f9959f4c6781d6f3fa
Story: 2007403
Task: 40024
Signed-off-by: John Kung <john.kung@windriver.com>

distributedcloud/dccommon/consts.py

@@ -33,6 +33,7 @@ CLOUD_0 = "RegionOne"
 VIRTUAL_MASTER_CLOUD = "SystemController"
 
 SW_UPDATE_DEFAULT_TITLE = "all clouds default"
+LOAD_VAULT_DIR = '/opt/dc-vault/loads'
 
 USER_HEADER_VALUE = "distcloud"
 USER_HEADER = {'User-Header': USER_HEADER_VALUE}
@@ -41,3 +42,4 @@ ADMIN_USER_NAME = "admin"
 ADMIN_PROJECT_NAME = "admin"
 SYSINV_USER_NAME = "sysinv"
 DCMANAGER_USER_NAME = "dcmanager"
+SERVICES_USER_NAME = "services"

distributedcloud/dccommon/drivers/openstack/barbican.py

@@ -0,0 +1,75 @@
+# Copyright 2016 Ericsson AB
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# Copyright (c) 2020 Wind River Systems, Inc.
+#
+# The right to copy, distribute, modify, or otherwise make use
+# of this software may be licensed only pursuant to the terms
+# of an applicable Wind River license agreement.
+#
+
+
+from barbicanclient import client
+
+from oslo_log import log
+
+from dccommon import consts as dccommon_consts
+from dccommon.drivers import base
+from dccommon import exceptions
+
+
+LOG = log.getLogger(__name__)
+API_VERSION = 'v1'
+
+
+class BarbicanClient(base.DriverBase):
+    """Barbican driver.
+
+    The session needs to be associated with synchronized 'services' project
+    in order for the client to get the host bmc password.
+    """
+
+    def __init__(
+            self, region, session, endpoint_type=dccommon_consts.KS_ENDPOINT_DEFAULT):
+
+        try:
+            self.barbican_client = client.Client(
+                API_VERSION,
+                session=session,
+                region_name=region,
+                interface=endpoint_type)
+
+            self.region_name = region
+        except exceptions.ServiceUnavailable:
+            raise
+
+    def get_host_bmc_password(self, host_uuid):
+        """Get the Board Management Controller password corresponding to the host
+
+        :param host_uuid The host uuid
+        """
+
+        secrets = self.barbican_client.secrets.list()
+        for secret in secrets:
+            if secret.name == host_uuid:
+                secret_ref = secret.secret_ref
+                break
+        else:
+            return
+
+        secret = self.barbican_client.secrets.get(secret_ref)
+
+        bmc_password = secret.payload
+
+        return bmc_password

distributedcloud/dccommon/drivers/openstack/sdk_platform.py

@@ -23,6 +23,7 @@ from oslo_log import log
 from oslo_utils import timeutils
 
 from dccommon import consts
+from dccommon.drivers.openstack.barbican import BarbicanClient
 from dccommon.drivers.openstack.fm import FmClient
 from dccommon.drivers.openstack.keystone_v3 import KeystoneClient
 from dccommon.drivers.openstack.sysinv_v1 import SysinvClient
@@ -39,6 +40,7 @@ STALE_TOKEN_DURATION_STEP = 20
 KEYSTONE_CLIENT_NAME = 'keystone'
 SYSINV_CLIENT_NAME = 'sysinv'
 FM_CLIENT_NAME = 'fm'
+BARBICAN_CLIENT_NAME = 'barbican'
 
 LOG = log.getLogger(__name__)
 
@@ -46,13 +48,15 @@ LOCK_NAME = 'dc-openstackdriver-platform'
 
 SUPPORTED_REGION_CLIENTS = [
     SYSINV_CLIENT_NAME,
-    FM_CLIENT_NAME
+    FM_CLIENT_NAME,
+    BARBICAN_CLIENT_NAME,
 ]
 
 # region client type and class mappings
 region_client_class_map = {
     SYSINV_CLIENT_NAME: SysinvClient,
     FM_CLIENT_NAME: FmClient,
+    BARBICAN_CLIENT_NAME: BarbicanClient,
 }
 
 
@@ -68,6 +72,7 @@ class OpenStackDriver(object):
         self.keystone_client = None
         self.sysinv_client = None
         self.fm_client = None
+        self.barbican_client = None
 
         if region_clients:
             # check if the requested clients are in the supported client list

distributedcloud/dccommon/drivers/openstack/sysinv_v1.py

@@ -130,6 +130,39 @@ class SysinvClient(base.DriverBase):
             action_value = 'unlock'
         return self._do_host_action(host_id, action_value)
 
+    def configure_bmc_host(self,
+                           host_id,
+                           bm_username,
+                           bm_ip,
+                           bm_password,
+                           bm_type='ipmi'):
+        """Configure bmc of a host"""
+        patch = [
+            {'op': 'replace',
+             'path': '/bm_username',
+             'value': bm_username},
+            {'op': 'replace',
+             'path': '/bm_ip',
+             'value': bm_ip},
+            {'op': 'replace',
+             'path': '/bm_password',
+             'value': bm_password},
+            {'op': 'replace',
+             'path': '/bm_type',
+             'value': bm_type},
+        ]
+        return self.sysinv_client.ihost.update(host_id, patch)
+
+    def power_on_host(self, host_id):
+        """Power on a host"""
+        action_value = 'power-on'
+        return self._do_host_action(host_id, action_value)
+
+    def power_off_host(self, host_id):
+        """Power off a host"""
+        action_value = 'power-off'
+        return self._do_host_action(host_id, action_value)
+
     def get_management_interface(self, hostname):
         """Get the management interface for a host."""
         interfaces = self.sysinv_client.iinterface.list(hostname)

distributedcloud/dccommon/install_consts.py

@@ -31,3 +31,6 @@ MANDATORY_INSTALL_VALUES = [
     'bmc_password',
     'install_type'
 ]
+
+ANSIBLE_SUBCLOUD_INSTALL_PLAYBOOK = \
+    '/usr/share/ansible/stx-ansible/playbooks/install.yml'

distributedcloud/dccommon/subcloud_install.py

@@ -24,19 +24,17 @@ from eventlet.green import subprocess
 import json
 import netaddr
 import os
-import socket
-
+from oslo_log import log as logging
 from six.moves.urllib import error as urllib_error
 from six.moves.urllib import parse
 from six.moves.urllib import request
+import socket
 
+from dccommon import consts
 from dccommon.drivers.openstack.keystone_v3 import KeystoneClient
 from dccommon.drivers.openstack.sysinv_v1 import SysinvClient
-from dcmanager.common import consts
-from dcmanager.common import exceptions
-from dcmanager.common import install_consts
-
-from oslo_log import log as logging
+from dccommon import exceptions
+from dccommon import install_consts
 
 LOG = logging.getLogger(__name__)
 
@@ -88,7 +86,7 @@ class SubcloudInstall(object):
         ks_client = KeystoneClient()
         session = ks_client.endpoint_cache.get_session_from_token(
             context.auth_token, context.project)
-        self.sysinv_client = SysinvClient(consts.DEFAULT_REGION_NAME, session)
+        self.sysinv_client = SysinvClient(consts.CLOUD_0, session)
         self.name = subcloud_name
         self.input_iso = None
         self.www_root = None
@@ -288,14 +286,14 @@ class SubcloudInstall(object):
             msg = "Error: Downloading file %s may be interrupted: %s" % (
                 values['image'], e)
             LOG.error(msg)
-            raise exceptions.DCManagerException(
+            raise exceptions.DCCommonException(
                 resource=self.name,
                 msg=msg)
         except Exception as e:
             msg = "Error: Could not download file %s: %s" % (
                 values['image'], e)
             LOG.error(msg)
-            raise exceptions.DCManagerException(
+            raise exceptions.DCCommonException(
                 resource=self.name,
                 msg=msg)
 

distributedcloud/dcmanager/api/controllers/v1/subclouds.py

@@ -21,6 +21,7 @@
 from requests_toolbelt.multipart import decoder
 
 import base64
+import json
 import keyring
 from netaddr import AddrFormatError
 from netaddr import IPAddress
@@ -39,6 +40,7 @@ from pecan import request
 from dccommon.drivers.openstack.keystone_v3 import KeystoneClient
 from dccommon.drivers.openstack.sysinv_v1 import SysinvClient
 from dccommon import exceptions as dccommon_exceptions
+from dccommon import install_consts
 
 from keystoneauth1 import exceptions as keystone_exceptions
 
@@ -48,7 +50,6 @@ from dcmanager.api.controllers import restcomm
 from dcmanager.common import consts
 from dcmanager.common import exceptions
 from dcmanager.common.i18n import _
-from dcmanager.common import install_consts
 from dcmanager.common import utils
 from dcmanager.db import api as db_api
 
@@ -495,6 +496,10 @@ class SubcloudsController(object):
         # if group_id has been omitted from payload, use 'Default'.
         group_id = payload.get('group_id',
                                consts.DEFAULT_SUBCLOUD_GROUP_ID)
+        data_install = None
+        if 'install_values' in payload:
+            data_install = json.dumps(payload['install_values'])
+
         subcloud = db_api.subcloud_create(
             context,
             payload['name'],
@@ -508,7 +513,8 @@ class SubcloudsController(object):
             payload['systemcontroller_gateway_address'],
             consts.DEPLOY_STATE_NONE,
             False,
-            group_id)
+            group_id,
+            data_install=data_install)
         return subcloud
 
     @index.when(method='GET', template='json')

distributedcloud/dcmanager/common/consts.py

@@ -123,6 +123,7 @@ DEPLOY_STATE_PRE_INSTALL = 'pre-install'
 DEPLOY_STATE_PRE_INSTALL_FAILED = 'pre-install-failed'
 DEPLOY_STATE_INSTALLING = 'installing'
 DEPLOY_STATE_INSTALL_FAILED = 'install-failed'
+DEPLOY_STATE_INSTALLED = 'installed'
 DEPLOY_STATE_BOOTSTRAPPING = 'bootstrapping'
 DEPLOY_STATE_BOOTSTRAP_FAILED = 'bootstrap-failed'
 DEPLOY_STATE_DEPLOYING = 'deploying'
@@ -147,3 +148,12 @@ DEPLOY_COMMON_FILE_OPTIONS = [
     DEPLOY_OVERRIDES,
     DEPLOY_CHART
 ]
+
+
+DC_LOG_DIR = '/var/log/dcmanager/'
+INVENTORY_FILE_POSTFIX = '_inventory.yml'
+
+# The following password is just a temporary and internal password that is used
+# after a remote install as part of the upgrade. The real sysadmin password
+# will be restored af the subcloud is re-managed at the end of the upgrade.
+TEMP_SYSADMIN_PASSWORD = 'St8rlingX*'

distributedcloud/dcmanager/common/utils.py

@@ -207,3 +207,30 @@ def get_filename_by_prefix(dir_path, prefix):
         if filename.startswith(prefix):
             return filename
     return None
+
+
+def create_subcloud_inventory(subcloud, inventory_file):
+    """Create the ansible inventory file for the specified subcloud"""
+
+    # Delete the file if it already exists
+    delete_subcloud_inventory(inventory_file)
+
+    with open(inventory_file, 'w') as f_out_inventory:
+        f_out_inventory.write(
+            '---\n'
+            'all:\n'
+            '  vars:\n'
+            '    ansible_ssh_user: sysadmin\n'
+            '  hosts:\n'
+            '    ' + subcloud['name'] + ':\n'
+            '      ansible_host: ' +
+            subcloud['bootstrap-address'] + '\n'
+        )
+
+
+def delete_subcloud_inventory(inventory_file):
+    """Delete the ansible inventory file for the specified subcloud"""
+
+    # Delete the file if it exists
+    if os.path.isfile(inventory_file):
+        os.remove(inventory_file)

distributedcloud/dcmanager/db/api.py

@@ -66,6 +66,8 @@ def subcloud_db_model_to_dict(subcloud):
               "openstack-installed": subcloud.openstack_installed,
               "systemcontroller-gateway-ip":
                   subcloud.systemcontroller_gateway_ip,
+              "data_install": subcloud.data_install,
+              "data_upgrade": subcloud.data_upgrade,
               "created-at": subcloud.created_at,
               "updated-at": subcloud.updated_at,
               "group_id": subcloud.group_id}
@@ -76,14 +78,14 @@ def subcloud_create(context, name, description, location, software_version,
                     management_subnet, management_gateway_ip,
                     management_start_ip, management_end_ip,
                     systemcontroller_gateway_ip, deploy_status,
-                    openstack_installed, group_id):
+                    openstack_installed, group_id, data_install=None):
     """Create a subcloud."""
     return IMPL.subcloud_create(context, name, description, location,
                                 software_version,
                                 management_subnet, management_gateway_ip,
                                 management_start_ip, management_end_ip,
                                 systemcontroller_gateway_ip, deploy_status,
-                                openstack_installed, group_id)
+                                openstack_installed, group_id, data_install)
 
 
 def subcloud_get(context, subcloud_id):
@@ -115,12 +117,13 @@ def subcloud_update(context, subcloud_id, management_state=None,
                     availability_status=None, software_version=None,
                     description=None, location=None, audit_fail_count=None,
                     deploy_status=None, openstack_installed=None,
-                    group_id=None):
+                    group_id=None, data_install=None, data_upgrade=None):
     """Update a subcloud or raise if it does not exist."""
     return IMPL.subcloud_update(context, subcloud_id, management_state,
                                 availability_status, software_version,
                                 description, location, audit_fail_count,
-                                deploy_status, openstack_installed, group_id)
+                                deploy_status, openstack_installed, group_id,
+                                data_install, data_upgrade)
 
 
 def subcloud_destroy(context, subcloud_id):

distributedcloud/dcmanager/db/sqlalchemy/api.py

@@ -213,7 +213,7 @@ def subcloud_create(context, name, description, location, software_version,
                     management_subnet, management_gateway_ip,
                     management_start_ip, management_end_ip,
                     systemcontroller_gateway_ip, deploy_status,
-                    openstack_installed, group_id):
+                    openstack_installed, group_id, data_install=None):
     with write_session() as session:
         subcloud_ref = models.Subcloud()
         subcloud_ref.name = name
@@ -231,6 +231,8 @@ def subcloud_create(context, name, description, location, software_version,
         subcloud_ref.audit_fail_count = 0
         subcloud_ref.openstack_installed = openstack_installed
         subcloud_ref.group_id = group_id
+        if data_install is not None:
+            subcloud_ref.data_install = data_install
         session.add(subcloud_ref)
         return subcloud_ref
 
@@ -239,8 +241,11 @@ def subcloud_create(context, name, description, location, software_version,
 def subcloud_update(context, subcloud_id, management_state=None,
                     availability_status=None, software_version=None,
                     description=None, location=None, audit_fail_count=None,
-                    deploy_status=None, openstack_installed=None,
-                    group_id=None):
+                    deploy_status=None,
+                    openstack_installed=None,
+                    group_id=None,
+                    data_install=None,
+                    data_upgrade=None):
     with write_session() as session:
         subcloud_ref = subcloud_get(context, subcloud_id)
         if management_state is not None:
@@ -255,8 +260,12 @@ def subcloud_update(context, subcloud_id, management_state=None,
             subcloud_ref.location = location
         if audit_fail_count is not None:
             subcloud_ref.audit_fail_count = audit_fail_count
+        if data_install is not None:
+            subcloud_ref.data_install = data_install
         if deploy_status is not None:
             subcloud_ref.deploy_status = deploy_status
+        if data_upgrade is not None:
+            subcloud_ref.data_upgrade = data_upgrade
         if openstack_installed is not None:
             subcloud_ref.openstack_installed = openstack_installed
         if group_id is not None:

distributedcloud/dcmanager/db/sqlalchemy/migrate_repo/versions/007_add_subcloud_install.py

@@ -0,0 +1,40 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# Copyright (c) 2020 Wind River Systems, Inc.
+#
+# The right to copy, distribute, modify, or otherwise make use
+# of this software may be licensed only pursuant to the terms
+# of an applicable Wind River license agreement.
+#
+
+from sqlalchemy import Column
+from sqlalchemy import MetaData
+from sqlalchemy import Table
+from sqlalchemy import Text
+
+
+def upgrade(migrate_engine):
+    meta = MetaData()
+    meta.bind = migrate_engine
+
+    subclouds = Table('subclouds', meta, autoload=True)
+
+    # Add the 'data_install' to persist data_install data
+    subclouds.create_column(Column('data_install', Text))
+
+    # Add the data_upgrade which persist over an upgrade
+    subclouds.create_column(Column('data_upgrade', Text))
+
+
+def downgrade(migrate_engine):
+    raise NotImplementedError('Database downgrade is unsupported.')

distributedcloud/dcmanager/db/sqlalchemy/models.py

@@ -99,7 +99,9 @@ class Subcloud(BASE, DCManagerBase):
     software_version = Column(String(255))
     management_state = Column(String(255))
     availability_status = Column(String(255))
+    data_install = Column(String())
     deploy_status = Column(String(255))
+    data_upgrade = Column(String())
     management_subnet = Column(String(255))
     management_gateway_ip = Column(String(255))
     management_start_ip = Column(String(255), unique=True)
@@ -107,6 +109,7 @@ class Subcloud(BASE, DCManagerBase):
     openstack_installed = Column(Boolean, nullable=False, default=False)
     systemcontroller_gateway_ip = Column(String(255))
     audit_fail_count = Column(Integer)
+
     # multiple subclouds can be in a particular group
     group_id = Column(Integer,
                       ForeignKey('subcloud_group.id'))

distributedcloud/dcmanager/manager/states/base.py

@@ -8,9 +8,11 @@ import six
 
 from oslo_log import log as logging
 
+from dccommon.drivers.openstack.barbican import BarbicanClient
 from dccommon.drivers.openstack.sdk_platform import OpenStackDriver
 from dccommon.drivers.openstack.sysinv_v1 import SysinvClient
 from dcmanager.common import consts
+from dcmanager.common import context
 
 LOG = logging.getLogger(__name__)
 
@@ -20,6 +22,7 @@ class BaseState(object):
 
     def __init__(self):
         super(BaseState, self).__init__()
+        self.context = context.get_admin_context()
 
     def debug_log(self, strategy_step, details):
         LOG.debug("Stage: %s, State: %s, Subcloud: %s, Details: %s"
@@ -35,6 +38,13 @@ class BaseState(object):
                     self.get_region_name(strategy_step),
                     details))
 
+    def error_log(self, strategy_step, details):
+        LOG.error("Stage: %s, State: %s, Subcloud: %s, Details: %s"
+                  % (strategy_step.stage,
+                     strategy_step.state,
+                     self.get_region_name(strategy_step),
+                     details))
+
     @staticmethod
     def get_region_name(strategy_step):
         """Get the region name for a strategy step"""
@@ -64,6 +74,13 @@ class BaseState(object):
         """
         return SysinvClient(region_name, session)
 
+    @staticmethod
+    def get_barbican_client(region_name, session):
+        """construct a barbican client
+
+        """
+        return BarbicanClient(region_name, session)
+
     @abc.abstractmethod
     def perform_state_action(self, strategy_step):
         """Perform the action for this state on the strategy_step"""

distributedcloud/dcmanager/manager/states/upgrade/upgrading_simplex.py

@@ -3,9 +3,22 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 #
-from oslo_log import log as logging
+import json
+import keyring
+import os
 
+from dccommon.install_consts import ANSIBLE_SUBCLOUD_INSTALL_PLAYBOOK
+from dccommon.subcloud_install import SubcloudInstall
+
+from dcmanager.common import consts
+from dcmanager.common.consts import INVENTORY_FILE_POSTFIX
+from dcmanager.common import utils
+from dcmanager.db import api as db_api
 from dcmanager.manager.states.base import BaseState
+from dcmanager.manager.states.upgrade import utils as upgrade_utils
+
+from oslo_log import log as logging
+from tsconfig.tsconfig import SW_VERSION
 
 LOG = logging.getLogger(__name__)
 
@@ -22,9 +35,349 @@ class UpgradingSimplexState(BaseState):
         Any exceptions raised by this method set the strategy to FAILED
         Returning normally from this method set the strategy to the next step
         """
-        LOG.warning("UpgradingSimplexState has not been implemented yet.")
 
-        # When we return from this method without throwing an exception, the
-        # state machine can proceed to the next state
-        LOG.warning("Faking transition to next state")
+        LOG.info("Performing simplex upgrade for subcloud %s" %
+                 strategy_step.subcloud.name)
+
+        subcloud_sysinv_client = None
+        subcloud_barbican_client = None
+        try:
+            subcloud_ks_client = self.get_keystone_client(strategy_step.subcloud.name)
+            subcloud_sysinv_client = self.get_sysinv_client(
+                strategy_step.subcloud.name,
+                subcloud_ks_client.session)
+            subcloud_barbican_client = self.get_barbican_client(
+                strategy_step.subcloud.name,
+                subcloud_ks_client.session)
+        except Exception:
+            # if getting the token times out, the orchestrator may have
+            # restarted and subcloud may be offline; so will attempt
+            # to use the persisted values
+            message = ("Simplex upgrade perform_subcloud_install "
+                       "subcloud %s failed to get subcloud client" %
+                       strategy_step.subcloud.name)
+            self.error_log(strategy_step, message)
+            pass
+
+        # Check whether subcloud is already re-installed with N+1 load
+        target_version = SW_VERSION
+        if self._check_load_already_active(
+                target_version, subcloud_sysinv_client):
+            self.info_log(strategy_step,
+                          "Load:%s already active" % target_version)
+            return True
+
+        # Check whether subcloud supports redfish, and if not, fail.
+        # This needs to be inferred from absence of install_values as
+        # there is currrently no external api to query.
+        install_values = self.get_subcloud_upgrade_install_values(
+            strategy_step, subcloud_sysinv_client, subcloud_barbican_client)
+
+        local_ks_client = self.get_keystone_client()
+
+        # Upgrade the subcloud to the install_values image
+        self.perform_subcloud_install(
+            strategy_step, local_ks_client.session, install_values)
+
+    def _check_load_already_active(self, target_version, subcloud_sysinv_client):
+        """Check if the target_version is already active in subcloud"""
+
+        if subcloud_sysinv_client:
+            current_loads = subcloud_sysinv_client.get_loads()
+            for load in current_loads:
+                if (load.software_version == target_version and
+                        load.state == 'active'):
+                    return True
+        return False
+
+    def get_subcloud_upgrade_install_values(
+            self, strategy_step,
+            subcloud_sysinv_client, subcloud_barbican_client):
+        """Get the data required for the remote subcloud install.
+
+        subcloud data_install are obtained from:
+
+        dcmanager database:
+            subcloud.subcloud_install_initial::for values which are persisted at subcloud_add time
+
+            INSTALL: (needed for upgrade install)
+                bootstrap_interface
+                bootstrap_vlan
+                bootstrap_address
+                bootstrap_address_prefix
+                install_type  # could also be from host-show
+
+                # This option can be set to extend the installing stage timeout value
+                # wait_for_timeout: 3600
+
+                # Set this options for https with self-signed certificate
+                # no_check_certificate
+
+                # Override default filesystem device: also from host-show, but is static.
+                # rootfs_device: "/dev/disk/by-path/pci-0000:00:1f.2-ata-1.0"
+                # boot_device: "/dev/disk/by-path/pci-0000:00:1f.2-ata-1.0"
+
+            BOOTSTRAP: (also needed for bootstrap)
+                # If the subcloud's bootstrap IP interface and the system controller are not on the
+                # same network then the customer must configure a default route or static route
+                # so that the Central Cloud can login bootstrap the newly installed subcloud.
+                # If nexthop_gateway is specified and the network_address is not specified then a
+                # default route will be configured. Otherwise, if a network_address is specified
+                then
+                # a static route will be configured.
+                nexthop_gateway: default_route_address
+                network_address: static_route_address
+                network_mask: static_route_mask
+
+            subcloud.data_upgrade - persist for upgrade duration
+                for values from subcloud online sysinv host-show (persist since upgrade-start)
+                    bmc_address  # sysinv_v1 host-show
+                    bmc_username # sysinv_v1 host-show
+                for values from barbican_client (as barbican user), or from upgrade-start:
+                    bmc_password --- obtain from barbican_client as barbican user
+        """
+
+        install_values = {'name': strategy_step.subcloud.name}
+
+        install_values.update(
+            self._get_subcloud_upgrade_load_info(strategy_step))
+
+        upgrade_data_install_values = self._get_subcloud_upgrade_data_install(
+            strategy_step)
+        install_values.update(upgrade_data_install_values)
+
+        install_values.update(
+            self._get_subcloud_upgrade_data(
+                strategy_step, subcloud_sysinv_client, subcloud_barbican_client))
+
+        # Check bmc values
+        if not self._bmc_data_available(install_values):
+            if self._bmc_data_available(upgrade_data_install_values):
+                # It is possible the bmc data is only latched on install if it
+                # was not part of the deployment configuration
+                install_values.update({
+                    'bmc_address':
+                        upgrade_data_install_values.get('bmc_address'),
+                    'bmc_username':
+                        upgrade_data_install_values.get('bmc_username'),
+                    'bmc_password':
+                        upgrade_data_install_values.get('bmc_password'),
+                })
+            else:
+                message = ("Failed to get bmc credentials for subcloud %s" %
+                           strategy_step.subcloud.name)
+                raise Exception(message)
+
+        self.info_log(strategy_step,
+                      "get_subcloud_upgrade_data_install %s" % install_values)
+        return install_values
+
+    @staticmethod
+    def _bmc_data_available(bmc_values):
+        if (not bmc_values.get('bmc_username') or
+                not bmc_values.get('bmc_address') or
+                not bmc_values.get('bmc_password')):
+            return False
         return True
+
+    def _get_subcloud_upgrade_load_info(self, strategy_step):
+        """Get the subcloud upgrade load information"""
+
+        # The 'software_version' is the active running load on SystemController
+        matching_iso, _ = upgrade_utils.get_vault_load_files(SW_VERSION)
+        if not os.path.isfile(matching_iso):
+            message = ("Failed to get upgrade load info for subcloud %s" %
+                       strategy_step.subcloud.name)
+            raise Exception(message)
+
+        load_info = {'software_version': SW_VERSION,
+                     'image': matching_iso}
+
+        return load_info
+
+    def _get_subcloud_upgrade_data_install(self, strategy_step):
+        """Get subcloud upgrade data_install from persisted values"""
+
+        upgrade_data_install = {}
+
+        subcloud = db_api.subcloud_get(self.context, strategy_step.subcloud_id)
+        if not subcloud.data_install:
+            message = ("Failed to get upgrade data from install "
+                       "for subcloud %s." %
+                       strategy_step.subcloud.name)
+            LOG.warn(message)
+            raise Exception(message)
+
+        data_install = json.loads(subcloud.data_install)
+
+        # base64 encoded sysadmin_password is default
+        upgrade_data_install.update({
+            'ansible_become_pass': consts.TEMP_SYSADMIN_PASSWORD,
+            'ansible_ssh_pass': consts.TEMP_SYSADMIN_PASSWORD,
+        })
+        # Get mandatory bootstrap info from data_install
+        # bootstrap_address is referenced in SubcloudInstall
+        # bootstrap-address is referenced in create_subcloud_inventory and
+        # subcloud manager.
+        # todo(jkung): refactor to just use one bootstrap address index
+        upgrade_data_install.update({
+            'bootstrap_interface': data_install.get('bootstrap_interface'),
+            'bootstrap-address': data_install.get('bootstrap_address'),
+            'bootstrap_address': data_install.get('bootstrap_address'),
+            'bootstrap_address_prefix': data_install.get('bootstrap_address_prefix'),
+            'bmc_username': data_install.get('bmc_username'),
+            'bmc_address': data_install.get('bmc_address'),
+            'bmc_password': data_install.get('bmc_password'),
+        })
+
+        # optional bootstrap parameters
+        optional_bootstrap_parameters = [
+            'nexthop_gateway',  # default route address
+            'network_address',  # static route address
+            'network_mask',  # static route mask
+            'bootstrap_vlan',
+            'wait_for_timeout',
+            'no_check_certificate',
+        ]
+
+        for p in optional_bootstrap_parameters:
+            if p in data_install:
+                upgrade_data_install.update({p: data_install.get(p)})
+
+        return upgrade_data_install
+
+    def _get_subcloud_upgrade_data(
+        self, strategy_step, subcloud_sysinv_client, subcloud_barbican_client):
+        """Get the subcloud data required for upgrades.
+
+           In case the subcloud is no longer reachable, get upgrade_data from
+           persisted database values.  For example, this may be required in
+           the scenario where the subcloud experiences an unexpected error
+           (e.g. loss of power) and this step needs to be rerun.
+        """
+
+        volatile_data_install = {}
+
+        if subcloud_sysinv_client is None:
+            # subcloud is not reachable, use previously saved values
+            subcloud = db_api.subcloud_get(
+                self.context, strategy_step.subcloud_id)
+            if subcloud.data_upgrade:
+                return json.loads(subcloud.data_upgrade)
+            else:
+                message = ('Cannot retrieve upgrade data install '
+                           'for subcloud: %s' %
+                           strategy_step.subcloud.name)
+                raise Exception(message)
+
+        subcloud_system = subcloud_sysinv_client.get_system()
+
+        if subcloud_system.system_type != 'All-in-one':
+            message = ('subcloud %s install unsupported for system type: %s' %
+                       (strategy_step.subcloud.name,
+                        subcloud_system.system_type))
+            raise Exception(message)
+
+        host = subcloud_sysinv_client.get_host('controller-0')
+
+        install_type = self._get_install_type(host)
+
+        bmc_password = None
+        if subcloud_barbican_client:
+            bmc_password = subcloud_barbican_client.get_host_bmc_password(host.uuid)
+
+        volatile_data_install.update({
+            'bmc_address': host.bm_ip,
+            'bmc_username': host.bm_username,
+            'bmc_password': bmc_password,
+            'install_type': install_type,
+            'boot_device': host.boot_device,
+            'rootfs_device': host.rootfs_device,
+        })
+
+        # Persist the volatile data
+        db_api.subcloud_update(
+            self.context, strategy_step.subcloud_id,
+            data_upgrade=json.dumps(volatile_data_install))
+
+        admin_password = str(keyring.get_password('CGCS', 'admin'))
+        volatile_data_install.update({'admin_password': admin_password})
+
+        return volatile_data_install
+
+    @staticmethod
+    def _get_install_type(host):
+        if 'lowlatency' in host.subfunctions.split(','):
+            lowlatency = True
+        else:
+            lowlatency = False
+
+        if 'graphical' in host.console.split(','):  # graphical console
+            if lowlatency:
+                install_type = 5
+            else:
+                install_type = 3
+        else:  # serial console
+            if lowlatency:
+                install_type = 4
+            else:
+                install_type = 2
+        return install_type
+
+    def perform_subcloud_install(self, strategy_step, session, install_values):
+
+        db_api.subcloud_update(
+            self.context, strategy_step.subcloud_id,
+            deploy_status=consts.DEPLOY_STATE_PRE_INSTALL)
+        self.context.auth_token = session.get_token()
+        self.context.project = session.get_project_id()
+        try:
+            install = SubcloudInstall(
+                self.context, strategy_step.subcloud.name)
+            install.prep(consts.ANSIBLE_OVERRIDES_PATH,
+                         install_values)
+        except Exception as e:
+            db_api.subcloud_update(
+                self.context, strategy_step.subcloud_id,
+                deploy_status=consts.DEPLOY_STATE_PRE_INSTALL_FAILED)
+            self.error_log(strategy_step, e.message)
+            # TODO(jkung): cleanup to be implemented within SubcloudInstall
+            install.cleanup()
+            raise
+
+        ansible_subcloud_inventory_file = os.path.join(
+            consts.ANSIBLE_OVERRIDES_PATH,
+            strategy_step.subcloud.name + INVENTORY_FILE_POSTFIX)
+
+        # Create the ansible inventory for the upgrade subcloud
+        utils.create_subcloud_inventory(install_values,
+                                        ansible_subcloud_inventory_file)
+
+        # SubcloudInstall.prep creates data_install.yml (install overrides)
+        install_command = [
+            "ansible-playbook", ANSIBLE_SUBCLOUD_INSTALL_PLAYBOOK,
+            "-i", ansible_subcloud_inventory_file,
+            "-e", "@%s" % consts.ANSIBLE_OVERRIDES_PATH + "/" +
+                  strategy_step.subcloud.name + '/' + "install_values.yml"
+        ]
+
+        # Run the remote install playbook
+        db_api.subcloud_update(
+            self.context, strategy_step.subcloud_id,
+            deploy_status=consts.DEPLOY_STATE_INSTALLING)
+        try:
+            install.install(consts.DC_LOG_DIR, install_command)
+        except Exception as e:
+            db_api.subcloud_update(
+                self.context, strategy_step.subcloud_id,
+                deploy_status=consts.DEPLOY_STATE_INSTALL_FAILED)
+            self.error_log(strategy_step, e.message)
+            install.cleanup()
+            raise
+
+        db_api.subcloud_update(
+            self.context, strategy_step.subcloud_id,
+            deploy_status=consts.DEPLOY_STATE_INSTALLED)
+        install.cleanup()
+        LOG.info("Successfully installed subcloud %s" %
+                 strategy_step.subcloud.name)

distributedcloud/dcmanager/manager/subcloud_manager.py

@@ -39,11 +39,13 @@ from dccommon import consts as dccommon_consts
 from dccommon.drivers.openstack.keystone_v3 import KeystoneClient
 from dccommon.drivers.openstack.sysinv_v1 import SysinvClient
 from dccommon import kubeoperator
+from dccommon.subcloud_install import SubcloudInstall
 
 from dcorch.common import consts as dcorch_consts
 from dcorch.rpc import client as dcorch_rpc_client
 
 from dcmanager.common import consts
+from dcmanager.common.consts import INVENTORY_FILE_POSTFIX
 from dcmanager.common import context
 from dcmanager.common import exceptions
 from dcmanager.common.i18n import _
@@ -51,7 +53,6 @@ from dcmanager.common import manager
 from dcmanager.common import utils
 
 from dcmanager.db import api as db_api
-from dcmanager.manager.subcloud_install import SubcloudInstall
 
 from fm_api import constants as fm_const
 from fm_api import fm_api
@@ -63,12 +64,10 @@ LOG = logging.getLogger(__name__)
 ADDN_HOSTS_DC = 'dnsmasq.addn_hosts_dc'
 
 # Subcloud configuration paths
-INVENTORY_FILE_POSTFIX = '_inventory.yml'
 ANSIBLE_SUBCLOUD_PLAYBOOK = \
     '/usr/share/ansible/stx-ansible/playbooks/bootstrap.yml'
 ANSIBLE_SUBCLOUD_INSTALL_PLAYBOOK = \
     '/usr/share/ansible/stx-ansible/playbooks/install.yml'
-DC_LOG_DIR = '/var/log/dcmanager/'
 
 USERS_TO_REPLICATE = [
     'sysinv',
@@ -79,8 +78,6 @@ USERS_TO_REPLICATE = [
     'barbican',
     'dcmanager']
 
-SERVICES_USER = 'services'
-
 SC_INTERMEDIATE_CERT_DURATION = "87600h"
 SC_INTERMEDIATE_CERT_RENEW_BEFORE = "720h"
 CERT_NAMESPACE = "dc-cert"
@@ -191,7 +188,6 @@ class SubcloudManager(manager.Manager):
         """Add subcloud and notify orchestrators.
 
         :param context: request context object
-        :param name: name of subcloud to add
         :param payload: subcloud configuration
         """
         LOG.info("Adding subcloud %s." % payload['name'])
@@ -303,7 +299,8 @@ class SubcloudManager(manager.Manager):
                 dccommon_consts.ADMIN_USER_NAME)
             admin_project = m_ks_client.get_project_by_name(
                 dccommon_consts.ADMIN_PROJECT_NAME)
-            services_project = m_ks_client.get_project_by_name(SERVICES_USER)
+            services_project = m_ks_client.get_project_by_name(
+                dccommon_consts.SERVICES_USER_NAME)
             sysinv_user = m_ks_client.get_user_by_name(
                 dccommon_consts.SYSINV_USER_NAME)
             dcmanager_user = m_ks_client.get_user_by_name(
@@ -342,14 +339,14 @@ class SubcloudManager(manager.Manager):
                 ]
 
             del payload['sysadmin_password']
-
             payload['users'] = dict()
             for user in USERS_TO_REPLICATE:
                 payload['users'][user] = \
-                    str(keyring.get_password(user, SERVICES_USER))
+                    str(keyring.get_password(
+                        user, dccommon_consts.SERVICES_USER_NAME))
 
             # Create the ansible inventory for the new subcloud
-            self._create_subcloud_inventory(payload,
+            utils.create_subcloud_inventory(payload,
                                             ansible_subcloud_inventory_file)
 
             # create subcloud intermediate certificate and pass in keys
@@ -467,7 +464,7 @@ class SubcloudManager(manager.Manager):
                 context, subcloud.id,
                 deploy_status=consts.DEPLOY_STATE_INSTALLING)
             try:
-                install.install(DC_LOG_DIR, install_command)
+                install.install(consts.DC_LOG_DIR, install_command)
             except Exception as e:
                 db_api.subcloud_update(
                     context, subcloud.id,
@@ -490,7 +487,7 @@ class SubcloudManager(manager.Manager):
 
             # Run the ansible boostrap-subcloud playbook
             log_file = \
-                DC_LOG_DIR + subcloud.name + '_bootstrap_' + \
+                consts.DC_LOG_DIR + subcloud.name + '_bootstrap_' + \
                 str(datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')) \
                 + '.log'
             with open(log_file, "w") as f_out_log:
@@ -519,7 +516,7 @@ class SubcloudManager(manager.Manager):
                 context, subcloud.id,
                 deploy_status=consts.DEPLOY_STATE_DEPLOYING)
             log_file = \
-                DC_LOG_DIR + subcloud.name + '_deploy_' + \
+                consts.DC_LOG_DIR + subcloud.name + '_deploy_' + \
                 str(datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')) \
                 + '.log'
             with open(log_file, "w") as f_out_log:
@@ -569,35 +566,6 @@ class SubcloudManager(manager.Manager):
             # restart dnsmasq so it can re-read our addn_hosts file.
             os.system("pkill -HUP dnsmasq")
 
-    def _create_subcloud_inventory(self,
-                                   subcloud,
-                                   inventory_file):
-        """Create the inventory file for the specified subcloud"""
-
-        # Delete the file if it already exists
-        if os.path.isfile(inventory_file):
-            os.remove(inventory_file)
-
-        with open(inventory_file, 'w') as f_out_inventory:
-            f_out_inventory.write(
-                '---\n'
-                'all:\n'
-                '  vars:\n'
-                '    ansible_ssh_user: sysadmin\n'
-                '  hosts:\n'
-                '    ' + subcloud['name'] + ':\n'
-                '      ansible_host: ' +
-                subcloud['bootstrap-address'] + '\n'
-            )
-
-    def _delete_subcloud_inventory(self,
-                                   inventory_file):
-        """Delete the inventory file for the specified subcloud"""
-
-        # Delete the file if it exists
-        if os.path.isfile(inventory_file):
-            os.remove(inventory_file)
-
     def _write_subcloud_ansible_config(self, context, payload):
         """Create the override file for usage with the specified subcloud"""
 
@@ -736,7 +704,7 @@ class SubcloudManager(manager.Manager):
             raise e
 
         # Delete the ansible inventory for the new subcloud
-        self._delete_subcloud_inventory(ansible_subcloud_inventory_file)
+        utils.delete_subcloud_inventory(ansible_subcloud_inventory_file)
 
         # Delete the subcloud intermediate certificate
         SubcloudManager._delete_subcloud_cert(subcloud.name)

distributedcloud/dcmanager/tests/base.py

@@ -20,6 +20,7 @@
 # of an applicable Wind River license agreement.
 #
 
+import json
 import sqlalchemy
 
 from oslo_config import cfg
@@ -65,7 +66,8 @@ SUBCLOUD_SAMPLE_DATA_0 = [
     "10.10.10.12",  # external_oam_floating_address
     "testpass",  # sysadmin_password
     1,  # group_id
-    consts.DEPLOY_STATE_DONE  # deploy_status
+    consts.DEPLOY_STATE_DONE,  # deploy_status
+    json.dumps({'data_install': 'test data install values'}),  # data_install
 ]
 
 

distributedcloud/dcmanager/tests/unit/manager/test_subcloud_manager.py

@@ -30,6 +30,7 @@ import threading
 from dccommon import consts as dccommon_consts
 from dcmanager.common import consts
 from dcmanager.common import exceptions
+from dcmanager.common import utils as cutils
 from dcmanager.db.sqlalchemy import api as db_api
 from dcmanager.manager import subcloud_manager
 from dcmanager.tests import base
@@ -123,6 +124,7 @@ class Subcloud(object):
             data['external_oam_floating_address']
         self.systemcontroller_gateway_ip = \
             data['systemcontroller_gateway_address']
+        self.data_install = data['data_install']
         self.created_at = timeutils.utcnow()
         self.updated_at = timeutils.utcnow()
 
@@ -159,6 +161,7 @@ class TestSubcloudManager(base.DCManagerTestCase):
             'deploy_status': "not-deployed",
             'openstack_installed': False,
             'group_id': 1,
+            'data_install': 'data from install',
         }
         values.update(kwargs)
         return db_api.subcloud_create(ctxt, **values)
@@ -172,15 +175,13 @@ class TestSubcloudManager(base.DCManagerTestCase):
 
     @mock.patch.object(subcloud_manager.SubcloudManager,
                        '_create_intermediate_ca_cert')
-    @mock.patch.object(subcloud_manager.SubcloudManager,
-                       '_delete_subcloud_inventory')
+    @mock.patch.object(cutils, 'delete_subcloud_inventory')
     @mock.patch.object(subcloud_manager, 'KeystoneClient')
     @mock.patch.object(subcloud_manager, 'db_api')
     @mock.patch.object(subcloud_manager, 'SysinvClient')
     @mock.patch.object(subcloud_manager.SubcloudManager,
                        '_create_addn_hosts_dc')
-    @mock.patch.object(subcloud_manager.SubcloudManager,
-                       '_create_subcloud_inventory')
+    @mock.patch.object(cutils, 'create_subcloud_inventory')
     @mock.patch.object(subcloud_manager.SubcloudManager,
                        '_write_subcloud_ansible_config')
     @mock.patch.object(subcloud_manager,

distributedcloud/dcmanager/tests/utils.py

@@ -119,4 +119,5 @@ def create_subcloud_dict(data_list):
             'external_oam_floating_address': data_list[21],
             'sysadmin_password': data_list[22],
             'group_id': data_list[23],
-            'deploy_status': data_list[24]}
+            'deploy_status': data_list[24],
+            'data_install': data_list[25]}

distributedcloud/requirements.txt

@@ -41,6 +41,7 @@ oslo.utils>=3.20.0 # Apache-2.0
 oslo.versionedobjects>=1.17.0 # Apache-2.0
 sqlalchemy-migrate>=0.11.0 # Apache-2.0
 python-openstackclient!=3.10.0,>=3.3.0 # Apache-2.0
+python-barbicanclient>=4.5.2
 python-neutronclient>=6.3.0 # Apache-2.0
 python-cinderclient>=2.1.0 # Apache-2.0
 python-novaclient>=7.1.0 # Apache-2.0