393 lines
16 KiB
Python
393 lines
16 KiB
Python
#
|
|
# Copyright (c) 2020-2023 Wind River Systems, Inc.
|
|
#
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
#
|
|
import json
|
|
import keyring
|
|
import os
|
|
|
|
from oslo_serialization import base64
|
|
from tsconfig.tsconfig import SW_VERSION
|
|
|
|
from dccommon import consts as dccommon_consts
|
|
from dccommon.subcloud_install import SubcloudInstall
|
|
from dcmanager.common import consts
|
|
from dcmanager.common import utils
|
|
from dcmanager.db import api as db_api
|
|
from dcmanager.orchestrator.states.base import BaseState
|
|
|
|
|
|
class UpgradingSimplexState(BaseState):
|
|
"""Upgrade state for upgrading a simplex subcloud host"""
|
|
|
|
def __init__(self, region_name):
|
|
super(UpgradingSimplexState, self).__init__(
|
|
next_state=consts.STRATEGY_STATE_MIGRATING_DATA, region_name=region_name)
|
|
|
|
def perform_state_action(self, strategy_step):
|
|
"""Upgrade a simplex host on a subcloud
|
|
|
|
Returns the next state in the state machine on success.
|
|
Any exceptions raised by this method set the strategy to FAILED.
|
|
"""
|
|
|
|
self.info_log(strategy_step, "Performing simplex upgrade for subcloud")
|
|
|
|
subcloud_sysinv_client = None
|
|
subcloud_barbican_client = None
|
|
try:
|
|
subcloud_sysinv_client = self.get_sysinv_client(
|
|
strategy_step.subcloud.name)
|
|
subcloud_barbican_client = self.get_barbican_client(
|
|
strategy_step.subcloud.name)
|
|
except Exception:
|
|
# if getting the token times out, the orchestrator may have
|
|
# restarted and subcloud may be offline; so will attempt
|
|
# to use the persisted values
|
|
message = ("Simplex upgrade perform_subcloud_install "
|
|
"subcloud %s failed to get subcloud client" %
|
|
strategy_step.subcloud.name)
|
|
self.error_log(strategy_step, message)
|
|
pass
|
|
|
|
# Check whether subcloud is already re-installed with N+1 load
|
|
target_version = SW_VERSION
|
|
if self._check_load_already_active(
|
|
target_version, subcloud_sysinv_client):
|
|
self.info_log(strategy_step,
|
|
"Load:%s already active" % target_version)
|
|
return self.next_state
|
|
|
|
# Check whether subcloud supports redfish, and if not, fail.
|
|
# This needs to be inferred from absence of install_values as
|
|
# there is currrently no external api to query.
|
|
install_values = self.get_subcloud_upgrade_install_values(
|
|
strategy_step, subcloud_sysinv_client, subcloud_barbican_client)
|
|
|
|
local_ks_client = self.get_keystone_client()
|
|
|
|
# Upgrade the subcloud to the install_values image
|
|
self.perform_subcloud_install(
|
|
strategy_step, local_ks_client.session, install_values)
|
|
return self.next_state
|
|
|
|
def _check_load_already_active(self, target_version, subcloud_sysinv_client):
|
|
"""Check if the target_version is already active in subcloud"""
|
|
|
|
if subcloud_sysinv_client:
|
|
current_loads = subcloud_sysinv_client.get_loads()
|
|
for load in current_loads:
|
|
if (load.software_version == target_version and
|
|
load.state == 'active'):
|
|
return True
|
|
return False
|
|
|
|
def get_subcloud_upgrade_install_values(
|
|
self, strategy_step,
|
|
subcloud_sysinv_client, subcloud_barbican_client):
|
|
"""Get the data required for the remote subcloud install.
|
|
|
|
subcloud data_install are obtained from:
|
|
|
|
dcmanager database:
|
|
subcloud.subcloud_install_initial::for values which are persisted at subcloud_add time
|
|
|
|
INSTALL: (needed for upgrade install)
|
|
bootstrap_interface
|
|
bootstrap_vlan
|
|
bootstrap_address
|
|
bootstrap_address_prefix
|
|
install_type # could also be from host-show
|
|
|
|
# This option can be set to extend the installing stage timeout value
|
|
# wait_for_timeout: 3600
|
|
|
|
# Set this options for https with self-signed certificate
|
|
# no_check_certificate
|
|
|
|
# Override default filesystem device: also from host-show, but is static.
|
|
# rootfs_device: "/dev/disk/by-path/pci-0000:00:1f.2-ata-1.0"
|
|
# boot_device: "/dev/disk/by-path/pci-0000:00:1f.2-ata-1.0"
|
|
|
|
# Set rd.net.timeout.ipv6dad to increase timeout on IPv6 NIC up
|
|
# rd.net.timeout.ipv6dad: 300
|
|
|
|
BOOTSTRAP: (also needed for bootstrap)
|
|
# If the subcloud's bootstrap IP interface and the system controller are not on the
|
|
# same network then the customer must configure a default route or static route
|
|
# so that the Central Cloud can login bootstrap the newly installed subcloud.
|
|
# If nexthop_gateway is specified and the network_address is not specified then a
|
|
# default route will be configured. Otherwise, if a network_address is specified
|
|
then
|
|
# a static route will be configured.
|
|
nexthop_gateway: default_route_address
|
|
network_address: static_route_address
|
|
network_mask: static_route_mask
|
|
|
|
subcloud.data_upgrade - persist for upgrade duration
|
|
for values from subcloud online sysinv host-show (persist since upgrade-start)
|
|
bmc_address # sysinv_v1 host-show
|
|
bmc_username # sysinv_v1 host-show
|
|
for values from barbican_client (as barbican user), or from upgrade-start:
|
|
bmc_password --- obtain from barbican_client as barbican user
|
|
"""
|
|
|
|
install_values = {'name': strategy_step.subcloud.name}
|
|
|
|
install_values.update(
|
|
self._get_subcloud_upgrade_load_info(strategy_step))
|
|
|
|
upgrade_data_install_values = self._get_subcloud_upgrade_data_install(
|
|
strategy_step)
|
|
install_values.update(upgrade_data_install_values)
|
|
|
|
install_values.update(
|
|
self._get_subcloud_upgrade_data(
|
|
strategy_step, subcloud_sysinv_client, subcloud_barbican_client))
|
|
|
|
# Check bmc values
|
|
if not self._bmc_data_available(install_values):
|
|
if self._bmc_data_available(upgrade_data_install_values):
|
|
# It is possible the bmc data is only latched on install if it
|
|
# was not part of the deployment configuration
|
|
install_values.update({
|
|
'bmc_address':
|
|
upgrade_data_install_values.get('bmc_address'),
|
|
'bmc_username':
|
|
upgrade_data_install_values.get('bmc_username'),
|
|
'bmc_password':
|
|
upgrade_data_install_values.get('bmc_password'),
|
|
})
|
|
else:
|
|
message = ("Failed to get bmc credentials for subcloud %s" %
|
|
strategy_step.subcloud.name)
|
|
raise Exception(message)
|
|
|
|
self.info_log(strategy_step,
|
|
"get_subcloud_upgrade_data_install %s" % install_values)
|
|
return install_values
|
|
|
|
@staticmethod
|
|
def _bmc_data_available(bmc_values):
|
|
if (not bmc_values.get('bmc_username') or
|
|
not bmc_values.get('bmc_address') or
|
|
not bmc_values.get('bmc_password')):
|
|
return False
|
|
return True
|
|
|
|
def _get_subcloud_upgrade_load_info(self, strategy_step):
|
|
"""Get the subcloud upgrade load information"""
|
|
|
|
# The 'software_version' is the active running load on SystemController
|
|
matching_iso, _ = utils.get_vault_load_files(SW_VERSION)
|
|
if not matching_iso:
|
|
message = ("Failed to get upgrade load info for subcloud %s" %
|
|
strategy_step.subcloud.name)
|
|
raise Exception(message)
|
|
|
|
load_info = {'software_version': SW_VERSION,
|
|
'image': matching_iso}
|
|
|
|
return load_info
|
|
|
|
def _get_subcloud_upgrade_data_install(self, strategy_step):
|
|
"""Get subcloud upgrade data_install from persisted values"""
|
|
|
|
upgrade_data_install = {}
|
|
|
|
subcloud = db_api.subcloud_get(self.context, strategy_step.subcloud_id)
|
|
if not subcloud.data_install:
|
|
# Set the deploy status to pre-install-failed so it can be
|
|
# handled accordingly in pre check step.
|
|
message = ("Failed to get upgrade data from install")
|
|
db_api.subcloud_update(
|
|
self.context, strategy_step.subcloud_id,
|
|
deploy_status=consts.DEPLOY_STATE_PRE_INSTALL_FAILED,
|
|
error_description=message)
|
|
|
|
self.warn_log(strategy_step, message)
|
|
raise Exception(message)
|
|
|
|
data_install = json.loads(subcloud.data_install)
|
|
|
|
# base64 encoded sysadmin_password is default
|
|
upgrade_data_install.update({
|
|
'ansible_become_pass': consts.TEMP_SYSADMIN_PASSWORD,
|
|
'ansible_ssh_pass': consts.TEMP_SYSADMIN_PASSWORD,
|
|
})
|
|
# Get mandatory bootstrap info from data_install
|
|
# bootstrap_address is referenced in SubcloudInstall
|
|
# bootstrap-address is referenced in create_subcloud_inventory and
|
|
# subcloud manager.
|
|
# todo(jkung): refactor to just use one bootstrap address index
|
|
upgrade_data_install.update({
|
|
'bootstrap_interface': data_install.get('bootstrap_interface'),
|
|
'bootstrap-address': data_install.get('bootstrap_address'),
|
|
'bootstrap_address': data_install.get('bootstrap_address'),
|
|
'bootstrap_address_prefix': data_install.get('bootstrap_address_prefix'),
|
|
'bmc_username': data_install.get('bmc_username'),
|
|
'bmc_address': data_install.get('bmc_address'),
|
|
'bmc_password': data_install.get('bmc_password'),
|
|
})
|
|
|
|
persistent_size = data_install.get('persistent_size')
|
|
if persistent_size is not None:
|
|
upgrade_data_install.update({'persistent_size': persistent_size})
|
|
|
|
for p in dccommon_consts.OPTIONAL_INSTALL_VALUES:
|
|
if p in data_install:
|
|
upgrade_data_install.update({p: data_install.get(p)})
|
|
|
|
return upgrade_data_install
|
|
|
|
def _get_subcloud_upgrade_data(
|
|
self, strategy_step, subcloud_sysinv_client, subcloud_barbican_client):
|
|
"""Get the subcloud data required for upgrades.
|
|
|
|
In case the subcloud is no longer reachable, get upgrade_data from
|
|
persisted database values. For example, this may be required in
|
|
the scenario where the subcloud experiences an unexpected error
|
|
(e.g. loss of power) and this step needs to be rerun.
|
|
"""
|
|
|
|
volatile_data_install = {}
|
|
|
|
if subcloud_sysinv_client is None:
|
|
# subcloud is not reachable, use previously saved values
|
|
subcloud = db_api.subcloud_get(
|
|
self.context, strategy_step.subcloud_id)
|
|
if subcloud.data_upgrade:
|
|
return json.loads(subcloud.data_upgrade)
|
|
else:
|
|
message = ('Cannot retrieve upgrade data install '
|
|
'for subcloud: %s' %
|
|
strategy_step.subcloud.name)
|
|
raise Exception(message)
|
|
|
|
subcloud_system = subcloud_sysinv_client.get_system()
|
|
|
|
if subcloud_system.system_type != 'All-in-one':
|
|
message = ('subcloud %s install unsupported for system type: %s' %
|
|
(strategy_step.subcloud.name,
|
|
subcloud_system.system_type))
|
|
raise Exception(message)
|
|
|
|
host = subcloud_sysinv_client.get_host('controller-0')
|
|
|
|
install_type = self._get_install_type(host)
|
|
|
|
bmc_password = None
|
|
if subcloud_barbican_client:
|
|
bmc_password = subcloud_barbican_client.get_host_bmc_password(host.uuid)
|
|
if bmc_password:
|
|
# If the host is configured to store bmc in its barbican database,
|
|
# encode the password. Otherwise leave it as None and it will be
|
|
# replaced with the value retrieved from the dcmanager database.
|
|
bmc_password = base64.encode_as_text(bmc_password)
|
|
|
|
volatile_data_install.update({
|
|
'bmc_address': host.bm_ip,
|
|
'bmc_username': host.bm_username,
|
|
'bmc_password': bmc_password,
|
|
'install_type': install_type,
|
|
'boot_device': host.boot_device,
|
|
'rootfs_device': host.rootfs_device,
|
|
})
|
|
|
|
# Persist the volatile data
|
|
db_api.subcloud_update(
|
|
self.context, strategy_step.subcloud_id,
|
|
data_upgrade=json.dumps(volatile_data_install))
|
|
|
|
admin_password = str(keyring.get_password('CGCS', 'admin'))
|
|
volatile_data_install.update({'admin_password': admin_password})
|
|
|
|
return volatile_data_install
|
|
|
|
@staticmethod
|
|
def _get_install_type(host):
|
|
if 'lowlatency' in host.subfunctions.split(','):
|
|
lowlatency = True
|
|
else:
|
|
lowlatency = False
|
|
|
|
if 'graphical' in host.console.split(','): # graphical console
|
|
if lowlatency:
|
|
install_type = 5
|
|
else:
|
|
install_type = 3
|
|
else: # serial console
|
|
if lowlatency:
|
|
install_type = 4
|
|
else:
|
|
install_type = 2
|
|
return install_type
|
|
|
|
def perform_subcloud_install(self, strategy_step, session, install_values):
|
|
|
|
log_file = os.path.join(consts.DC_ANSIBLE_LOG_DIR, strategy_step.subcloud.name) + \
|
|
'_playbook_output.log'
|
|
db_api.subcloud_update(
|
|
self.context, strategy_step.subcloud_id,
|
|
deploy_status=consts.DEPLOY_STATE_PRE_INSTALL)
|
|
self.context.auth_token = session.get_token()
|
|
self.context.project = session.get_project_id()
|
|
try:
|
|
install = SubcloudInstall(
|
|
self.context, strategy_step.subcloud.name)
|
|
install.prep(dccommon_consts.ANSIBLE_OVERRIDES_PATH,
|
|
install_values)
|
|
except Exception as e:
|
|
db_api.subcloud_update(
|
|
self.context, strategy_step.subcloud_id,
|
|
deploy_status=consts.DEPLOY_STATE_PRE_INSTALL_FAILED,
|
|
error_description=str(e)[0:consts.ERROR_DESCRIPTION_LENGTH])
|
|
self.error_log(strategy_step, str(e))
|
|
# TODO(jkung): cleanup to be implemented within SubcloudInstall
|
|
install.cleanup()
|
|
raise
|
|
|
|
ansible_subcloud_inventory_file = os.path.join(
|
|
dccommon_consts.ANSIBLE_OVERRIDES_PATH,
|
|
strategy_step.subcloud.name + consts.INVENTORY_FILE_POSTFIX)
|
|
|
|
# Create the ansible inventory for the upgrade subcloud
|
|
utils.create_subcloud_inventory(install_values,
|
|
ansible_subcloud_inventory_file)
|
|
|
|
# SubcloudInstall.prep creates data_install.yml (install overrides)
|
|
install_command = [
|
|
"ansible-playbook", dccommon_consts.ANSIBLE_SUBCLOUD_INSTALL_PLAYBOOK,
|
|
"-i", ansible_subcloud_inventory_file,
|
|
"-e", "@%s" % dccommon_consts.ANSIBLE_OVERRIDES_PATH + "/" +
|
|
strategy_step.subcloud.name + '/' + "install_values.yml"
|
|
]
|
|
|
|
# Run the remote install playbook
|
|
db_api.subcloud_update(
|
|
self.context, strategy_step.subcloud_id,
|
|
availability_status=dccommon_consts.AVAILABILITY_OFFLINE,
|
|
deploy_status=consts.DEPLOY_STATE_INSTALLING)
|
|
try:
|
|
install.install(consts.DC_ANSIBLE_LOG_DIR, install_command)
|
|
except Exception as e:
|
|
# Detailed error message for subcloud error description field.
|
|
# Exception message for strategy_step detail.
|
|
msg = utils.find_ansible_error_msg(
|
|
strategy_step.subcloud.name, log_file, consts.DEPLOY_STATE_INSTALLING)
|
|
db_api.subcloud_update(
|
|
self.context, strategy_step.subcloud_id,
|
|
deploy_status=consts.DEPLOY_STATE_INSTALL_FAILED,
|
|
error_description=msg[0:consts.ERROR_DESCRIPTION_LENGTH])
|
|
self.error_log(strategy_step, msg)
|
|
self.error_log(strategy_step, str(e))
|
|
install.cleanup()
|
|
raise
|
|
|
|
db_api.subcloud_update(
|
|
self.context, strategy_step.subcloud_id,
|
|
deploy_status=consts.DEPLOY_STATE_INSTALLED)
|
|
install.cleanup()
|
|
self.info_log(strategy_step, "Successfully installed subcloud")
|