starlingx
/
distcloud
Code Issues Proposed changes

Display prestage status and versions in subcloud list 

The prestage release can be either the previous or current release for
a subcloud. Checking the prestage release individually on thousands of
subclouds is impractical. Therefore, a new column called "prestage
versions" is added to the output of the "dcmanager subcloud list'
command. Besides, decoupling prestage status from deploy status.

Test plan:
PASS: Successful subcloud prestage with specified 22.12 or 23.09
      release, and verified the correct "prestage versions" output of
      the "dcmanager subcloud list" command.
PASS: Successful prestage strategy with specified 22.12 or 23.09
      release, and verified the correct "prestage versions" output of
      the "dcmanager subcloud list".
PASS: Verified the "prestage status" at each prestage stage.
PASS: Verified that the 'deploy status' was updated to 'complete' after
      upgrading the system controller from version 22.12 to 23.09.
      This verification was performed specifically for those subclouds
      that had a 'prestage-complete' deploy status with the previous
      22.12 system controller. The new columns, "perstage status" and
      "prestage versions", were empty after upgrading.

Depends-on:
https://review.opendev.org/c/starlingx/ansible-playbooks/+/904546

Story: 2010611
Task: 49367

Change-Id: Ica65b1e2e8e44b96352e7d45439a2e7a9063f7c9
Signed-off-by: lzhu1 <li.zhu@windriver.com>
This commit is contained in:
Li Zhu 2024-01-02 22:30:26 -05:00
parent 91c9d301d8
commit 00a3e50e71
31 changed files with 322 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
api-ref/source/api-ref-dcmanager-v1.rst
View File

@ -78,6 +78,8 @@ Response
- deploy-status: deploy_status
- backup-status: backup_status
- backup-datetime: backup_datetime
- prestage-status: prestage_status
- prestage-versions: prestage_versions
- region-name: region_name
- openstack-installed: openstack_installed
- management-state: management_state
@ -222,6 +224,8 @@ This operation does not accept a request body.
- deploy-status: deploy_status
- backup-status: backup_status
- backup-datetime: backup_datetime
- prestage-status: prestage_status
- prestage-versions: prestage_versions
- openstack-installed: openstack_installed
- management-state: management_state
- systemcontroller-gateway-ip: systemcontroller_gateway_ip
@ -286,6 +290,8 @@ This operation does not accept a request body.
- deploy-status: deploy_status
- backup-status: backup_status
- backup-datetime: backup_datetime
- prestage-status: prestage_status
- prestage-versions: prestage_versions
- region-name: region_name
- openstack-installed: openstack_installed
- management-state: management_state
@ -399,6 +405,8 @@ Request Example
- deploy-status: deploy_status
- backup-status: backup_status
- backup-datetime: backup_datetime
- prestage-status: prestage_status
- prestage-versions: prestage_versions
- region-name: region_name
- openstack-installed: openstack_installed
- management-state: management_state
@ -473,6 +481,8 @@ Request Example
- deploy-status: deploy_status
- backup-status: backup_status
- backup-datetime: backup_datetime
- prestage-status: prestage_status
- prestage-versions: prestage_versions
- openstack-installed: openstack_installed
- management-state: management_state
- systemcontroller-gateway-ip: systemcontroller_gateway_ip
@ -544,6 +554,8 @@ Request Example
- deploy-status: deploy_status
- backup-status: backup_status
- backup-datetime: backup_datetime
- prestage-status: prestage_status
- prestage-versions: prestage_versions
- region-name: region_name
- openstack-installed: openstack_installed
- management-state: management_state
@ -809,6 +821,8 @@ This operation does not accept a request body.
- deploy-status: deploy_status
- backup-status: backup_status
- backup-datetime: backup_datetime
- prestage-status: prestage_status
- prestage-versions: prestage_versions
- region-name: region_name
- openstack-installed: openstack_installed
- management-state: management_state
@ -978,6 +992,8 @@ Request Example
- deploy-status: deploy_status
- backup-status: backup_status
- backup-datetime: backup_datetime
- prestage-status: prestage_status
- prestage-versions: prestage_versions
- region-name: region_name
- openstack-installed: openstack_installed
- management-state: management_state
@ -1090,6 +1106,8 @@ Request Example
- deploy-status: deploy_status
- backup-status: backup_status
- backup-datetime: backup_datetime
- prestage-status: prestage_status
- prestage-versions: prestage_versions
- region-name: region_name
- openstack-installed: openstack_installed
- management-state: management_state
@ -1815,6 +1833,8 @@ Request Example
- deploy-status: deploy_status
- backup-status: backup_status
- backup-datetime: backup_datetime
- prestage-status: prestage_status
- prestage-versions: prestage_versions
- error-description: error_description
- region-name: region_name
- management-subnet: management_subnet
@ -1883,6 +1903,8 @@ Request Example
- deploy-status: deploy_status
- backup-status: backup_status
- backup-datetime: backup_datetime
- prestage-status: prestage_status
- prestage-versions: prestage_versions
- error-description: error_description
- region-name: region_name
- management-subnet: management_subnet
@ -1951,6 +1973,8 @@ Request Example
- deploy-status: deploy_status
- backup-status: backup_status
- backup-datetime: backup_datetime
- prestage-status: prestage_status
- prestage-versions: prestage_versions
- region-name: region_name
- openstack-installed: openstack_installed
- management-state: management_state
@ -2025,6 +2049,8 @@ Request Example
- deploy-status: deploy_status
- backup-status: backup_status
- backup-datetime: backup_datetime
- prestage-status: prestage_status
- prestage-versions: prestage_versions
- region-name: region_name
- openstack-installed: openstack_installed
- management-state: management_state
@ -2093,6 +2119,8 @@ Request Example
- deploy-status: deploy_status
- backup-status: backup_status
- backup-datetime: backup_datetime
- prestage-status: prestage_status
- prestage-versions: prestage_versions
- region-name: region_name
- openstack-installed: openstack_installed
- management-state: management_state
@ -2161,6 +2189,8 @@ Request Example
- deploy-status: deploy_status
- backup-status: backup_status
- backup-datetime: backup_datetime
- prestage-status: prestage_status
- prestage-versions: prestage_versions
- region-name: region_name
- openstack-installed: openstack_installed
- management-state: management_state
@ -2237,6 +2267,8 @@ Request Example
- deploy-status: deploy_status
- backup-status: backup_status
- backup-datetime: backup_datetime
- prestage-status: prestage_status
- prestage-versions: prestage_versions
- error-description: error_description
- region-name: region_name
- management-subnet: management_subnet
@ -2774,6 +2806,8 @@ This operation does not accept a request body.
- deploy-status: deploy_status
- backup-status: backup_status
- backup-datetime: backup_datetime
- prestage-status: prestage_status
- prestage-versions: prestage_versions
- openstack-installed: openstack_installed
- management-state: management_state
- systemcontroller-gateway-ip: systemcontroller_gateway_ip

12
api-ref/source/parameters.yaml
View File

@ -478,6 +478,18 @@ prestage_software_version:
in: body
required: true
type: string
prestage_status:
description: |
The prestage status of the subcloud.
in: body
required: true
type: string
prestage_versions:
description: |
All of the prestage versions of the subcloud.
in: body
required: true
type: string
region_name:
description: |
The name provisioned for the subcloud (synonym for subcloud name).

2
api-ref/source/samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-abort-response.json
View File

@ -9,6 +9,8 @@
"deploy-status": "aborting-install",
"backup-status": null,
"backup-datetime": null,
"prestage-status": null,
"prestage-versions": null,
"region-name": "bbadb3e8e2ab473792c80ef09c5a12a4",
"description": "Ottawa Site",
"group_id": 1,

2
api-ref/source/samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-configure-response.json
View File

@ -9,6 +9,8 @@
"deploy-status": "complete",
"backup-status": "complete",
"backup-datetime": "2023-05-02 11:23:58.132134",
"prestage-status": "complete",
"prestage-versions": "22.12",
"region-name": "bbadb3e8e2ab473792c80ef09c5a12a4",
"description": "Ottawa Site",
"group_id": 1,

2
api-ref/source/samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-continue-response.json
View File

@ -9,6 +9,8 @@
"deploy-status": "complete",
"backup-status": "complete",
"backup-datetime": "2023-05-02 11:23:58.132134",
"prestage-status": "complete",
"prestage-versions": "22.12",
"region-name": "bbadb3e8e2ab473792c80ef09c5a12a4",
"description": "Ottawa Site",
"group_id": 1,

2
api-ref/source/samples/phased-subcloud-deploy/phased-subcloud-deploy-patch-resume-response.json
View File

@ -9,6 +9,8 @@
"deploy-status": "pre-install",
"backup-status": null,
"backup-datetime": null,
"prestage-status": null,
"prestage-versions": null,
"error-description": "No errors present",
"region-name": "bbadb3e8e2ab473792c80ef09c5a12a4",
"management-subnet": "192.168.102.0/24",

2
api-ref/source/samples/phased-subcloud-deploy/phased-subcloud-deploy-post-install-response.json
View File

@ -9,6 +9,8 @@
"deploy-status": "pre-install",
"backup-status": null,
"backup-datetime": null,
"prestage-status": null,
"prestage-versions": null,
"region-name": "bbadb3e8e2ab473792c80ef09c5a12a4",
"description": "Ottawa Site",
"group_id": 1,

2
api-ref/source/samples/phased-subcloud-deploy/phased-subcloud-deploy-post-response.json
View File

@ -9,6 +9,8 @@
"deploy-status": "not-deployed",
"backup-status": null,
"backup-datetime": null,
"prestage-status": null,
"prestage-versions": null,
"error-description": "No errors present",
"region-name": "bbadb3e8e2ab473792c80ef09c5a12a4",
"management-subnet": "192.168.102.0/24",

2
api-ref/source/samples/subcloud-backup/subcloud-create-backup-response.json
View File

@ -14,6 +14,8 @@
"error-description": "",
"backup-status": "complete",
"backup-datetime": "2022-07-08 11:23:58.132134",
"prestage-status": "complete",
"prestage-versions": "22.12",
"region-name": "bbadb3e8e2ab473792c80ef09c5a12a4",
"description": "Ottawa Site",
"group_id": 1,

2
api-ref/source/samples/subcloud-backup/subcloud-restore-backup-response.json
View File

@ -14,6 +14,8 @@
"error-description": "",
"backup-status": "complete",
"backup-datetime": "2022-07-08 11:23:58.132134",
"prestage-status": "complete",
"prestage-versions": "22.12",
"region-name": "bbadb3e8e2ab473792c80ef09c5a12a4",
"description": "Ottawa Site",
"group_id": 1,

2
api-ref/source/samples/subcloud-groups/subcloud-groups-get-subclouds-response.json
View File

@ -12,6 +12,8 @@
"deploy-status": "complete",
"backup-status": "complete",
"backup-datetime": "2022-07-08 11:23:58.132134",
"prestage-status": "complete",
"prestage-versions": "22.12",
"region-name": "bbadb3e8e2ab473792c80ef09c5a12a4",
"openstack-installed": false,
"management-state": "managed",

2
api-ref/source/samples/subcloud-peer-groups/subcloud-peer-groups-get-subclouds-response.json
View File

@ -10,6 +10,8 @@
"deploy-status": "secondary",
"backup-status": null,
"backup-datetime": null,
"prestage-status": null,
"prestage-versions": null,
"error-description": "No errors present",
"management-subnet": "192.168.38.0/24",
"management-start-ip": "192.168.38.2",

2
api-ref/source/samples/subclouds/subcloud-get-detail-response.json
View File

@ -9,6 +9,8 @@
"deploy-status": "complete",
"backup-status": "complete",
"backup-datetime": "2022-07-08 11:23:58.132134",
"prestage-status": "complete",
"prestage-versions": "21.12,22.12",
"openstack-installed": false,
"management-state": "managed",
"systemcontroller-gateway-ip": "192.168.204.101",

2
api-ref/source/samples/subclouds/subcloud-get-response.json
View File

@ -9,6 +9,8 @@
"deploy-status": "complete",
"backup-status": "complete",
"backup-datetime": "2022-07-08 11:23:58.132134",
"prestage-status": "complete",
"prestage-versions": "21.12,22.12",
"openstack-installed": false,
"management-state": "managed",
"systemcontroller-gateway-ip": "192.168.204.101",

2
api-ref/source/samples/subclouds/subcloud-patch-prestage-response.json
View File

@ -11,6 +11,8 @@
"deploy-status": "complete",
"backup-status": "complete",
"backup-datetime": "2022-07-08 11:23:58.132134",
"prestage-status": "complete",
"prestage-versions": "21.12,22.12",
"region-name": "bbadb3e8e2ab473792c80ef09c5a12a4",
"description": "Ottawa Site",
"group_id": 1,

2
api-ref/source/samples/subclouds/subcloud-patch-redeploy-response.json
View File

@ -11,6 +11,8 @@
"deploy-status": "pre-install",
"backup-status": "complete",
"backup-datetime": "2022-07-08 11:23:58.132134",
"prestage-status": "complete",
"prestage-versions": "21.12,22.12",
"description": "Ottawa Site",
"group_id": 1,
"location": "YOW",

2
api-ref/source/samples/subclouds/subcloud-patch-response.json
View File

@ -9,6 +9,8 @@
"deploy-status": "complete",
"backup-status": "complete",
"backup-datetime": "2022-07-08 11:23:58.132134",
"prestage-status": "complete",
"prestage-versions": "21.12,22.12",
"region-name": "bbadb3e8e2ab473792c80ef09c5a12a4",
"description": "Ottawa Site",
"group_id": 1,

2
api-ref/source/samples/subclouds/subclouds-get-response.json
View File

@ -12,6 +12,8 @@
"deploy-status": "complete",
"backup-status": "complete",
"backup-datetime": "2022-07-08 11:23:58.132134",
"prestage-status": "complete",
"prestage-versions": "21.12,22.12",
"region-name": "bbadb3e8e2ab473792c80ef09c5a12a4",
"openstack-installed": false,
"management-state": "managed",

12
distributedcloud/dcmanager/api/controllers/v1/phased_subcloud_deploy.py
View File

@ -20,7 +20,6 @@ from dcmanager.common.context import RequestContext
from dcmanager.common import exceptions
from dcmanager.common.i18n import _
from dcmanager.common import phased_subcloud_deploy as psd_common
from dcmanager.common import prestage
from dcmanager.common import utils
from dcmanager.db import api as db_api
from dcmanager.db.sqlalchemy import models
@ -301,11 +300,14 @@ class PhasedSubcloudDeployController(object):
if not payload:
pecan.abort(400, _('Body required'))
if not (subcloud.deploy_status in VALID_STATES_FOR_DEPLOY_CONFIG or
prestage.is_deploy_status_prestage(subcloud.deploy_status)):
if subcloud.deploy_status not in VALID_STATES_FOR_DEPLOY_CONFIG:
allowed_states_str = ', '.join(VALID_STATES_FOR_DEPLOY_CONFIG)
pecan.abort(400, _('Subcloud deploy status must be either '
'%s or prestage-...') % allowed_states_str)
pecan.abort(400, _('Subcloud deploy status must be %s') %
allowed_states_str)
if subcloud.prestage_status in consts.STATES_FOR_ONGOING_PRESTAGE:
pecan.abort(400, _('Subcloud prestage is ongoing %s') %
subcloud.prestage_status)
psd_common.populate_payload_with_pre_existing_data(
payload, subcloud, SUBCLOUD_CONFIG_GET_FILE_CONTENTS)

21
distributedcloud/dcmanager/api/controllers/v1/subclouds.py
View File

@ -601,16 +601,16 @@ class SubcloudsController(object):
# Rename the subcloud
new_subcloud_name = payload.get('name')
if new_subcloud_name is not None:
# To be renamed the subcloud must be in unmanaged and valid deploy
# state
# To be renamed the subcloud must be in unmanaged, valid deploy
# state, and no going prestage
if (subcloud.management_state !=
dccommon_consts.MANAGEMENT_UNMANAGED or
subcloud.deploy_status not in
consts.STATES_FOR_SUBCLOUD_RENAME):
msg = (
'Subcloud %s must be unmanaged and in a valid deploy state '
'for the subcloud rename operation.' % subcloud.name
)
subcloud.deploy_status != consts.DEPLOY_STATE_DONE or
subcloud.prestage_status in
consts.STATES_FOR_ONGOING_PRESTAGE):
msg = ('Subcloud %s must be deployed, unmanaged and '
'no ongoing prestage for the subcloud rename '
'operation.' % subcloud.name)
pecan.abort(400, msg)
# Validates new name
@ -890,8 +890,9 @@ class SubcloudsController(object):
try:
self.dcmanager_rpc_client.prestage_subcloud(context, payload)
# local update to deploy_status - this is just for CLI response:
subcloud.deploy_status = consts.PRESTAGE_STATE_PACKAGES
# local update to prestage_status - this is just for
# CLI response:
subcloud.prestage_status = consts.PRESTAGE_STATE_PACKAGES
subcloud_dict = db_api.subcloud_db_model_to_dict(subcloud)
subcloud_dict.update(

8
distributedcloud/dcmanager/audit/subcloud_audit_worker_manager.py
View File

@ -33,7 +33,6 @@ from dcmanager.common import context
from dcmanager.common import exceptions
from dcmanager.common.i18n import _
from dcmanager.common import manager
from dcmanager.common import prestage
from dcmanager.common import scheduler
from dcmanager.db import api as db_api
from dcmanager.rpc import client as dcmanager_rpc_client
@ -124,9 +123,7 @@ class SubcloudAuditWorkerManager(manager.Manager):
consts.DEPLOY_STATE_RESTORING,
consts.DEPLOY_STATE_RESTORE_PREP_FAILED,
consts.DEPLOY_STATE_RESTORE_FAILED,
consts.DEPLOY_STATE_REHOME_PENDING]
and not prestage.is_deploy_status_prestage(
subcloud.deploy_status)) or (
consts.DEPLOY_STATE_REHOME_PENDING]) or (
(subcloud.deploy_status in [
consts.DEPLOY_STATE_INSTALLING,
consts.DEPLOY_STATE_REHOME_PENDING])
@ -431,8 +428,7 @@ class SubcloudAuditWorkerManager(manager.Manager):
# Avoid a network call to sysinv here if possible:
# If prestaging is active we can assume that the subcloud
# is online (otherwise prestaging will fail):
if subcloud.deploy_status in (consts.PRESTAGE_STATE_PACKAGES,
consts.PRESTAGE_STATE_IMAGES):
if subcloud.prestage_status in consts.STATES_FOR_ONGOING_PRESTAGE:
avail_to_set = dccommon_consts.AVAILABILITY_ONLINE
else:
avail_to_set = self._get_subcloud_availability_status(

14
distributedcloud/dcmanager/common/consts.py
View File

@ -325,8 +325,12 @@ UPGRADE_STATE_ACTIVATION_COMPLETE = 'activation-complete'
# Prestage States
PRESTAGE_STATE_PACKAGES = STRATEGY_STATE_PRESTAGE_PACKAGES
PRESTAGE_STATE_IMAGES = STRATEGY_STATE_PRESTAGE_IMAGES
PRESTAGE_STATE_FAILED = 'prestage-failed'
PRESTAGE_STATE_COMPLETE = 'prestage-complete'
PRESTAGE_STATE_FAILED = 'failed'
PRESTAGE_STATE_COMPLETE = 'complete'
# States to indicate if a prestage operation is currently in progress
STATES_FOR_ONGOING_PRESTAGE = [PRESTAGE_STATE_PACKAGES,
PRESTAGE_STATE_IMAGES]
# Alarm aggregation
ALARMS_DISABLED = "disabled"
@ -406,10 +410,6 @@ DEFAULT_PERSISTENT_SIZE = 30000
PLATFORM_RETRY_MAX_ATTEMPTS = 5
PLATFORM_RETRY_SLEEP_MILLIS = 5000
# States to reject when processing a subcloud-backup create request
VALID_DEPLOY_STATES_FOR_BACKUP = [DEPLOY_STATE_DONE,
PRESTAGE_STATE_COMPLETE]
# States to reject when processing a subcloud-backup restore request
INVALID_DEPLOY_STATES_FOR_RESTORE = [DEPLOY_STATE_CREATING,
DEPLOY_STATE_PRE_INSTALL,
@ -455,8 +455,6 @@ SUPPORTED_UPGRADES_METADATA_FILE_PATH = '/usr/rootdirs/opt/upgrades/metadata.xml
# Required for subcloud name configuration
CERT_MON_HTTP_AGENT = 'cert-mon/1.0'
OS_REGION_NAME = "OS_REGION_NAME"
STATES_FOR_SUBCLOUD_RENAME = [DEPLOY_STATE_DONE,
PRESTAGE_STATE_COMPLETE]
# Required for GEO-redundancy
# User-Agent check for subcloud by region_name request.

59
distributedcloud/dcmanager/common/prestage.py
View File

@ -1,4 +1,4 @@
# Copyright (c) 2022-2023 Wind River Systems, Inc.
# Copyright (c) 2022-2024 Wind River Systems, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -50,13 +50,9 @@ ANSIBLE_PRESTAGE_SUBCLOUD_PACKAGES_PLAYBOOK = \
ANSIBLE_PRESTAGE_SUBCLOUD_IMAGES_PLAYBOOK = \
"/usr/share/ansible/stx-ansible/playbooks/prestage_images.yml"
ANSIBLE_PRESTAGE_INVENTORY_SUFFIX = '_prestage_inventory.yml'
def is_deploy_status_prestage(deploy_status):
return deploy_status in (consts.PRESTAGE_STATE_PACKAGES,
consts.PRESTAGE_STATE_IMAGES,
consts.PRESTAGE_STATE_FAILED,
consts.PRESTAGE_STATE_COMPLETE)
PRINT_PRESTAGE_VERSIONS_TASK = \
'prestage\/prestage-versions : Print prestage versions'
PRESTAGE_VERSIONS_KEY_STR = 'prestage_versions:'
def _get_system_controller_upgrades():
@ -138,17 +134,24 @@ def initial_subcloud_validate(subcloud, installed_loads, software_version):
details="Prestage operation is not allowed while"
" backup is in progress.")
allowed_deploy_states = [consts.DEPLOY_STATE_DONE,
consts.PRESTAGE_STATE_FAILED,
consts.PRESTAGE_STATE_COMPLETE]
if subcloud.deploy_status not in allowed_deploy_states:
if subcloud.deploy_status != consts.DEPLOY_STATE_DONE:
raise exceptions.PrestagePreCheckFailedException(
subcloud=subcloud.name,
orch_skip=True,
details="Prestage operation is not allowed when"
" subcloud deploy is not completed.")
allowed_prestage_states = [consts.PRESTAGE_STATE_FAILED,
consts.PRESTAGE_STATE_COMPLETE]
if (subcloud.prestage_status and
(subcloud.prestage_status not in allowed_prestage_states)):
raise exceptions.PrestagePreCheckFailedException(
subcloud=subcloud.name,
orch_skip=True,
details="Prestage operation is only allowed while"
" subcloud deploy status is one of: %s."
" The current deploy status is %s."
% (', '.join(allowed_deploy_states), subcloud.deploy_status))
" subcloud prestage status is one of: %s."
" The current prestage status is %s."
% (', '.join(allowed_prestage_states), subcloud.prestage_status))
# The request software version must be either the same as the software version
# of the subcloud or any active/inactive/imported load on the system controller
@ -212,20 +215,21 @@ def validate_prestage(subcloud, payload):
def prestage_start(context, subcloud_id):
subcloud = db_api.subcloud_update(
context, subcloud_id,
deploy_status=consts.PRESTAGE_STATE_PACKAGES)
prestage_status=consts.PRESTAGE_STATE_PACKAGES)
return subcloud
def prestage_complete(context, subcloud_id):
def prestage_complete(context, subcloud_id, prestage_versions):
db_api.subcloud_update(
context, subcloud_id,
deploy_status=consts.PRESTAGE_STATE_COMPLETE)
prestage_status=consts.PRESTAGE_STATE_COMPLETE,
prestage_versions=prestage_versions)
def prestage_fail(context, subcloud_id):
db_api.subcloud_update(
context, subcloud_id,
deploy_status=consts.PRESTAGE_STATE_FAILED)
prestage_status=consts.PRESTAGE_STATE_FAILED)
def is_local(subcloud_version, specified_version):
@ -275,11 +279,16 @@ def prestage_subcloud(context, payload):
def _prestage_standalone_thread(context, subcloud, payload):
"""Run the prestage operations inside a separate thread"""
log_file = utils.get_subcloud_ansible_log_file(subcloud.name)
try:
prestage_packages(context, subcloud, payload)
prestage_images(context, subcloud, payload)
# Get the prestage versions from the logs generated by
# the prestage packages playbook
prestage_versions = utils.get_msg_output_info(
log_file, PRINT_PRESTAGE_VERSIONS_TASK, PRESTAGE_VERSIONS_KEY_STR)
prestage_complete(context, subcloud.id)
prestage_images(context, subcloud, payload)
prestage_complete(context, subcloud.id, prestage_versions)
LOG.info("Prestage complete: %s", subcloud.name)
except Exception:
@ -314,7 +323,7 @@ def _get_prestage_subcloud_info(subcloud):
def _run_ansible(context, prestage_command, phase,
subcloud, deploy_status,
subcloud, prestage_status,
sysadmin_password, oam_floating_ip,
software_version,
ansible_subcloud_inventory_file,
@ -328,9 +337,7 @@ def _run_ansible(context, prestage_command, phase,
db_api.subcloud_update(context,
subcloud.id,
deploy_status=deploy_status)
log_file = os.path.join(consts.DC_ANSIBLE_LOG_DIR, subcloud.name) + \
'_playbook_output.log'
prestage_status=prestage_status)
# Create the ansible inventory for the new subcloud
utils.create_subcloud_inventory_with_admin_creds(
@ -339,6 +346,8 @@ def _run_ansible(context, prestage_command, phase,
oam_floating_ip,
ansible_pass=utils.decode_and_normalize_passwd(sysadmin_password))
log_file = utils.get_subcloud_ansible_log_file(subcloud.name)
try:
ansible = AnsiblePlaybook(subcloud.name)
ansible.run_playbook(log_file, prestage_command, timeout=timeout_seconds,

109
distributedcloud/dcmanager/common/utils.py
View File

@ -788,7 +788,6 @@ def find_ansible_error_msg(subcloud_name, log_file, stage=None):
error_found = False
error_msg = []
failed_task = ''
files_for_search = []
cmd_1 = 'awk'
# awk command to get the information iside the last match found
@ -801,23 +800,16 @@ def find_ansible_error_msg(subcloud_name, log_file, stage=None):
f = f ? (f "\\n" $0) : $0} # assign or append to f
END {print f}
''')
# necessary check since is possible to have
# the error in rotated ansible log
log_file_temp = log_file + '.1'
if os.path.exists(log_file_temp):
files_for_search.append(log_file_temp)
if os.path.exists(log_file):
files_for_search.append(log_file)
else:
files_for_search.append(log_file)
if len(files_for_search) < 2:
cmd_list = ([cmd_1, cmd_2, files_for_search[0]])
else:
cmd_list = ([cmd_1, cmd_2, files_for_search[0], files_for_search[1]])
try:
# necessary check since is possible to have
# the error in rotated ansible log
files_for_search = add_latest_rotated_file(log_file)
if len(files_for_search) < 2:
cmd_list = ([cmd_1, cmd_2, files_for_search[0]])
else:
cmd_list = ([cmd_1, cmd_2, files_for_search[0], files_for_search[1]])
error_msg_raw = subprocess.check_output(
cmd_list,
stderr=subprocess.STDOUT).decode('utf-8')
@ -851,6 +843,38 @@ def find_ansible_error_msg(subcloud_name, log_file, stage=None):
return msg
def add_latest_rotated_file(log_file):
"""Find the latest rotated file for the given log file.
Check the existence of the given log file with its latest rotated file.
Returns the log file itself if it exists and the latest rotated file
doesn't exist;
or the log file and its latest rotated file if both exist;
or the latest rotated file only if it exists but the log file itself
doesn't exit.
Raises exception if both of the log file and its latest rotated file
don't exist.
"""
log_files = []
# the latest rotated log file
log_file_temp = log_file + '.1'
if os.path.exists(log_file_temp):
log_files.append(log_file_temp)
if os.path.exists(log_file):
log_files.append(log_file)
if len(log_files) == 0:
raise Exception("Log file %s and its latest rotated file don't exist."
% log_file)
return log_files
def get_failed_task(files):
"""Get last task failed
@ -933,9 +957,10 @@ def is_valid_for_backup_operation(operation, subcloud, bootstrap_address_dict=No
def _is_valid_for_backup_create(subcloud):
if subcloud.availability_status != dccommon_consts.AVAILABILITY_ONLINE \
or subcloud.management_state != dccommon_consts.MANAGEMENT_MANAGED \
or subcloud.deploy_status not in consts.VALID_DEPLOY_STATES_FOR_BACKUP:
msg = ('Subcloud %s must be online, managed and have valid '
'deploy-status for the subcloud-backup '
or subcloud.deploy_status != consts.DEPLOY_STATE_DONE \
or subcloud.prestage_status in consts.STATES_FOR_ONGOING_PRESTAGE:
msg = ('Subcloud %s must be deployed, online, managed, '
'and no ongoing prestage for the subcloud-backup '
'create operation.' % subcloud.name)
raise exceptions.ValidateFail(msg)
@ -1559,3 +1584,47 @@ def get_local_system():
endpoint=endpoint)
system = sysinv_client.get_system()
return system
def get_msg_output_info(log_file, target_task, target_str):
"""Get msg output by searching the target string from the given task.
It receives an ansible log file and searches for the last msg output
matching the target string from the given task.
Returns the msg output
"""
# awk command to get the last occurrence string after 'msg: {target_str}'
# between 'TASK \[{target_task}' and 'PLAY RECAP' delimiters.
awk_script = f'''
/TASK \[{target_task}/,/PLAY RECAP/ {{
if ($0 ~ /msg: \'{target_str}(.+)\'/) {{
result = $0
}}
}}
END {{
if (result) {{
match(result, /msg: \'{target_str}(.+)\'/, arr)
print arr[1]
}}
}}
'''
try:
# necessary check since is possible to have
# the message in rotated ansible log
files_for_search = add_latest_rotated_file(log_file)
awk_cmd = ['awk', awk_script] + files_for_search
# Run the AWK script using subprocess
result = subprocess.run(
awk_cmd, capture_output=True, text=True, check=True)
return result.stdout.strip()
except Exception as e:
LOG.error("Failed getting msg output by searching '%s' "
"from task '%s': %s" % (target_str, target_task, e))
return None
def get_subcloud_ansible_log_file(subcloud_name):
return os.path.join(consts.DC_ANSIBLE_LOG_DIR,
subcloud_name + '_playbook_output.log')

7
distributedcloud/dcmanager/db/api.py
View File

@ -119,6 +119,8 @@ def subcloud_db_model_to_dict(subcloud):
"management-end-ip": subcloud.management_end_ip,
"management-gateway-ip": subcloud.management_gateway_ip,
"openstack-installed": subcloud.openstack_installed,
"prestage-status": subcloud.prestage_status,
"prestage-versions": subcloud.prestage_versions,
"systemcontroller-gateway-ip":
subcloud.systemcontroller_gateway_ip,
"data_install": subcloud.data_install,
@ -193,7 +195,8 @@ def subcloud_update(
backup_datetime=None, error_description=None, openstack_installed=None,
group_id=None, data_install=None, data_upgrade=None,
first_identity_sync_complete=None, systemcontroller_gateway_ip=None,
peer_group_id=None, rehome_data=None, rehomed=None
peer_group_id=None, rehome_data=None, rehomed=None,
prestage_status=None, prestage_versions=None
):
"""Update a subcloud or raise if it does not exist."""
return IMPL.subcloud_update(
@ -203,7 +206,7 @@ def subcloud_update(
audit_fail_count, deploy_status, backup_status, backup_datetime,
error_description, openstack_installed, group_id, data_install, data_upgrade,
first_identity_sync_complete, systemcontroller_gateway_ip, peer_group_id,
rehome_data, rehomed
rehome_data, rehomed, prestage_status, prestage_versions
)

7
distributedcloud/dcmanager/db/sqlalchemy/api.py
View File

@ -422,7 +422,8 @@ def subcloud_update(context, subcloud_id, management_state=None,
first_identity_sync_complete=None,
systemcontroller_gateway_ip=None,
peer_group_id=None,
rehome_data=None, rehomed=None):
rehome_data=None, rehomed=None,
prestage_status=None, prestage_versions=None):
with write_session() as session:
subcloud_ref = subcloud_get(context, subcloud_id)
if management_state is not None:
@ -477,6 +478,10 @@ def subcloud_update(context, subcloud_id, management_state=None,
subcloud_ref.rehome_data = rehome_data
if rehomed is not None:
subcloud_ref.rehomed = rehomed
if prestage_status is not None:
subcloud_ref.prestage_status = prestage_status
if prestage_versions is not None:
subcloud_ref.prestage_versions = prestage_versions
subcloud_ref.save(session)
return subcloud_ref

30
distributedcloud/dcmanager/db/sqlalchemy/migrate_repo/versions/017_add_subcloud_prestage_columns.py Normal file
View File

@ -0,0 +1,30 @@
#
# Copyright (c) 2024 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
from sqlalchemy import Column, MetaData, Table, String
def upgrade(migrate_engine):
meta = MetaData()
meta.bind = migrate_engine
subclouds = Table('subclouds', meta, autoload=True)
# Add the 'prestage_status' and 'prestage_versions' columns to
# the subclouds table.
subclouds.create_column(Column('prestage_status', String(255)))
subclouds.create_column(Column('prestage_versions', String(255)))
# Update existing subclouds that have the old prestaging deploy status
subclouds.update().where( # pylint: disable=E1120
subclouds.c.deploy_status.like('prestage%')).values(
{'deploy_status': 'complete'}).execute()
return True
def downgrade(migrate_engine):
raise NotImplementedError('Database downgrade is unsupported.')

2
distributedcloud/dcmanager/db/sqlalchemy/models.py
View File

@ -186,6 +186,8 @@ class Subcloud(BASE, DCManagerBase):
peer_group_id = Column(Integer,
ForeignKey('subcloud_peer_group.id'))
rehome_data = Column(Text())
prestage_status = Column(String(255))
prestage_versions = Column(String(255))
# multiple subclouds can be in a particular group
group_id = Column(Integer,

30
distributedcloud/dcmanager/manager/subcloud_manager.py
View File

@ -135,8 +135,6 @@ TRANSITORY_STATES = {
consts.DEPLOY_STATE_RESTORING: consts.DEPLOY_STATE_RESTORE_FAILED,
consts.DEPLOY_STATE_PRE_REHOME: consts.DEPLOY_STATE_REHOME_PREP_FAILED,
consts.DEPLOY_STATE_REHOMING: consts.DEPLOY_STATE_REHOME_FAILED,
consts.PRESTAGE_STATE_PACKAGES: consts.PRESTAGE_STATE_FAILED,
consts.PRESTAGE_STATE_IMAGES: consts.PRESTAGE_STATE_FAILED,
# The next two states are needed due to upgrade scenario:
# TODO(gherzman): remove states when they are no longer needed
consts.DEPLOY_STATE_PRE_DEPLOY: consts.DEPLOY_STATE_PRE_CONFIG_FAILED,
@ -149,6 +147,11 @@ TRANSITORY_BACKUP_STATES = {
consts.BACKUP_STATE_IN_PROGRESS: consts.BACKUP_STATE_FAILED
}
TRANSITORY_PRESTAGE_STATES = {
consts.PRESTAGE_STATE_PACKAGES: consts.PRESTAGE_STATE_FAILED,
consts.PRESTAGE_STATE_IMAGES: consts.PRESTAGE_STATE_FAILED
}
MAX_PARALLEL_SUBCLOUD_BACKUP_CREATE = 250
MAX_PARALLEL_SUBCLOUD_BACKUP_DELETE = 250
MAX_PARALLEL_SUBCLOUD_BACKUP_RESTORE = 100
@ -2744,16 +2747,13 @@ class SubcloudManager(manager.Manager):
# No need for further validation
return
deploy_status_complete = (
subcloud.deploy_status == consts.DEPLOY_STATE_DONE
or prestage.is_deploy_status_prestage(subcloud.deploy_status)
)
allowed_deploy_transition = (
subcloud.deploy_status == consts.DEPLOY_STATE_REHOME_PENDING
and new_deploy_status == consts.DEPLOY_STATE_DONE
)
if not deploy_status_complete and not allowed_deploy_transition:
if (subcloud.deploy_status != consts.DEPLOY_STATE_DONE and
not allowed_deploy_transition):
msg = (f"Unable to manage {subcloud.name}: its deploy_status "
f"must be either '{consts.DEPLOY_STATE_DONE}' or "
f"'{consts.DEPLOY_STATE_REHOME_PENDING}'")
@ -3269,9 +3269,12 @@ class SubcloudManager(manager.Manager):
# Identify subclouds in transitory states
new_deploy_status = TRANSITORY_STATES.get(subcloud.deploy_status)
new_backup_status = TRANSITORY_BACKUP_STATES.get(subcloud.backup_status)
new_prestage_status = TRANSITORY_PRESTAGE_STATES.get(
subcloud.prestage_status)
# update deploy and backup states to the corresponding failure states
if new_deploy_status or new_backup_status:
# update deploy, backup and prestage states to
# the corresponding failure states
if new_deploy_status or new_backup_status or new_prestage_status:
if new_deploy_status:
LOG.info("Changing subcloud %s deploy status from %s to %s."
% (subcloud.name, subcloud.deploy_status,
@ -3281,11 +3284,18 @@ class SubcloudManager(manager.Manager):
% (subcloud.name, subcloud.backup_status,
new_backup_status))
if new_prestage_status:
LOG.info("Changing subcloud %s prestage status from %s"
" to %s."
% (subcloud.name, subcloud.prestage_status,
new_prestage_status))
db_api.subcloud_update(
self.context,
subcloud.id,
deploy_status=new_deploy_status or subcloud.deploy_status,
backup_status=new_backup_status or subcloud.backup_status
backup_status=new_backup_status or subcloud.backup_status,
prestage_status=new_prestage_status or subcloud.prestage_status
)
@staticmethod

18
distributedcloud/dcmanager/orchestrator/states/prestage/states.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2022-2023 Wind River Systems, Inc.
# Copyright (c) 2022-2024 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -33,10 +33,10 @@ class PrestageState(BaseState):
try:
self._do_state_action(strategy_step)
except exceptions.StrategySkippedException:
# Move deploy_status back to complete (nothing has changed)
# Move prestage_status back to None (nothing has changed)
db_api.subcloud_update(
self.context, strategy_step.subcloud.id,
deploy_status=consts.DEPLOY_STATE_DONE)
prestage_status=None)
raise
except Exception:
prestage.prestage_fail(self.context, strategy_step.subcloud.id)
@ -141,6 +141,15 @@ class PrestageImagesState(PrestageState):
region_name=region_name)
def _do_state_action(self, strategy_step):
log_file = utils.get_subcloud_ansible_log_file(
strategy_step.subcloud.name)
# Get the prestage versions from the ansible playbook logs
# generated by the previous step - prestage packages.
prestage_versions = utils.get_msg_output_info(
log_file,
prestage.PRINT_PRESTAGE_VERSIONS_TASK,
prestage.PRESTAGE_VERSIONS_KEY_STR)
extra_args = utils.get_sw_update_strategy_extra_args(self.context)
payload = {
'sysadmin_password': extra_args['sysadmin_password'],
@ -153,4 +162,5 @@ class PrestageImagesState(PrestageState):
extra_args.get(consts.PRESTAGE_SOFTWARE_VERSION)})
prestage.prestage_images(self.context, strategy_step.subcloud, payload)
self.info_log(strategy_step, "Images finished")
prestage.prestage_complete(self.context, strategy_step.subcloud.id)
prestage.prestage_complete(
self.context, strategy_step.subcloud.id, prestage_versions)

36
distributedcloud/dcmanager/tests/unit/manager/test_subcloud_manager.py
View File

@ -2264,8 +2264,8 @@ class TestSubcloudManager(base.DCManagerTestCase):
mock_run_playbook.assert_called_once()
mock_is_healthy.assert_called_once()
# Verify that subcloud has the correct deploy status
# consts.PRESTAGE_STATE_PACKAGES
# Verify that subcloud has the correct backup status
# consts.BACKUP_STATE_PRE_BACKUP
updated_subcloud = db_api.subcloud_get_by_name(self.ctx, subcloud.name)
self.assertEqual(consts.BACKUP_STATE_PRE_BACKUP,
updated_subcloud.backup_status)
@ -2293,8 +2293,8 @@ class TestSubcloudManager(base.DCManagerTestCase):
mock_parallel_group_operation.assert_called_once()
# Verify that subcloud has the correct deploy status
# consts.PRESTAGE_STATE_PACKAGES
# Verify that subcloud has the correct backup status
# consts.BACKUP_STATE_VALIDATE_FAILED
updated_subcloud = db_api.subcloud_get_by_name(self.ctx, subcloud.name)
self.assertEqual(consts.BACKUP_STATE_VALIDATE_FAILED,
updated_subcloud.backup_status)
@ -2322,8 +2322,8 @@ class TestSubcloudManager(base.DCManagerTestCase):
mock_parallel_group_operation.assert_called_once()
# Verify that subcloud has the correct deploy status
# consts.PRESTAGE_STATE_PACKAGES
# Verify that subcloud has the correct backup status
# consts.BACKUP_STATE_VALIDATE_FAILED
updated_subcloud = db_api.subcloud_get_by_name(self.ctx, subcloud.name)
self.assertEqual(consts.BACKUP_STATE_VALIDATE_FAILED,
updated_subcloud.backup_status)
@ -2351,8 +2351,8 @@ class TestSubcloudManager(base.DCManagerTestCase):
mock_parallel_group_operation.assert_called_once()
# Verify that subcloud has the correct deploy status
# consts.PRESTAGE_STATE_PACKAGES
# Verify that subcloud has the correct backup status
# consts.BACKUP_STATE_VALIDATE_FAILED
updated_subcloud = db_api.subcloud_get_by_name(self.ctx, subcloud.name)
self.assertEqual(consts.BACKUP_STATE_VALIDATE_FAILED,
updated_subcloud.backup_status)
@ -2382,8 +2382,8 @@ class TestSubcloudManager(base.DCManagerTestCase):
mock_parallel_group_operation.assert_called_once()
# Verify that subcloud has the correct deploy status
# consts.PRESTAGE_STATE_PACKAGES
# Verify that subcloud has the correct backup status
# consts.BACKUP_STATE_UNKNOWN
updated_subcloud = db_api.subcloud_get_by_name(self.ctx, subcloud.name)
self.assertEqual(consts.BACKUP_STATE_UNKNOWN,
updated_subcloud.backup_status)
@ -2413,8 +2413,8 @@ class TestSubcloudManager(base.DCManagerTestCase):
mock_parallel_group_operation.assert_called_once()
# Verify that subcloud has the correct deploy status
# consts.PRESTAGE_STATE_PACKAGES
# Verify that subcloud has the correct backup status
# consts.BACKUP_STATE_UNKNOWN
updated_subcloud = db_api.subcloud_get_by_name(self.ctx, subcloud.name)
self.assertEqual(consts.BACKUP_STATE_UNKNOWN,
updated_subcloud.backup_status)
@ -2628,7 +2628,7 @@ class TestSubcloudManager(base.DCManagerTestCase):
# Verify that subcloud has the correct deploy status
updated_subcloud = db_api.subcloud_get_by_name(self.ctx, subcloud.name)
self.assertEqual(consts.PRESTAGE_STATE_COMPLETE,
updated_subcloud.deploy_status)
updated_subcloud.prestage_status)
# Verify both of prestage package and image ansible playbooks were called
self.assertEqual(mock_run_ansible.call_count, 2)
@ -2664,7 +2664,7 @@ class TestSubcloudManager(base.DCManagerTestCase):
# Verify that subcloud has the correct deploy status
updated_subcloud = db_api.subcloud_get_by_name(self.ctx, subcloud.name)
self.assertEqual(consts.PRESTAGE_STATE_COMPLETE,
updated_subcloud.deploy_status)
updated_subcloud.prestage_status)
# Verify that only prestage package playbook is called
self.assertEqual(mock_run_ansible.call_count, 1)
@ -2695,7 +2695,7 @@ class TestSubcloudManager(base.DCManagerTestCase):
# Verify that subcloud has the correct deploy status
updated_subcloud = db_api.subcloud_get_by_name(self.ctx, subcloud.name)
self.assertEqual(consts.PRESTAGE_STATE_COMPLETE,
updated_subcloud.deploy_status)
updated_subcloud.prestage_status)
# Verify both of prestage package and image ansible playbooks were called
self.assertEqual(mock_run_ansible.call_count, 2)
@ -2731,7 +2731,7 @@ class TestSubcloudManager(base.DCManagerTestCase):
# Verify that subcloud has the correct deploy status
updated_subcloud = db_api.subcloud_get_by_name(self.ctx, subcloud.name)
self.assertEqual(consts.PRESTAGE_STATE_COMPLETE,
updated_subcloud.deploy_status)
updated_subcloud.prestage_status)
# Verify both of prestage package and image ansible playbooks were called
self.assertEqual(mock_run_ansible.call_count, 2)
@ -2769,7 +2769,7 @@ class TestSubcloudManager(base.DCManagerTestCase):
# Verify that subcloud has the "prestage-complete" deploy status
updated_subcloud = db_api.subcloud_get_by_name(self.ctx, subcloud.name)
self.assertEqual(consts.PRESTAGE_STATE_COMPLETE,
updated_subcloud.deploy_status)
updated_subcloud.prestage_status)
def test_get_cached_regionone_data(self):
mock_keystone_client = FakeKeystoneClient()
@ -3096,7 +3096,7 @@ class TestSubcloudManager(base.DCManagerTestCase):
@mock.patch.object(subcloud_manager.SubcloudManager,
'_unmanage_system_peer_subcloud')
def test_migrate_manage_subcloud_called_unmanage_peer_subcloud(
self, mock_unmanage_system_peer_subcloud
self, mock_unmanage_system_peer_subcloud
):
sm = subcloud_manager.SubcloudManager()
system_peer_test = test_system_peer_manager.TestSystemPeerManager