Implement versioned deploy precheck script

This commit is to include a versioned deploy precheck script into
/opt/software/scripts/rel-<sw_version> to be able to run the correct
precheck code for a specific release.

Along with this commit, the precheck api is changed to use the new
location of the script, and the precheck script is changed to add
support to patch-only prechecks, and as a consequence, minor wording
changes were done to return more accurate messages to the user.

1. For the iso upload scenario:
The upload process will copy all scripts under
<iso_root>/upgrades/software-deploy to /opt/software/rel-<ver>/scripts

2. For the patch upload scenario:
The upload process will check if patch contains the deploy-precheck
script. If it does, then the script is copied to
/opt/software/rel-<ver>/scripts, if not then a symlink will be created
to the patch 'required patch' versioned precheck script.

Notes:
- iso (prepatched or not) will always come with deploy-precheck script
- <ver> assumes the format MM.mm.pp

Test Plan:
PASS: Upload multiples patches, both with and without precheck
      scripts, and verify the versioned directories are created
      and the precheck script is created as expected
PASS: Run deploy precheck for an iso release and verify the upgrade
      precheck output is returned as expected
PASS: Run deploy precheck for a patch release and verify the patch
      precheck output is returned as expected

Depends-on: https://review.opendev.org/c/starlingx/metal/+/911595

Story: 2010676
Task: 49263

Change-Id: I04ff89d43579fd71592f7ec534db57a1ead79483
Signed-off-by: Luis Eduardo Bonatti <LuizEduardo.Bonatti@windriver.com>
Co-signed-off-by: Heitor Matsui <heitorvieira.matsui@windriver.com>
This commit is contained in:
Luis Eduardo Bonatti 2023-12-10 23:48:43 -03:00 committed by Heitor Matsui
parent 88e95d5c1f
commit 32250d9b59
5 changed files with 294 additions and 116 deletions

View File

@ -45,6 +45,10 @@ class HealthCheck(object):
def __init__(self, config):
self._config = config
# get target release from script directory location
self._target_release = re.match("^.*/rel-(\d\d.\d\d.\d+)/", __file__).group(1)
self._major_release = self._target_release.rsplit(".", 1)[0]
# get sysinv token, endpoint and client
self._sysinv_token, self._sysinv_endpoint = \
upgrade_utils.get_token_endpoint(config, service_type="platform")
@ -54,72 +58,12 @@ class HealthCheck(object):
self._software_token, self._software_endpoint = \
upgrade_utils.get_token_endpoint(config, service_type="usm")
def run_health_check(self):
"""Run general health check using sysinv client"""
force = self._config.get("force", False)
output = self._sysinv_client.health.get_kube_upgrade(args={}, relaxed=force)
if HealthCheck.FAIL_MSG in output:
return False, output
return True, output
class UpgradeHealthCheck(HealthCheck):
"""This class represents a upgrade-specific health check object
that verifies if system is in a valid state for upgrade"""
def _check_valid_upgrade_path(self):
"""Checks if active release to specified release is a valid upgrade path"""
# Get active release
isystem = self._sysinv_client.isystem.list()[0]
active_release = isystem.software_version
# supported_release is a dict with {release: required_patch}
supported_releases = dict()
# Parse upgrade metadata file for supported upgrade paths
root = ElementTree.parse("%s/../metadata.xml" % os.path.dirname(__file__))
upgrade_root = root.find("supported_upgrades").findall("upgrade")
for upgrade in upgrade_root:
version = upgrade.find("version")
required_patch = upgrade.find("required_patch")
supported_releases.update({version.text: required_patch.text if
required_patch is not None else None})
success = active_release in supported_releases
return success, active_release, supported_releases.get(active_release, None)
# TODO(heitormatsui): implement patch precheck targeted against USM
# and implement patch precheck for subcloud
def _check_required_patch(self, required_patch):
"""Checks if required patch for the supported release is installed"""
url = self._software_endpoint + '/query?show=deployed'
headers = {"X-Auth-Token": self._software_token}
response = requests.get(url, headers=headers, timeout=10)
success = True
required_patch = [required_patch] if required_patch else []
if response.status_code != 200:
print("Could not check required patches...")
return False, required_patch
applied_patches = list(response.json()["sd"].keys())
missing_patch = list(set(required_patch) - set(applied_patches))
if missing_patch:
success = False
return success, missing_patch
# TODO(heitormatsui) do we need this check on USM? Remove if we don't
def _check_active_is_controller_0(self):
"""Checks that active controller is controller-0"""
controllers = self._sysinv_client.ihost.list()
for controller in controllers:
if controller.hostname == "controller-0" and \
"Controller-Active" in controller.capabilities["Personality"]:
return True
return False
def _check_license(self, version):
"""Validates the current license is valid for the specified version"""
"""
Validates the current license is valid for the specified version
:param version: version to be checked against installed license
:return: True is license is valid for version, False otherwise
"""
license_dict = self._sysinv_client.license.show()
if license_dict["error"]:
return False
@ -135,9 +79,91 @@ class UpgradeHealthCheck(HealthCheck):
return False
return True
# TODO(heitormatsui): implement patch precheck targeted against USM
# and implement patch precheck for subcloud
def _check_deployed_state(self, required_patches):
"""
Checks if every patch in a list is in 'deployed' state
:param required_patches: list of patches to be checked
:return: boolean indicating success/failure and list of patches
that are not in the 'deployed' state
"""
url = self._software_endpoint + '/query?show=deployed'
headers = {"X-Auth-Token": self._software_token}
response = requests.get(url, headers=headers, timeout=10)
success = True
if response.status_code != 200:
print("Could not check required patches...")
return False, required_patches
applied_patches = list(response.json()["sd"].keys())
missing_patch = list(set(required_patches) - set(applied_patches))
if missing_patch:
success = False
return success, missing_patch
def run_health_check(self):
"""Run general health check using sysinv client"""
force = self._config.get("force", False)
health_ok = success = True
output = self._sysinv_client.health.get_kube_upgrade(args={}, relaxed=force)
if HealthCheck.FAIL_MSG in output:
success = False
health_ok = health_ok and success
# check installed license
success = self._check_license(self._major_release)
output += 'Installed license is valid: [%s]\n' \
% (HealthCheck.SUCCESS_MSG if success else HealthCheck.FAIL_MSG)
health_ok = health_ok and success
return health_ok, output
class UpgradeHealthCheck(HealthCheck):
"""This class represents a upgrade-specific health check object
that verifies if system is in a valid state for upgrade"""
# TODO(heitormatsui): switch from using upgrade metadata xml to
# the new USM metadata format
def _check_valid_upgrade_path(self):
"""Checks if active release to specified release is a valid upgrade path"""
# Get active release
isystem = self._sysinv_client.isystem.list()[0]
active_release = isystem.software_version
# supported_release is a dict with {release: required_patch}
supported_releases = dict()
# Parse upgrade metadata file for supported upgrade paths
root = ElementTree.parse("/var/www/pages/feed/rel-%s/upgrades/metadata.xml" % self._major_release)
upgrade_root = root.find("supported_upgrades").findall("upgrade")
for upgrade in upgrade_root:
version = upgrade.find("version")
required_patch = upgrade.find("required_patch")
supported_releases.update({version.text: [required_patch.text] if
required_patch is not None else []})
success = active_release in supported_releases
return success, active_release, supported_releases.get(active_release, [])
# TODO(heitormatsui) do we need this check on USM? Remove if we don't
def _check_active_is_controller_0(self):
"""Checks that active controller is controller-0"""
controllers = self._sysinv_client.ihost.list()
for controller in controllers:
if controller.hostname == "controller-0" and \
"Controller-Active" in controller.capabilities["Personality"]:
return True
return False
def _check_kube_version(self, supported_versions):
"""Check if active k8s version is in a list of supported versions
:param supported_versions: list of supported k8s versions
"""
Check if active k8s version is in a list of supported versions
:param supported_versions: list of supported k8s versions
:return: boolean indicating success/failure and active k8s version
"""
kube_versions = self._sysinv_client.kube_version.list()
active_version = None
@ -153,35 +179,21 @@ class UpgradeHealthCheck(HealthCheck):
health_ok = True
output = ""
# get target release from script directory location
upgrade_release = re.match("^.*/rel-(\d\d.\d\d.\d*)/", __file__).group(1)
# check installed license
success = self._check_license(upgrade_release)
output += 'License valid for upgrade: [%s]\n' \
% (HealthCheck.SUCCESS_MSG if success else HealthCheck.FAIL_MSG)
health_ok = health_ok and success
# check if it is a valid upgrade path
success, active_release, required_patch = self._check_valid_upgrade_path()
success, active_release, required_patches = self._check_valid_upgrade_path()
output += 'Valid upgrade path from release %s to %s: [%s]\n' \
% (active_release, upgrade_release,
% (active_release, self._major_release,
HealthCheck.SUCCESS_MSG if success else HealthCheck.FAIL_MSG)
health_ok = health_ok and success
# check if required patches are applied/committed if is a valid upgrade path
if success:
success, missing_patches = self._check_required_patch(required_patch)
output += 'Required patches are applied: [%s]\n' \
% (HealthCheck.SUCCESS_MSG if success else HealthCheck.FAIL_MSG)
if not success:
output += 'Patches not applied: %s\n' \
% ', '.join(missing_patches)
health_ok = health_ok and success
else:
output += 'Invalid upgrade path, skipping required patches check...'
# check if required patches are deployed
success, missing_patches = self._check_deployed_state(required_patches)
output += 'Required patches are applied: [%s]\n' \
% (HealthCheck.SUCCESS_MSG if success else HealthCheck.FAIL_MSG)
if not success:
output += '-> Patches not applied: [%s]\n' \
% ', '.join(missing_patches)
health_ok = health_ok and success
# check if k8s version is valid
success, active_version = self._check_kube_version(SUPPORTED_K8S_VERSIONS)
@ -207,7 +219,46 @@ class UpgradeHealthCheck(HealthCheck):
output += \
'Active controller is controller-0: [%s]\n' \
% (HealthCheck.SUCCESS_MSG if success else HealthCheck.FAIL_MSG)
health_ok = health_ok and success
return health_ok, output
class PatchHealthCheck(HealthCheck):
"""This class represents a patch-specific health check object
that verifies if system is in valid state to apply a patch"""
def _get_required_patches(self):
"""Get required patches for a target release"""
url = self._software_endpoint + '/query'
headers = {"X-Auth-Token": self._software_token}
response = requests.get(url, headers=headers, timeout=10)
if response.status_code != 200:
print("Could not get required patches...")
return []
required_patches = []
for release, values in response.json()["sd"].items():
if values["sw_version"] == self._target_release:
required_patches.extend(values["requires"])
break
return required_patches
def run_health_check(self):
"""Run specific patch health checks"""
health_ok = True
output = ""
# check required patches for target release
required_patches = self._get_required_patches()
success, missing_patches = self._check_deployed_state(required_patches)
output += 'Required patches are applied: [%s]\n' \
% (HealthCheck.SUCCESS_MSG if success else HealthCheck.FAIL_MSG)
if not success:
output += '-> Patches not applied: [%s]\n' \
% ', '.join(missing_patches)
health_ok = health_ok and success
return health_ok, output
@ -242,6 +293,9 @@ def parse_config(args=None):
parser.add_argument("--force",
help="Ignore non-critical health checks",
action="store_true")
parser.add_argument("--patch",
help="Set precheck to run against a patch release",
action="store_true")
# if args was not passed will use sys.argv by default
parsed_args = parser.parse_args(args)
@ -250,19 +304,25 @@ def parse_config(args=None):
def main(argv=None):
config = parse_config(argv)
patch_release = config.get("patch", False)
general_health_check = HealthCheck(config)
upgrade_health_check = UpgradeHealthCheck(config)
health_ok = True
output = ""
# execute general health check
general_health_check = HealthCheck(config)
general_health_ok, general_output = general_health_check.run_health_check()
# execute upgrade-specific health check
upgrade_health_ok, upgrade_output = upgrade_health_check.run_health_check()
# execute release-specific health check
if patch_release:
specific_health_check = PatchHealthCheck(config)
else:
specific_health_check = UpgradeHealthCheck(config)
specific_health_ok, specific_output = specific_health_check.run_health_check()
# combine health check results removing extra line breaks/blank spaces from the output
health_ok = general_health_ok and upgrade_health_ok
output = general_output.strip() + "\n" + upgrade_output.strip()
health_ok = general_health_ok and specific_health_ok
output = general_output.strip() + "\n" + specific_output.strip()
# print health check output and exit
print(output)

View File

@ -159,6 +159,7 @@ RELEASE_GA_NAME = "starlingx-%s"
# Precheck constants
LICENSE_FILE = "/etc/platform/.license"
VERIFY_LICENSE_BINARY = "/usr/bin/verify-license"
VERSIONED_SCRIPTS_DIR = "%s/rel-%%s/bin/" % SOFTWARE_STORAGE_DIR
SOFTWARE_JSON_FILE = "%s/software.json" % SOFTWARE_STORAGE_DIR
SYNCED_SOFTWARE_JSON_FILE = "%s/synced/software.json" % SOFTWARE_STORAGE_DIR

View File

@ -117,6 +117,11 @@ class ReleaseVersionDoNotExist(SoftwareError):
pass
class VersionedDeployPrecheckFailure(SoftwareError):
"""Versioned deploy-precheck script cannot be created"""
pass
class FileSystemError(SoftwareError):
"""
A failure during a linux file operation.

View File

@ -1270,11 +1270,11 @@ class PatchController(PatchService):
return local_info, local_warning, local_error, release_meta_info
def _process_upload_patch_files(self, patch_files):
'''
"""
Process the uploaded patch files
:param patch_files: list of patch files
:return: info, warning, error messages
'''
"""
local_info = ""
local_warning = ""
@ -1342,7 +1342,6 @@ class PatchController(PatchService):
metadata_dir=constants.AVAILABLE_DIR,
base_pkgdata=self.base_pkgdata)
PatchFile.unpack_patch(patch_file)
local_info += "%s is now uploaded\n" % release_id
self.release_data.add_release(this_release)
@ -1372,6 +1371,32 @@ class PatchController(PatchService):
}
})
# create versioned precheck for uploaded patches
for patch in upload_patch_info:
filename, values = list(patch.items())[0]
LOG.info("Creating precheck for release %s..." % values.get("id"))
for pf in patch_files:
if filename in pf:
patch_file = pf
sw_version = values.get("sw_version")
required_patches = self.release_data.metadata[values.get("id")].get("requires")
# sort the required patches list and get the latest, if available
req_patch_id = None
req_patch_metadata = None
req_patch_version = None
if required_patches:
req_patch_id = sorted(required_patches)[-1]
if req_patch_id:
req_patch_metadata = self.release_data.metadata.get(req_patch_id)
if req_patch_metadata:
req_patch_version = req_patch_metadata.get("sw_version")
if req_patch_id and not req_patch_metadata:
LOG.warning("Required patch '%s' is not uploaded." % req_patch_id)
PatchFile.create_versioned_precheck(patch_file, sw_version, req_patch_version=req_patch_version)
return local_info, local_warning, local_error, upload_patch_info
def software_release_upload(self, release_files):
@ -1547,6 +1572,9 @@ class PatchController(PatchService):
LOG.info(msg)
msg_info += msg + "\n"
# TODO(lbonatti): treat the upcoming versioning changes
PatchFile.delete_versioned_directory(self.release_data.metadata[release_id]["sw_version"])
try:
# Delete the metadata
deploystate = self.release_data.metadata[release_id]["state"]
@ -2112,10 +2140,15 @@ class PatchController(PatchService):
return release, success, msg_info, msg_warning, msg_error
def _deploy_precheck(self, release_version: str, force: bool, region_name: str = "RegionOne") -> dict:
def _deploy_precheck(self, release_version: str, force: bool = False,
region_name: str = "RegionOne", patch: bool = False) -> dict:
"""
Verify if system is capable to upgrade to a specified deployment
return: dict of info, warning and error messages
Verify if system satisfy the requisites to upgrade to a specified deployment.
:param release_version: full release name, e.g. starlingx-MM.mm.pp
:param force: if True will ignore minor alarms during precheck
:param region_name: region_name
:param patch: if True then indicate precheck is for patch release
:return: dict of info, warning and error messages
"""
msg_info = ""
@ -2123,8 +2156,9 @@ class PatchController(PatchService):
msg_error = ""
precheck_script = utils.get_precheck_script(release_version)
if not os.path.isfile(precheck_script):
msg = "Upgrade files for deployment %s are not present on the system, " \
msg = "Release files for deployment %s are not present on the system, " \
"cannot proceed with the precheck." % release_version
LOG.error(msg)
msg_error = "Fail to perform deploy precheck. " \
@ -2165,6 +2199,8 @@ class PatchController(PatchService):
"--region_name=%s" % region_name]
if force:
cmd.append("--force")
if patch:
cmd.append("--patch")
# Call precheck from the deployment files
precheck_return = subprocess.run(
@ -2181,17 +2217,20 @@ class PatchController(PatchService):
return dict(info=msg_info, warning=msg_warning, error=msg_error)
def software_deploy_precheck_api(self, deployment: str, force: bool, **kwargs) -> dict:
def software_deploy_precheck_api(self, deployment: str, force: bool = False, **kwargs) -> dict:
"""
Verify if system is capable to upgrade to a specified deployment
return: dict of info, warning and error messages
Verify if system satisfy the requisites to upgrade to a specified deployment.
:param deployment: full release name, e.g. starlingx-MM.mm.pp
:param force: if True will ignore minor alarms during precheck
:return: dict of info, warning and error messages
"""
release, success, msg_info, msg_warning, msg_error = self._release_basic_checks(deployment)
if not success:
return dict(info=msg_info, warning=msg_warning, error=msg_error)
region_name = kwargs["region_name"]
release_version = release["sw_version"]
return self._deploy_precheck(release_version, force, region_name)
patch = not utils.is_upgrade_deploy(SW_VERSION, release_version)
return self._deploy_precheck(release_version, force, region_name, patch)
def _deploy_upgrade_start(self, to_release):
LOG.info("start deploy upgrade to %s from %s" % (to_release, SW_VERSION))
@ -2255,9 +2294,12 @@ class PatchController(PatchService):
if not success:
return dict(info=msg_info, warning=msg_warning, error=msg_error)
# TODO(heitormatsui) Enforce deploy-precheck for patch release
patch_release = True
if utils.is_upgrade_deploy(SW_VERSION, release["sw_version"]):
patch_release = False
to_release = release["sw_version"]
ret = self._deploy_precheck(to_release, force)
ret = self._deploy_precheck(to_release, force, patch=patch_release)
if ret["error"]:
ret["error"] = "The following issues have been detected which prevent " \
"deploying %s\n" % deployment + \
@ -2309,13 +2351,12 @@ class PatchController(PatchService):
else:
operation = "remove"
# If releases are such that R2 requires R1
# R3 requires R2
# R4 requires R3
# And current running release is R2
# And command issued is "software deploy start R4"
# Order for apply operation: [R3, R4]
# Order for remove operation: [R3]
# If releases are such that:
# R2 requires R1, R3 requires R2, R4 requires R3
# If current running release is R2 and command issued is "software deploy start R4"
# operation is "apply" with order [R3, R4]
# If current running release is R4 and command issued is "software deploy start R2"
# operation is "remove" with order [R4, R3]
if operation == "apply":
collect_current_load_for_hosts()

View File

@ -33,6 +33,7 @@ from software.exceptions import ReleaseValidationFailure
from software.exceptions import ReleaseMismatchFailure
from software.exceptions import SoftwareFail
from software.exceptions import SoftwareServiceError
from software.exceptions import VersionedDeployPrecheckFailure
import software.constants as constants
import software.utils as utils
@ -980,6 +981,69 @@ class PatchFile(object):
os.chdir(orig_wd)
shutil.rmtree(patch_tmpdir)
@staticmethod
def create_versioned_precheck(patch, sw_version, req_patch_version=None):
"""
Extract the deploy-precheck script from the patch into
a versioned directory under SOFTWARE_STORAGE_DIR and,
if script is not available in the patch, then create a
symlink to the versioned directory of the required patch.
:param patch: path to patch file
:param sw_version: patch version in MM.mm.pp format
:param req_patch_version: required patch version in MM.mm.pp format
"""
# open patch and create versioned scripts directory
tar = tarfile.open(patch, "r:gz")
versioned_dir = constants.VERSIONED_SCRIPTS_DIR % sw_version
versioned_script = os.path.join(versioned_dir, constants.DEPLOY_PRECHECK_SCRIPT)
if os.path.exists(versioned_dir):
shutil.rmtree(versioned_dir)
os.makedirs(versioned_dir)
error_msg = "Versioned precheck script cannot be created, "
try:
# if patch contains precheck script, copy it to versioned directory
if constants.DEPLOY_PRECHECK_SCRIPT in tar.getnames():
tar.extract(constants.DEPLOY_PRECHECK_SCRIPT, path=versioned_dir)
os.chmod(versioned_script, mode=0o755)
LOG.info("Versioned precheck script copied to %s." % versioned_script)
# in case patch does not contain a precheck script
# then symlink to required patch versioned directory
else:
LOG.info("'%s' script is not included in the patch, will attempt to "
"symlink to the 'required patch' precheck script." %
constants.DEPLOY_PRECHECK_SCRIPT)
if not req_patch_version:
error_msg += "'required patch' version could not be determined."
raise VersionedDeployPrecheckFailure
req_versioned_dir = constants.VERSIONED_SCRIPTS_DIR % req_patch_version
req_versioned_script = os.path.join(req_versioned_dir, constants.DEPLOY_PRECHECK_SCRIPT)
# if required patch directory does not exist create the link anyway
if not os.path.exists(req_versioned_dir):
LOG.warning("'required patch' versioned directory %s does not exist."
% req_versioned_dir)
os.symlink(req_versioned_script, versioned_script)
LOG.info("Versioned precheck script %s symlinked to %s." % (
versioned_script, req_versioned_script))
except Exception as e:
LOG.warning("%s: %s" % (error_msg, e))
@staticmethod
def delete_versioned_directory(sw_version):
"""
Delete the versioned deploy-precheck script.
:param sw_version: precheck script version to be deleted
"""
try:
opt_release_folder = "%s/rel-%s" % (constants.SOFTWARE_STORAGE_DIR,
sw_version)
if os.path.isdir(opt_release_folder):
shutil.rmtree(opt_release_folder, ignore_errors=True)
LOG.info("Versioned directory %s deleted." % opt_release_folder)
except Exception as e:
LOG.exception("Failed to delete versioned precheck: %s", e)
def patch_build():
configure_logging(logtofile=False)
@ -1198,5 +1262,12 @@ def parse_release_metadata(filename):
root = tree.getroot()
data = {}
for child in root:
# get requires under <req_patch_id> key
if child.tag == "requires":
requires = []
for item in child:
requires.append(item.text)
data[child.tag] = requires
continue
data[child.tag] = child.text
return data