diff --git a/software/scripts/deploy-precheck b/software/scripts/deploy-precheck index 3744c59c..fb6ecf24 100644 --- a/software/scripts/deploy-precheck +++ b/software/scripts/deploy-precheck @@ -45,6 +45,10 @@ class HealthCheck(object): def __init__(self, config): self._config = config + # get target release from script directory location + self._target_release = re.match("^.*/rel-(\d\d.\d\d.\d+)/", __file__).group(1) + self._major_release = self._target_release.rsplit(".", 1)[0] + # get sysinv token, endpoint and client self._sysinv_token, self._sysinv_endpoint = \ upgrade_utils.get_token_endpoint(config, service_type="platform") @@ -54,72 +58,12 @@ class HealthCheck(object): self._software_token, self._software_endpoint = \ upgrade_utils.get_token_endpoint(config, service_type="usm") - def run_health_check(self): - """Run general health check using sysinv client""" - force = self._config.get("force", False) - output = self._sysinv_client.health.get_kube_upgrade(args={}, relaxed=force) - if HealthCheck.FAIL_MSG in output: - return False, output - return True, output - - -class UpgradeHealthCheck(HealthCheck): - """This class represents a upgrade-specific health check object - that verifies if system is in a valid state for upgrade""" - - def _check_valid_upgrade_path(self): - """Checks if active release to specified release is a valid upgrade path""" - # Get active release - isystem = self._sysinv_client.isystem.list()[0] - active_release = isystem.software_version - - # supported_release is a dict with {release: required_patch} - supported_releases = dict() - - # Parse upgrade metadata file for supported upgrade paths - root = ElementTree.parse("%s/../metadata.xml" % os.path.dirname(__file__)) - upgrade_root = root.find("supported_upgrades").findall("upgrade") - for upgrade in upgrade_root: - version = upgrade.find("version") - required_patch = upgrade.find("required_patch") - supported_releases.update({version.text: required_patch.text if - required_patch is not None else None}) - success = active_release in supported_releases - return success, active_release, supported_releases.get(active_release, None) - - # TODO(heitormatsui): implement patch precheck targeted against USM - # and implement patch precheck for subcloud - def _check_required_patch(self, required_patch): - """Checks if required patch for the supported release is installed""" - url = self._software_endpoint + '/query?show=deployed' - headers = {"X-Auth-Token": self._software_token} - response = requests.get(url, headers=headers, timeout=10) - - success = True - required_patch = [required_patch] if required_patch else [] - if response.status_code != 200: - print("Could not check required patches...") - return False, required_patch - - applied_patches = list(response.json()["sd"].keys()) - missing_patch = list(set(required_patch) - set(applied_patches)) - if missing_patch: - success = False - - return success, missing_patch - - # TODO(heitormatsui) do we need this check on USM? Remove if we don't - def _check_active_is_controller_0(self): - """Checks that active controller is controller-0""" - controllers = self._sysinv_client.ihost.list() - for controller in controllers: - if controller.hostname == "controller-0" and \ - "Controller-Active" in controller.capabilities["Personality"]: - return True - return False - def _check_license(self, version): - """Validates the current license is valid for the specified version""" + """ + Validates the current license is valid for the specified version + :param version: version to be checked against installed license + :return: True is license is valid for version, False otherwise + """ license_dict = self._sysinv_client.license.show() if license_dict["error"]: return False @@ -135,9 +79,91 @@ class UpgradeHealthCheck(HealthCheck): return False return True + # TODO(heitormatsui): implement patch precheck targeted against USM + # and implement patch precheck for subcloud + def _check_deployed_state(self, required_patches): + """ + Checks if every patch in a list is in 'deployed' state + :param required_patches: list of patches to be checked + :return: boolean indicating success/failure and list of patches + that are not in the 'deployed' state + """ + url = self._software_endpoint + '/query?show=deployed' + headers = {"X-Auth-Token": self._software_token} + response = requests.get(url, headers=headers, timeout=10) + + success = True + if response.status_code != 200: + print("Could not check required patches...") + return False, required_patches + + applied_patches = list(response.json()["sd"].keys()) + missing_patch = list(set(required_patches) - set(applied_patches)) + if missing_patch: + success = False + + return success, missing_patch + + def run_health_check(self): + """Run general health check using sysinv client""" + force = self._config.get("force", False) + health_ok = success = True + + output = self._sysinv_client.health.get_kube_upgrade(args={}, relaxed=force) + if HealthCheck.FAIL_MSG in output: + success = False + health_ok = health_ok and success + + # check installed license + success = self._check_license(self._major_release) + output += 'Installed license is valid: [%s]\n' \ + % (HealthCheck.SUCCESS_MSG if success else HealthCheck.FAIL_MSG) + health_ok = health_ok and success + + return health_ok, output + + +class UpgradeHealthCheck(HealthCheck): + """This class represents a upgrade-specific health check object + that verifies if system is in a valid state for upgrade""" + + # TODO(heitormatsui): switch from using upgrade metadata xml to + # the new USM metadata format + def _check_valid_upgrade_path(self): + """Checks if active release to specified release is a valid upgrade path""" + # Get active release + isystem = self._sysinv_client.isystem.list()[0] + active_release = isystem.software_version + + # supported_release is a dict with {release: required_patch} + supported_releases = dict() + + # Parse upgrade metadata file for supported upgrade paths + root = ElementTree.parse("/var/www/pages/feed/rel-%s/upgrades/metadata.xml" % self._major_release) + upgrade_root = root.find("supported_upgrades").findall("upgrade") + for upgrade in upgrade_root: + version = upgrade.find("version") + required_patch = upgrade.find("required_patch") + supported_releases.update({version.text: [required_patch.text] if + required_patch is not None else []}) + success = active_release in supported_releases + return success, active_release, supported_releases.get(active_release, []) + + # TODO(heitormatsui) do we need this check on USM? Remove if we don't + def _check_active_is_controller_0(self): + """Checks that active controller is controller-0""" + controllers = self._sysinv_client.ihost.list() + for controller in controllers: + if controller.hostname == "controller-0" and \ + "Controller-Active" in controller.capabilities["Personality"]: + return True + return False + def _check_kube_version(self, supported_versions): - """Check if active k8s version is in a list of supported versions - :param supported_versions: list of supported k8s versions + """ + Check if active k8s version is in a list of supported versions + :param supported_versions: list of supported k8s versions + :return: boolean indicating success/failure and active k8s version """ kube_versions = self._sysinv_client.kube_version.list() active_version = None @@ -153,35 +179,21 @@ class UpgradeHealthCheck(HealthCheck): health_ok = True output = "" - # get target release from script directory location - upgrade_release = re.match("^.*/rel-(\d\d.\d\d.\d*)/", __file__).group(1) - - # check installed license - success = self._check_license(upgrade_release) - output += 'License valid for upgrade: [%s]\n' \ - % (HealthCheck.SUCCESS_MSG if success else HealthCheck.FAIL_MSG) - - health_ok = health_ok and success - # check if it is a valid upgrade path - success, active_release, required_patch = self._check_valid_upgrade_path() + success, active_release, required_patches = self._check_valid_upgrade_path() output += 'Valid upgrade path from release %s to %s: [%s]\n' \ - % (active_release, upgrade_release, + % (active_release, self._major_release, HealthCheck.SUCCESS_MSG if success else HealthCheck.FAIL_MSG) health_ok = health_ok and success - # check if required patches are applied/committed if is a valid upgrade path - if success: - success, missing_patches = self._check_required_patch(required_patch) - output += 'Required patches are applied: [%s]\n' \ - % (HealthCheck.SUCCESS_MSG if success else HealthCheck.FAIL_MSG) - if not success: - output += 'Patches not applied: %s\n' \ - % ', '.join(missing_patches) - - health_ok = health_ok and success - else: - output += 'Invalid upgrade path, skipping required patches check...' + # check if required patches are deployed + success, missing_patches = self._check_deployed_state(required_patches) + output += 'Required patches are applied: [%s]\n' \ + % (HealthCheck.SUCCESS_MSG if success else HealthCheck.FAIL_MSG) + if not success: + output += '-> Patches not applied: [%s]\n' \ + % ', '.join(missing_patches) + health_ok = health_ok and success # check if k8s version is valid success, active_version = self._check_kube_version(SUPPORTED_K8S_VERSIONS) @@ -207,7 +219,46 @@ class UpgradeHealthCheck(HealthCheck): output += \ 'Active controller is controller-0: [%s]\n' \ % (HealthCheck.SUCCESS_MSG if success else HealthCheck.FAIL_MSG) + health_ok = health_ok and success + return health_ok, output + + +class PatchHealthCheck(HealthCheck): + """This class represents a patch-specific health check object + that verifies if system is in valid state to apply a patch""" + + def _get_required_patches(self): + """Get required patches for a target release""" + url = self._software_endpoint + '/query' + headers = {"X-Auth-Token": self._software_token} + response = requests.get(url, headers=headers, timeout=10) + + if response.status_code != 200: + print("Could not get required patches...") + return [] + + required_patches = [] + for release, values in response.json()["sd"].items(): + if values["sw_version"] == self._target_release: + required_patches.extend(values["requires"]) + break + + return required_patches + + def run_health_check(self): + """Run specific patch health checks""" + health_ok = True + output = "" + + # check required patches for target release + required_patches = self._get_required_patches() + success, missing_patches = self._check_deployed_state(required_patches) + output += 'Required patches are applied: [%s]\n' \ + % (HealthCheck.SUCCESS_MSG if success else HealthCheck.FAIL_MSG) + if not success: + output += '-> Patches not applied: [%s]\n' \ + % ', '.join(missing_patches) health_ok = health_ok and success return health_ok, output @@ -242,6 +293,9 @@ def parse_config(args=None): parser.add_argument("--force", help="Ignore non-critical health checks", action="store_true") + parser.add_argument("--patch", + help="Set precheck to run against a patch release", + action="store_true") # if args was not passed will use sys.argv by default parsed_args = parser.parse_args(args) @@ -250,19 +304,25 @@ def parse_config(args=None): def main(argv=None): config = parse_config(argv) + patch_release = config.get("patch", False) - general_health_check = HealthCheck(config) - upgrade_health_check = UpgradeHealthCheck(config) + health_ok = True + output = "" # execute general health check + general_health_check = HealthCheck(config) general_health_ok, general_output = general_health_check.run_health_check() - # execute upgrade-specific health check - upgrade_health_ok, upgrade_output = upgrade_health_check.run_health_check() + # execute release-specific health check + if patch_release: + specific_health_check = PatchHealthCheck(config) + else: + specific_health_check = UpgradeHealthCheck(config) + specific_health_ok, specific_output = specific_health_check.run_health_check() # combine health check results removing extra line breaks/blank spaces from the output - health_ok = general_health_ok and upgrade_health_ok - output = general_output.strip() + "\n" + upgrade_output.strip() + health_ok = general_health_ok and specific_health_ok + output = general_output.strip() + "\n" + specific_output.strip() # print health check output and exit print(output) diff --git a/software/software/constants.py b/software/software/constants.py index f358e027..8b8d6a3b 100644 --- a/software/software/constants.py +++ b/software/software/constants.py @@ -163,6 +163,7 @@ RELEASE_GA_NAME = "starlingx-%s" # Precheck constants LICENSE_FILE = "/etc/platform/.license" VERIFY_LICENSE_BINARY = "/usr/bin/verify-license" +VERSIONED_SCRIPTS_DIR = "%s/rel-%%s/bin/" % SOFTWARE_STORAGE_DIR SOFTWARE_JSON_FILE = "%s/software.json" % SOFTWARE_STORAGE_DIR SYNCED_SOFTWARE_JSON_FILE = "%s/synced/software.json" % SOFTWARE_STORAGE_DIR diff --git a/software/software/exceptions.py b/software/software/exceptions.py index 6a905789..3e49897d 100644 --- a/software/software/exceptions.py +++ b/software/software/exceptions.py @@ -117,6 +117,11 @@ class ReleaseVersionDoNotExist(SoftwareError): pass +class VersionedDeployPrecheckFailure(SoftwareError): + """Versioned deploy-precheck script cannot be created""" + pass + + class FileSystemError(SoftwareError): """ A failure during a linux file operation. diff --git a/software/software/software_controller.py b/software/software/software_controller.py index edd39b8a..c1d4da70 100644 --- a/software/software/software_controller.py +++ b/software/software/software_controller.py @@ -1273,11 +1273,11 @@ class PatchController(PatchService): return local_info, local_warning, local_error, release_meta_info def _process_upload_patch_files(self, patch_files): - ''' + """ Process the uploaded patch files :param patch_files: list of patch files :return: info, warning, error messages - ''' + """ local_info = "" local_warning = "" @@ -1345,7 +1345,6 @@ class PatchController(PatchService): metadata_dir=constants.AVAILABLE_DIR, base_pkgdata=self.base_pkgdata) PatchFile.unpack_patch(patch_file) - local_info += "%s is now uploaded\n" % release_id self.release_data.add_release(this_release) @@ -1375,6 +1374,32 @@ class PatchController(PatchService): } }) + # create versioned precheck for uploaded patches + for patch in upload_patch_info: + filename, values = list(patch.items())[0] + LOG.info("Creating precheck for release %s..." % values.get("id")) + for pf in patch_files: + if filename in pf: + patch_file = pf + + sw_version = values.get("sw_version") + required_patches = self.release_data.metadata[values.get("id")].get("requires") + + # sort the required patches list and get the latest, if available + req_patch_id = None + req_patch_metadata = None + req_patch_version = None + if required_patches: + req_patch_id = sorted(required_patches)[-1] + if req_patch_id: + req_patch_metadata = self.release_data.metadata.get(req_patch_id) + if req_patch_metadata: + req_patch_version = req_patch_metadata.get("sw_version") + if req_patch_id and not req_patch_metadata: + LOG.warning("Required patch '%s' is not uploaded." % req_patch_id) + + PatchFile.create_versioned_precheck(patch_file, sw_version, req_patch_version=req_patch_version) + return local_info, local_warning, local_error, upload_patch_info def software_release_upload(self, release_files): @@ -1586,6 +1611,9 @@ class PatchController(PatchService): LOG.info(msg) msg_info += msg + "\n" + # TODO(lbonatti): treat the upcoming versioning changes + PatchFile.delete_versioned_directory(self.release_data.metadata[release_id]["sw_version"]) + try: # Delete the metadata deploystate = self.release_data.metadata[release_id]["state"] @@ -2151,10 +2179,15 @@ class PatchController(PatchService): return release, success, msg_info, msg_warning, msg_error - def _deploy_precheck(self, release_version: str, force: bool, region_name: str = "RegionOne") -> dict: + def _deploy_precheck(self, release_version: str, force: bool = False, + region_name: str = "RegionOne", patch: bool = False) -> dict: """ - Verify if system is capable to upgrade to a specified deployment - return: dict of info, warning and error messages + Verify if system satisfy the requisites to upgrade to a specified deployment. + :param release_version: full release name, e.g. starlingx-MM.mm.pp + :param force: if True will ignore minor alarms during precheck + :param region_name: region_name + :param patch: if True then indicate precheck is for patch release + :return: dict of info, warning and error messages """ msg_info = "" @@ -2162,8 +2195,9 @@ class PatchController(PatchService): msg_error = "" precheck_script = utils.get_precheck_script(release_version) + if not os.path.isfile(precheck_script): - msg = "Upgrade files for deployment %s are not present on the system, " \ + msg = "Release files for deployment %s are not present on the system, " \ "cannot proceed with the precheck." % release_version LOG.error(msg) msg_error = "Fail to perform deploy precheck. " \ @@ -2204,6 +2238,8 @@ class PatchController(PatchService): "--region_name=%s" % region_name] if force: cmd.append("--force") + if patch: + cmd.append("--patch") # Call precheck from the deployment files precheck_return = subprocess.run( @@ -2220,17 +2256,20 @@ class PatchController(PatchService): return dict(info=msg_info, warning=msg_warning, error=msg_error) - def software_deploy_precheck_api(self, deployment: str, force: bool, **kwargs) -> dict: + def software_deploy_precheck_api(self, deployment: str, force: bool = False, **kwargs) -> dict: """ - Verify if system is capable to upgrade to a specified deployment - return: dict of info, warning and error messages + Verify if system satisfy the requisites to upgrade to a specified deployment. + :param deployment: full release name, e.g. starlingx-MM.mm.pp + :param force: if True will ignore minor alarms during precheck + :return: dict of info, warning and error messages """ release, success, msg_info, msg_warning, msg_error = self._release_basic_checks(deployment) if not success: return dict(info=msg_info, warning=msg_warning, error=msg_error) region_name = kwargs["region_name"] release_version = release["sw_version"] - return self._deploy_precheck(release_version, force, region_name) + patch = not utils.is_upgrade_deploy(SW_VERSION, release_version) + return self._deploy_precheck(release_version, force, region_name, patch) def _deploy_upgrade_start(self, to_release): LOG.info("start deploy upgrade to %s from %s" % (to_release, SW_VERSION)) @@ -2294,9 +2333,12 @@ class PatchController(PatchService): if not success: return dict(info=msg_info, warning=msg_warning, error=msg_error) + # TODO(heitormatsui) Enforce deploy-precheck for patch release + patch_release = True if utils.is_upgrade_deploy(SW_VERSION, release["sw_version"]): + patch_release = False to_release = release["sw_version"] - ret = self._deploy_precheck(to_release, force) + ret = self._deploy_precheck(to_release, force, patch=patch_release) if ret["error"]: ret["error"] = "The following issues have been detected which prevent " \ "deploying %s\n" % deployment + \ @@ -2349,13 +2391,12 @@ class PatchController(PatchService): else: operation = "remove" - # If releases are such that R2 requires R1 - # R3 requires R2 - # R4 requires R3 - # And current running release is R2 - # And command issued is "software deploy start R4" - # Order for apply operation: [R3, R4] - # Order for remove operation: [R3] + # If releases are such that: + # R2 requires R1, R3 requires R2, R4 requires R3 + # If current running release is R2 and command issued is "software deploy start R4" + # operation is "apply" with order [R3, R4] + # If current running release is R4 and command issued is "software deploy start R2" + # operation is "remove" with order [R4, R3] if operation == "apply": collect_current_load_for_hosts() diff --git a/software/software/software_functions.py b/software/software/software_functions.py index fbaa4a47..9aeda761 100644 --- a/software/software/software_functions.py +++ b/software/software/software_functions.py @@ -34,6 +34,7 @@ from software.exceptions import ReleaseValidationFailure from software.exceptions import ReleaseMismatchFailure from software.exceptions import SoftwareFail from software.exceptions import SoftwareServiceError +from software.exceptions import VersionedDeployPrecheckFailure import software.constants as constants import software.utils as utils @@ -981,6 +982,69 @@ class PatchFile(object): os.chdir(orig_wd) shutil.rmtree(patch_tmpdir) + @staticmethod + def create_versioned_precheck(patch, sw_version, req_patch_version=None): + """ + Extract the deploy-precheck script from the patch into + a versioned directory under SOFTWARE_STORAGE_DIR and, + if script is not available in the patch, then create a + symlink to the versioned directory of the required patch. + :param patch: path to patch file + :param sw_version: patch version in MM.mm.pp format + :param req_patch_version: required patch version in MM.mm.pp format + """ + # open patch and create versioned scripts directory + tar = tarfile.open(patch, "r:gz") + versioned_dir = constants.VERSIONED_SCRIPTS_DIR % sw_version + versioned_script = os.path.join(versioned_dir, constants.DEPLOY_PRECHECK_SCRIPT) + if os.path.exists(versioned_dir): + shutil.rmtree(versioned_dir) + os.makedirs(versioned_dir) + + error_msg = "Versioned precheck script cannot be created, " + try: + # if patch contains precheck script, copy it to versioned directory + if constants.DEPLOY_PRECHECK_SCRIPT in tar.getnames(): + tar.extract(constants.DEPLOY_PRECHECK_SCRIPT, path=versioned_dir) + os.chmod(versioned_script, mode=0o755) + LOG.info("Versioned precheck script copied to %s." % versioned_script) + # in case patch does not contain a precheck script + # then symlink to required patch versioned directory + else: + LOG.info("'%s' script is not included in the patch, will attempt to " + "symlink to the 'required patch' precheck script." % + constants.DEPLOY_PRECHECK_SCRIPT) + if not req_patch_version: + error_msg += "'required patch' version could not be determined." + raise VersionedDeployPrecheckFailure + + req_versioned_dir = constants.VERSIONED_SCRIPTS_DIR % req_patch_version + req_versioned_script = os.path.join(req_versioned_dir, constants.DEPLOY_PRECHECK_SCRIPT) + # if required patch directory does not exist create the link anyway + if not os.path.exists(req_versioned_dir): + LOG.warning("'required patch' versioned directory %s does not exist." + % req_versioned_dir) + os.symlink(req_versioned_script, versioned_script) + LOG.info("Versioned precheck script %s symlinked to %s." % ( + versioned_script, req_versioned_script)) + except Exception as e: + LOG.warning("%s: %s" % (error_msg, e)) + + @staticmethod + def delete_versioned_directory(sw_version): + """ + Delete the versioned deploy-precheck script. + :param sw_version: precheck script version to be deleted + """ + try: + opt_release_folder = "%s/rel-%s" % (constants.SOFTWARE_STORAGE_DIR, + sw_version) + if os.path.isdir(opt_release_folder): + shutil.rmtree(opt_release_folder, ignore_errors=True) + LOG.info("Versioned directory %s deleted." % opt_release_folder) + except Exception as e: + LOG.exception("Failed to delete versioned precheck: %s", e) + def patch_build(): configure_logging(logtofile=False) @@ -1211,5 +1275,12 @@ def parse_release_metadata(filename): root = tree.getroot() data = {} for child in root: + # get requires under key + if child.tag == "requires": + requires = [] + for item in child: + requires.append(item.text) + data[child.tag] = requires + continue data[child.tag] = child.text return data