Merge "Implement versioned deploy precheck script"

software/scripts/deploy-precheck

@@ -45,6 +45,10 @@ class HealthCheck(object):
     def __init__(self, config):
         self._config = config
 
+        # get target release from script directory location
+        self._target_release = re.match("^.*/rel-(\d\d.\d\d.\d+)/", __file__).group(1)
+        self._major_release = self._target_release.rsplit(".", 1)[0]
+
         # get sysinv token, endpoint and client
         self._sysinv_token, self._sysinv_endpoint = \
             upgrade_utils.get_token_endpoint(config, service_type="platform")
@@ -54,72 +58,12 @@ class HealthCheck(object):
         self._software_token, self._software_endpoint = \
             upgrade_utils.get_token_endpoint(config, service_type="usm")
 
-    def run_health_check(self):
-        """Run general health check using sysinv client"""
-        force = self._config.get("force", False)
-        output = self._sysinv_client.health.get_kube_upgrade(args={}, relaxed=force)
-        if HealthCheck.FAIL_MSG in output:
-            return False, output
-        return True, output
-
-
-class UpgradeHealthCheck(HealthCheck):
-    """This class represents a upgrade-specific health check object
-    that verifies if system is in a valid state for upgrade"""
-
-    def _check_valid_upgrade_path(self):
-        """Checks if active release to specified release is a valid upgrade path"""
-        # Get active release
-        isystem = self._sysinv_client.isystem.list()[0]
-        active_release = isystem.software_version
-
-        # supported_release is a dict with {release: required_patch}
-        supported_releases = dict()
-
-        # Parse upgrade metadata file for supported upgrade paths
-        root = ElementTree.parse("%s/../metadata.xml" % os.path.dirname(__file__))
-        upgrade_root = root.find("supported_upgrades").findall("upgrade")
-        for upgrade in upgrade_root:
-            version = upgrade.find("version")
-            required_patch = upgrade.find("required_patch")
-            supported_releases.update({version.text: required_patch.text if
-                                       required_patch is not None else None})
-        success = active_release in supported_releases
-        return success, active_release, supported_releases.get(active_release, None)
-
-    # TODO(heitormatsui): implement patch precheck targeted against USM
-    #  and implement patch precheck for subcloud
-    def _check_required_patch(self, required_patch):
-        """Checks if required patch for the supported release is installed"""
-        url = self._software_endpoint + '/query?show=deployed'
-        headers = {"X-Auth-Token": self._software_token}
-        response = requests.get(url, headers=headers, timeout=10)
-
-        success = True
-        required_patch = [required_patch] if required_patch else []
-        if response.status_code != 200:
-            print("Could not check required patches...")
-            return False, required_patch
-
-        applied_patches = list(response.json()["sd"].keys())
-        missing_patch = list(set(required_patch) - set(applied_patches))
-        if missing_patch:
-            success = False
-
-        return success, missing_patch
-
-    # TODO(heitormatsui) do we need this check on USM? Remove if we don't
-    def _check_active_is_controller_0(self):
-        """Checks that active controller is controller-0"""
-        controllers = self._sysinv_client.ihost.list()
-        for controller in controllers:
-            if controller.hostname == "controller-0" and \
-                    "Controller-Active" in controller.capabilities["Personality"]:
-                return True
-        return False
-
     def _check_license(self, version):
-        """Validates the current license is valid for the specified version"""
+        """
+        Validates the current license is valid for the specified version
+        :param version: version to be checked against installed license
+        :return: True is license is valid for version, False otherwise
+        """
         license_dict = self._sysinv_client.license.show()
         if license_dict["error"]:
             return False
@@ -135,9 +79,91 @@ class UpgradeHealthCheck(HealthCheck):
                 return False
         return True
 
+    # TODO(heitormatsui): implement patch precheck targeted against USM
+    #  and implement patch precheck for subcloud
+    def _check_deployed_state(self, required_patches):
+        """
+        Checks if every patch in a list is in 'deployed' state
+        :param required_patches: list of patches to be checked
+        :return: boolean indicating success/failure and list of patches
+                 that are not in the 'deployed' state
+        """
+        url = self._software_endpoint + '/query?show=deployed'
+        headers = {"X-Auth-Token": self._software_token}
+        response = requests.get(url, headers=headers, timeout=10)
+
+        success = True
+        if response.status_code != 200:
+            print("Could not check required patches...")
+            return False, required_patches
+
+        applied_patches = list(response.json()["sd"].keys())
+        missing_patch = list(set(required_patches) - set(applied_patches))
+        if missing_patch:
+            success = False
+
+        return success, missing_patch
+
+    def run_health_check(self):
+        """Run general health check using sysinv client"""
+        force = self._config.get("force", False)
+        health_ok = success = True
+
+        output = self._sysinv_client.health.get_kube_upgrade(args={}, relaxed=force)
+        if HealthCheck.FAIL_MSG in output:
+            success = False
+        health_ok = health_ok and success
+
+        # check installed license
+        success = self._check_license(self._major_release)
+        output += 'Installed license is valid: [%s]\n' \
+                  % (HealthCheck.SUCCESS_MSG if success else HealthCheck.FAIL_MSG)
+        health_ok = health_ok and success
+
+        return health_ok, output
+
+
+class UpgradeHealthCheck(HealthCheck):
+    """This class represents a upgrade-specific health check object
+    that verifies if system is in a valid state for upgrade"""
+
+    # TODO(heitormatsui): switch from using upgrade metadata xml to
+    #  the new USM metadata format
+    def _check_valid_upgrade_path(self):
+        """Checks if active release to specified release is a valid upgrade path"""
+        # Get active release
+        isystem = self._sysinv_client.isystem.list()[0]
+        active_release = isystem.software_version
+
+        # supported_release is a dict with {release: required_patch}
+        supported_releases = dict()
+
+        # Parse upgrade metadata file for supported upgrade paths
+        root = ElementTree.parse("/var/www/pages/feed/rel-%s/upgrades/metadata.xml" % self._major_release)
+        upgrade_root = root.find("supported_upgrades").findall("upgrade")
+        for upgrade in upgrade_root:
+            version = upgrade.find("version")
+            required_patch = upgrade.find("required_patch")
+            supported_releases.update({version.text: [required_patch.text] if
+                                       required_patch is not None else []})
+        success = active_release in supported_releases
+        return success, active_release, supported_releases.get(active_release, [])
+
+    # TODO(heitormatsui) do we need this check on USM? Remove if we don't
+    def _check_active_is_controller_0(self):
+        """Checks that active controller is controller-0"""
+        controllers = self._sysinv_client.ihost.list()
+        for controller in controllers:
+            if controller.hostname == "controller-0" and \
+                    "Controller-Active" in controller.capabilities["Personality"]:
+                return True
+        return False
+
     def _check_kube_version(self, supported_versions):
-        """Check if active k8s version is in a list of supported versions
+        """
-           :param supported_versions: list of supported k8s versions
+        Check if active k8s version is in a list of supported versions
+        :param supported_versions: list of supported k8s versions
+        :return: boolean indicating success/failure and active k8s version
         """
         kube_versions = self._sysinv_client.kube_version.list()
         active_version = None
@@ -153,35 +179,21 @@ class UpgradeHealthCheck(HealthCheck):
         health_ok = True
         output = ""
 
-        # get target release from script directory location
-        upgrade_release = re.match("^.*/rel-(\d\d.\d\d.\d*)/", __file__).group(1)
-
-        # check installed license
-        success = self._check_license(upgrade_release)
-        output += 'License valid for upgrade: [%s]\n' \
-                  % (HealthCheck.SUCCESS_MSG if success else HealthCheck.FAIL_MSG)
-
-        health_ok = health_ok and success
-
         # check if it is a valid upgrade path
-        success, active_release, required_patch = self._check_valid_upgrade_path()
+        success, active_release, required_patches = self._check_valid_upgrade_path()
         output += 'Valid upgrade path from release %s to %s: [%s]\n' \
-                  % (active_release, upgrade_release,
+                  % (active_release, self._major_release,
                      HealthCheck.SUCCESS_MSG if success else HealthCheck.FAIL_MSG)
         health_ok = health_ok and success
 
-        # check if required patches are applied/committed if is a valid upgrade path
+        # check if required patches are deployed
-        if success:
+        success, missing_patches = self._check_deployed_state(required_patches)
-            success, missing_patches = self._check_required_patch(required_patch)
+        output += 'Required patches are applied: [%s]\n' \
-            output += 'Required patches are applied: [%s]\n' \
+                  % (HealthCheck.SUCCESS_MSG if success else HealthCheck.FAIL_MSG)
-                      % (HealthCheck.SUCCESS_MSG if success else HealthCheck.FAIL_MSG)
+        if not success:
-            if not success:
+            output += '-> Patches not applied: [%s]\n' \
-                output += 'Patches not applied: %s\n' \
+                % ', '.join(missing_patches)
-                    % ', '.join(missing_patches)
+        health_ok = health_ok and success
-
-            health_ok = health_ok and success
-        else:
-            output += 'Invalid upgrade path, skipping required patches check...'
 
         # check if k8s version is valid
         success, active_version = self._check_kube_version(SUPPORTED_K8S_VERSIONS)
@@ -207,7 +219,46 @@ class UpgradeHealthCheck(HealthCheck):
         output += \
             'Active controller is controller-0: [%s]\n' \
             % (HealthCheck.SUCCESS_MSG if success else HealthCheck.FAIL_MSG)
+        health_ok = health_ok and success
 
+        return health_ok, output
+
+
+class PatchHealthCheck(HealthCheck):
+    """This class represents a patch-specific health check object
+    that verifies if system is in valid state to apply a patch"""
+
+    def _get_required_patches(self):
+        """Get required patches for a target release"""
+        url = self._software_endpoint + '/query'
+        headers = {"X-Auth-Token": self._software_token}
+        response = requests.get(url, headers=headers, timeout=10)
+
+        if response.status_code != 200:
+            print("Could not get required patches...")
+            return []
+
+        required_patches = []
+        for release, values in response.json()["sd"].items():
+            if values["sw_version"] == self._target_release:
+                required_patches.extend(values["requires"])
+                break
+
+        return required_patches
+
+    def run_health_check(self):
+        """Run specific patch health checks"""
+        health_ok = True
+        output = ""
+
+        # check required patches for target release
+        required_patches = self._get_required_patches()
+        success, missing_patches = self._check_deployed_state(required_patches)
+        output += 'Required patches are applied: [%s]\n' \
+                  % (HealthCheck.SUCCESS_MSG if success else HealthCheck.FAIL_MSG)
+        if not success:
+            output += '-> Patches not applied: [%s]\n' \
+                % ', '.join(missing_patches)
         health_ok = health_ok and success
 
         return health_ok, output
@@ -242,6 +293,9 @@ def parse_config(args=None):
     parser.add_argument("--force",
                         help="Ignore non-critical health checks",
                         action="store_true")
+    parser.add_argument("--patch",
+                        help="Set precheck to run against a patch release",
+                        action="store_true")
 
     # if args was not passed will use sys.argv by default
     parsed_args = parser.parse_args(args)
@@ -250,19 +304,25 @@ def parse_config(args=None):
 
 def main(argv=None):
     config = parse_config(argv)
+    patch_release = config.get("patch", False)
 
-    general_health_check = HealthCheck(config)
+    health_ok = True
-    upgrade_health_check = UpgradeHealthCheck(config)
+    output = ""
 
     # execute general health check
+    general_health_check = HealthCheck(config)
     general_health_ok, general_output = general_health_check.run_health_check()
 
-    # execute upgrade-specific health check
+    # execute release-specific health check
-    upgrade_health_ok, upgrade_output = upgrade_health_check.run_health_check()
+    if patch_release:
+        specific_health_check = PatchHealthCheck(config)
+    else:
+        specific_health_check = UpgradeHealthCheck(config)
+    specific_health_ok, specific_output = specific_health_check.run_health_check()
 
     # combine health check results removing extra line breaks/blank spaces from the output
-    health_ok = general_health_ok and upgrade_health_ok
+    health_ok = general_health_ok and specific_health_ok
-    output = general_output.strip() + "\n" + upgrade_output.strip()
+    output = general_output.strip() + "\n" + specific_output.strip()
 
     # print health check output and exit
     print(output)

software/software/constants.py

@@ -163,6 +163,7 @@ RELEASE_GA_NAME = "starlingx-%s"
 # Precheck constants
 LICENSE_FILE = "/etc/platform/.license"
 VERIFY_LICENSE_BINARY = "/usr/bin/verify-license"
+VERSIONED_SCRIPTS_DIR = "%s/rel-%%s/bin/" % SOFTWARE_STORAGE_DIR
 
 SOFTWARE_JSON_FILE = "%s/software.json" % SOFTWARE_STORAGE_DIR
 SYNCED_SOFTWARE_JSON_FILE = "%s/synced/software.json" % SOFTWARE_STORAGE_DIR

software/software/exceptions.py

@@ -117,6 +117,11 @@ class ReleaseVersionDoNotExist(SoftwareError):
     pass
 
 
+class VersionedDeployPrecheckFailure(SoftwareError):
+    """Versioned deploy-precheck script cannot be created"""
+    pass
+
+
 class FileSystemError(SoftwareError):
     """
     A failure during a linux file operation.

software/software/software_controller.py

@@ -1273,11 +1273,11 @@ class PatchController(PatchService):
         return local_info, local_warning, local_error, release_meta_info
 
     def _process_upload_patch_files(self, patch_files):
-        '''
+        """
         Process the uploaded patch files
         :param patch_files: list of patch files
         :return: info, warning, error messages
-        '''
+        """
 
         local_info = ""
         local_warning = ""
@@ -1345,7 +1345,6 @@ class PatchController(PatchService):
                                                            metadata_dir=constants.AVAILABLE_DIR,
                                                            base_pkgdata=self.base_pkgdata)
                     PatchFile.unpack_patch(patch_file)
-
                     local_info += "%s is now uploaded\n" % release_id
                     self.release_data.add_release(this_release)
 
@@ -1375,6 +1374,32 @@ class PatchController(PatchService):
                 }
             })
 
+        # create versioned precheck for uploaded patches
+        for patch in upload_patch_info:
+            filename, values = list(patch.items())[0]
+            LOG.info("Creating precheck for release %s..." % values.get("id"))
+            for pf in patch_files:
+                if filename in pf:
+                    patch_file = pf
+
+            sw_version = values.get("sw_version")
+            required_patches = self.release_data.metadata[values.get("id")].get("requires")
+
+            # sort the required patches list and get the latest, if available
+            req_patch_id = None
+            req_patch_metadata = None
+            req_patch_version = None
+            if required_patches:
+                req_patch_id = sorted(required_patches)[-1]
+            if req_patch_id:
+                req_patch_metadata = self.release_data.metadata.get(req_patch_id)
+            if req_patch_metadata:
+                req_patch_version = req_patch_metadata.get("sw_version")
+            if req_patch_id and not req_patch_metadata:
+                LOG.warning("Required patch '%s' is not uploaded." % req_patch_id)
+
+            PatchFile.create_versioned_precheck(patch_file, sw_version, req_patch_version=req_patch_version)
+
         return local_info, local_warning, local_error, upload_patch_info
 
     def software_release_upload(self, release_files):
@@ -1586,6 +1611,9 @@ class PatchController(PatchService):
                     LOG.info(msg)
                     msg_info += msg + "\n"
 
+            # TODO(lbonatti): treat the upcoming versioning changes
+            PatchFile.delete_versioned_directory(self.release_data.metadata[release_id]["sw_version"])
+
             try:
                 # Delete the metadata
                 deploystate = self.release_data.metadata[release_id]["state"]
@@ -2151,10 +2179,15 @@ class PatchController(PatchService):
 
         return release, success, msg_info, msg_warning, msg_error
 
-    def _deploy_precheck(self, release_version: str, force: bool, region_name: str = "RegionOne") -> dict:
+    def _deploy_precheck(self, release_version: str, force: bool = False,
+                         region_name: str = "RegionOne", patch: bool = False) -> dict:
         """
-        Verify if system is capable to upgrade to a specified deployment
+        Verify if system satisfy the requisites to upgrade to a specified deployment.
-        return: dict of info, warning and error messages
+        :param release_version: full release name, e.g. starlingx-MM.mm.pp
+        :param force: if True will ignore minor alarms during precheck
+        :param region_name: region_name
+        :param patch: if True then indicate precheck is for patch release
+        :return: dict of info, warning and error messages
         """
 
         msg_info = ""
@@ -2162,8 +2195,9 @@ class PatchController(PatchService):
         msg_error = ""
 
         precheck_script = utils.get_precheck_script(release_version)
+
         if not os.path.isfile(precheck_script):
-            msg = "Upgrade files for deployment %s are not present on the system, " \
+            msg = "Release files for deployment %s are not present on the system, " \
                   "cannot proceed with the precheck." % release_version
             LOG.error(msg)
             msg_error = "Fail to perform deploy precheck. " \
@@ -2204,6 +2238,8 @@ class PatchController(PatchService):
                "--region_name=%s" % region_name]
         if force:
             cmd.append("--force")
+        if patch:
+            cmd.append("--patch")
 
         # Call precheck from the deployment files
         precheck_return = subprocess.run(
@@ -2220,17 +2256,20 @@ class PatchController(PatchService):
 
         return dict(info=msg_info, warning=msg_warning, error=msg_error)
 
-    def software_deploy_precheck_api(self, deployment: str, force: bool, **kwargs) -> dict:
+    def software_deploy_precheck_api(self, deployment: str, force: bool = False, **kwargs) -> dict:
         """
-        Verify if system is capable to upgrade to a specified deployment
+        Verify if system satisfy the requisites to upgrade to a specified deployment.
-        return: dict of info, warning and error messages
+        :param deployment: full release name, e.g. starlingx-MM.mm.pp
+        :param force: if True will ignore minor alarms during precheck
+        :return: dict of info, warning and error messages
         """
         release, success, msg_info, msg_warning, msg_error = self._release_basic_checks(deployment)
         if not success:
             return dict(info=msg_info, warning=msg_warning, error=msg_error)
         region_name = kwargs["region_name"]
         release_version = release["sw_version"]
-        return self._deploy_precheck(release_version, force, region_name)
+        patch = not utils.is_upgrade_deploy(SW_VERSION, release_version)
+        return self._deploy_precheck(release_version, force, region_name, patch)
 
     def _deploy_upgrade_start(self, to_release):
         LOG.info("start deploy upgrade to %s from %s" % (to_release, SW_VERSION))
@@ -2294,9 +2333,12 @@ class PatchController(PatchService):
         if not success:
             return dict(info=msg_info, warning=msg_warning, error=msg_error)
 
+        # TODO(heitormatsui) Enforce deploy-precheck for patch release
+        patch_release = True
         if utils.is_upgrade_deploy(SW_VERSION, release["sw_version"]):
+            patch_release = False
             to_release = release["sw_version"]
-            ret = self._deploy_precheck(to_release, force)
+            ret = self._deploy_precheck(to_release, force, patch=patch_release)
             if ret["error"]:
                 ret["error"] = "The following issues have been detected which prevent " \
                                "deploying %s\n" % deployment + \
@@ -2349,13 +2391,12 @@ class PatchController(PatchService):
         else:
             operation = "remove"
 
-        # If releases are such that R2 requires R1
+        # If releases are such that:
-        # R3 requires R2
+        # R2 requires R1, R3 requires R2, R4 requires R3
-        # R4 requires R3
+        # If current running release is R2 and command issued is "software deploy start R4"
-        # And current running release is R2
+        # operation is "apply" with order [R3, R4]
-        # And command issued is "software deploy start R4"
+        # If current running release is R4 and command issued is "software deploy start R2"
-        # Order for apply operation: [R3, R4]
+        # operation is "remove" with order [R4, R3]
-        # Order for remove operation: [R3]
         if operation == "apply":
 
             collect_current_load_for_hosts()

software/software/software_functions.py

@@ -34,6 +34,7 @@ from software.exceptions import ReleaseValidationFailure
 from software.exceptions import ReleaseMismatchFailure
 from software.exceptions import SoftwareFail
 from software.exceptions import SoftwareServiceError
+from software.exceptions import VersionedDeployPrecheckFailure
 
 import software.constants as constants
 import software.utils as utils
@@ -981,6 +982,69 @@ class PatchFile(object):
             os.chdir(orig_wd)
             shutil.rmtree(patch_tmpdir)
 
+    @staticmethod
+    def create_versioned_precheck(patch, sw_version, req_patch_version=None):
+        """
+        Extract the deploy-precheck script from the patch into
+        a versioned directory under SOFTWARE_STORAGE_DIR and,
+        if script is not available in the patch, then create a
+        symlink to the versioned directory of the required patch.
+        :param patch: path to patch file
+        :param sw_version: patch version in MM.mm.pp format
+        :param req_patch_version: required patch version in MM.mm.pp format
+        """
+        # open patch and create versioned scripts directory
+        tar = tarfile.open(patch, "r:gz")
+        versioned_dir = constants.VERSIONED_SCRIPTS_DIR % sw_version
+        versioned_script = os.path.join(versioned_dir, constants.DEPLOY_PRECHECK_SCRIPT)
+        if os.path.exists(versioned_dir):
+            shutil.rmtree(versioned_dir)
+        os.makedirs(versioned_dir)
+
+        error_msg = "Versioned precheck script cannot be created, "
+        try:
+            # if patch contains precheck script, copy it to versioned directory
+            if constants.DEPLOY_PRECHECK_SCRIPT in tar.getnames():
+                tar.extract(constants.DEPLOY_PRECHECK_SCRIPT, path=versioned_dir)
+                os.chmod(versioned_script, mode=0o755)
+                LOG.info("Versioned precheck script copied to %s." % versioned_script)
+            # in case patch does not contain a precheck script
+            # then symlink to required patch versioned directory
+            else:
+                LOG.info("'%s' script is not included in the patch, will attempt to "
+                         "symlink to the 'required patch' precheck script." %
+                         constants.DEPLOY_PRECHECK_SCRIPT)
+                if not req_patch_version:
+                    error_msg += "'required patch' version could not be determined."
+                    raise VersionedDeployPrecheckFailure
+
+                req_versioned_dir = constants.VERSIONED_SCRIPTS_DIR % req_patch_version
+                req_versioned_script = os.path.join(req_versioned_dir, constants.DEPLOY_PRECHECK_SCRIPT)
+                # if required patch directory does not exist create the link anyway
+                if not os.path.exists(req_versioned_dir):
+                    LOG.warning("'required patch' versioned directory %s does not exist."
+                                % req_versioned_dir)
+                os.symlink(req_versioned_script, versioned_script)
+                LOG.info("Versioned precheck script %s symlinked to %s." % (
+                    versioned_script, req_versioned_script))
+        except Exception as e:
+            LOG.warning("%s: %s" % (error_msg, e))
+
+    @staticmethod
+    def delete_versioned_directory(sw_version):
+        """
+        Delete the versioned deploy-precheck script.
+        :param sw_version: precheck script version to be deleted
+        """
+        try:
+            opt_release_folder = "%s/rel-%s" % (constants.SOFTWARE_STORAGE_DIR,
+                                                sw_version)
+            if os.path.isdir(opt_release_folder):
+                shutil.rmtree(opt_release_folder, ignore_errors=True)
+            LOG.info("Versioned directory %s deleted." % opt_release_folder)
+        except Exception as e:
+            LOG.exception("Failed to delete versioned precheck: %s", e)
+
 
 def patch_build():
     configure_logging(logtofile=False)
@@ -1211,5 +1275,12 @@ def parse_release_metadata(filename):
     root = tree.getroot()
     data = {}
     for child in root:
+        # get requires under <req_patch_id> key
+        if child.tag == "requires":
+            requires = []
+            for item in child:
+                requires.append(item.text)
+            data[child.tag] = requires
+            continue
         data[child.tag] = child.text
     return data