diff --git a/software/software/constants.py b/software/software/constants.py index 39ddcf49..5e0f0bc7 100644 --- a/software/software/constants.py +++ b/software/software/constants.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2023 Wind River Systems, Inc. +Copyright (c) 2023-2024 Wind River Systems, Inc. SPDX-License-Identifier: Apache-2.0 @@ -26,12 +26,15 @@ SOFTWARE_CONFIG_FILE_LOCAL = "/etc/software/software.conf" AVAILABLE_DIR = "%s/metadata/available" % SOFTWARE_STORAGE_DIR UNAVAILABLE_DIR = "%s/metadata/unavailable" % SOFTWARE_STORAGE_DIR +DEPLOYING_DIR = "%s/metadata/deploying" % SOFTWARE_STORAGE_DIR +DEPLOYED_DIR = "%s/metadata/deployed" % SOFTWARE_STORAGE_DIR +REMOVING_DIR = "%s/metadata/removing" % SOFTWARE_STORAGE_DIR + +# TODO(bqian) states to be removed once current references are removed DEPLOYING_START_DIR = "%s/metadata/deploying_start" % SOFTWARE_STORAGE_DIR DEPLOYING_HOST_DIR = "%s/metadata/deploying_host" % SOFTWARE_STORAGE_DIR DEPLOYING_ACTIVATE_DIR = "%s/metadata/deploying_activate" % SOFTWARE_STORAGE_DIR DEPLOYING_COMPLETE_DIR = "%s/metadata/deploying_complete" % SOFTWARE_STORAGE_DIR -DEPLOYED_DIR = "%s/metadata/deployed" % SOFTWARE_STORAGE_DIR -REMOVING_DIR = "%s/metadata/removing" % SOFTWARE_STORAGE_DIR ABORTING_DIR = "%s/metadata/aborting" % SOFTWARE_STORAGE_DIR COMMITTED_DIR = "%s/metadata/committed" % SOFTWARE_STORAGE_DIR SEMANTICS_DIR = "%s/semantics" % SOFTWARE_STORAGE_DIR @@ -40,33 +43,57 @@ DEPLOY_STATE_METADATA_DIR = \ [ AVAILABLE_DIR, UNAVAILABLE_DIR, + DEPLOYING_DIR, + DEPLOYED_DIR, + REMOVING_DIR, + # TODO(bqian) states to be removed once current references are removed DEPLOYING_START_DIR, DEPLOYING_HOST_DIR, DEPLOYING_ACTIVATE_DIR, DEPLOYING_COMPLETE_DIR, - DEPLOYED_DIR, - REMOVING_DIR, ABORTING_DIR, COMMITTED_DIR, ] -ABORTING = 'aborting' +# new release state needs to be added to VALID_RELEASE_STATES list AVAILABLE = 'available' -COMMITTED = 'committed' +UNAVAILABLE = 'unavailable' +DEPLOYING = 'deploying' DEPLOYED = 'deployed' +REMOVING = 'removing' +UNKNOWN = 'n/a' + +# TODO(bqian) states to be removed once current references are removed +ABORTING = 'aborting' +COMMITTED = 'committed' DEPLOYING_ACTIVATE = 'deploying-activate' DEPLOYING_COMPLETE = 'deploying-complete' DEPLOYING_HOST = 'deploying-host' DEPLOYING_START = 'deploying-start' -REMOVING = 'removing' UNAVAILABLE = 'unavailable' -UNKNOWN = 'n/a' VALID_DEPLOY_START_STATES = [ AVAILABLE, DEPLOYED, ] +VALID_RELEASE_STATES = [AVAILABLE, UNAVAILABLE, DEPLOYING, DEPLOYED, + REMOVING] + +RELEASE_STATE_TO_DIR_MAP = {AVAILABLE: AVAILABLE_DIR, + UNAVAILABLE: UNAVAILABLE_DIR, + DEPLOYING: DEPLOYING_DIR, + DEPLOYED: DEPLOYED_DIR, + REMOVING: REMOVING_DIR} + +# valid release state transition below could still be changed as +# development continue +RELEASE_STATE_VALID_TRANSITION = { + AVAILABLE: [DEPLOYING], + DEPLOYING: [DEPLOYED], + DEPLOYED: [REMOVING, UNAVAILABLE] +} + STATUS_DEVELOPEMENT = 'DEV' STATUS_OBSOLETE = 'OBS' STATUS_RELEASED = 'REL' diff --git a/software/software/exceptions.py b/software/software/exceptions.py index b8ebb607..6a905789 100644 --- a/software/software/exceptions.py +++ b/software/software/exceptions.py @@ -117,6 +117,19 @@ class ReleaseVersionDoNotExist(SoftwareError): pass +class FileSystemError(SoftwareError): + """ + A failure during a linux file operation. + Likely fixable by a root user. + """ + pass + + +class InternalError(Exception): + """This is an internal error aka bug""" + pass + + class SoftwareServiceError(Exception): """ This is a service error, such as file system issue or configuration diff --git a/software/software/release_data.py b/software/software/release_data.py new file mode 100644 index 00000000..afd9a29e --- /dev/null +++ b/software/software/release_data.py @@ -0,0 +1,212 @@ +# +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright (c) 2024 Wind River Systems, Inc. +# + +import os +import shutil +from software import constants +from software.exceptions import FileSystemError +from software.exceptions import InternalError +from software.software_functions import LOG + + +class SWRelease(object): + '''wrapper class to group matching metadata and contents''' + + def __init__(self, rel_id, metadata, contents): + self._id = rel_id + self._metadata = metadata + self._contents = contents + + @property + def metadata(self): + return self._metadata + + @property + def contents(self): + return self._contents + + @property + def id(self): + return self._id + + @property + def state(self): + return self.metadata['state'] + + @staticmethod + def is_valid_state_transition(from_state, to_state): + if to_state not in constants.VALID_RELEASE_STATES: + msg = "Invalid state %s." % to_state + LOG.error(msg) + # this is a bug + raise InternalError(msg) + + if from_state in constants.RELEASE_STATE_VALID_TRANSITION: + if to_state in constants.RELEASE_STATE_VALID_TRANSITION[from_state]: + return True + return False + + @staticmethod + def ensure_state_transition(to_state): + to_dir = constants.RELEASE_STATE_TO_DIR_MAP[to_state] + if not os.path.isdir(to_dir): + try: + os.makedirs(to_dir, mode=0o755, exist_ok=True) + except FileExistsError: + error = "Cannot create directory %s" % to_dir + raise FileSystemError(error) + + def update_state(self, state): + if SWRelease.is_valid_state_transition(self.state, state): + LOG.info("%s state from %s to %s" % (self.id, self.state, state)) + SWRelease.ensure_state_transition(state) + + to_dir = constants.RELEASE_STATE_TO_DIR_MAP[state] + from_dir = constants.RELEASE_STATE_TO_DIR_MAP[self.state] + try: + shutil.move("%s/%s-metadata.xml" % (from_dir, self.id), + "%s/%s-metadata.xml" % (to_dir, self.id)) + except shutil.Error: + msg = "Failed to move the metadata for %s" % self.id + LOG.exception(msg) + raise FileSystemError(msg) + + self.metadata['state'] = state + else: + # this is a bug + error = "Invalid state transition %s, current is %s, target state is %s" % \ + (self.id, self.state, state) + LOG.info(error) + raise InternalError(error) + + @property + def sw_version(self): + return self.metadata['sw_version'] + + def _get_latest_commit(self): + num_commits = self.contents['number_of_commits'] + if int(num_commits) > 0: + commit_tag = "commit%s" % num_commits + return self.contents[commit_tag] + else: + # may consider raise InvalidRelease exception in this case after + # iso metadata comes with commit id + LOG.warning("Commit data not found in metadata. Release %s" % + self.id) + return None + + @property + def commit_id(self): + commit = self._get_latest_commit() + if commit is not None: + return commit['commit'] + else: + # may consider raise InvalidRelease exception when iso comes with + # latest commit + return None + + def _get_by_key(self, key, default=None): + if key in self._metadata: + return self._metadata[key] + else: + return default + + @property + def summary(self): + return self._get_by_key('summary') + + @property + def description(self): + return self._get_by_key('description') + + @property + def install_instructions(self): + return self._get_by_key('install_instructions') + + @property + def warnings(self): + return self._get_by_key('warnings') + + @property + def status(self): + return self._get_by_key('status') + + @property + def unremovable(self): + return self._get_by_key('unremovable') + + @property + def reboot_required(self): + return self._get_by_key('reboot_required') + + @property + def restart_script(self): + return self._get_by_key('restart_script') + + @property + def commit_checksum(self): + commit = self._get_latest_commit() + if commit is not None: + return commit['checksum'] + else: + # may consider raise InvalidRelease exception when iso comes with + # latest commit + return None + + +class SWReleaseCollection(object): + '''SWReleaseCollection encapsulates aggregated software release collection + managed by USM. + ''' + + def __init__(self, release_data): + self._sw_releases = {} + for rel_id in release_data.metadata: + rel_data = release_data.metadata[rel_id] + contents = release_data.contents[rel_id] + sw_release = SWRelease(rel_id, rel_data, contents) + self._sw_releases[rel_id] = sw_release + + def get_release_by_id(self, rel_id): + if rel_id in self._sw_releases: + return self._sw_releases[rel_id] + return None + + def get_release_by_commit_id(self, commit_id): + for _, sw_release in self._sw_releases: + if sw_release.commit_id == commit_id: + return sw_release + return None + + def iterate_releases_by_state(self, state): + '''return iteration of releases matching specified state. + sorted by id in ascending order + ''' + sorted_list = sorted(self._sw_releases) + for rel_id in sorted_list: + rel_data = self._sw_releases[rel_id] + if rel_data.metadata['state'] == state: + yield rel_data + + def iterate_releases(self): + '''return iteration of all releases sorted by id in ascending order''' + sorted_list = sorted(self._sw_releases) + for rel_id in sorted_list: + yield self._sw_releases[rel_id] + + def update_state(self, list_of_releases, state): + for release_id in list_of_releases: + release = self.get_release_by_id(release_id) + if release is not None: + if SWRelease.is_valid_state_transition(release.state, state): + SWRelease.ensure_state_transition(state) + else: + LOG.error("release %s not found" % release_id) + + for release_id in list_of_releases: + release = self.get_release_by_id(release_id) + if release is not None: + release.update_state(state) diff --git a/software/software/software_controller.py b/software/software/software_controller.py index 33372256..afe920d7 100644 --- a/software/software/software_controller.py +++ b/software/software/software_controller.py @@ -33,6 +33,7 @@ from software.constants import DEPLOY_STATES from software.base import PatchService from software.exceptions import APTOSTreeCommandFail from software.db import api as db_api +from software.exceptions import InternalError from software.exceptions import MetadataFail from software.exceptions import UpgradeNotSupported from software.exceptions import OSTreeCommandFail @@ -43,6 +44,7 @@ from software.exceptions import ReleaseInvalidRequest from software.exceptions import ReleaseValidationFailure from software.exceptions import ReleaseMismatchFailure from software.exceptions import ReleaseIsoDeleteFailure +from software.release_data import SWReleaseCollection from software.software_functions import collect_current_load_for_hosts from software.software_functions import parse_release_metadata from software.software_functions import configure_logging @@ -653,6 +655,13 @@ class PatchController(PatchService): else: self.write_state_file() + @property + def release_collection(self): + # for this stage, the SWReleaseCollection behaves as a broker which + # does not hold any release data. it only last one request + swrc = SWReleaseCollection(self.release_data) + return swrc + def update_config(self): cfg.read_config() @@ -2035,12 +2044,15 @@ class PatchController(PatchService): ret["error"] += "Please fix above issues then retry the deploy.\n" return ret - collect_current_load_for_hosts() if self._deploy_upgrade_start(to_release): collect_current_load_for_hosts() dbapi = db_api.get_instance() dbapi.create_deploy(SW_VERSION, to_release, True) dbapi.update_deploy(DEPLOY_STATES.DATA_MIGRATION) + sw_rel = self.release_collection.get_release_by_id(deployment) + if sw_rel is None: + raise InternalError("%s cannot be found" % to_release) + sw_rel.update_state(constants.DEPLOYING) msg_info = "Deployment for %s started" % deployment else: msg_error = "Deployment for %s failed to start" % deployment diff --git a/software/software/software_functions.py b/software/software/software_functions.py index 55ab702c..f33c11c5 100644 --- a/software/software/software_functions.py +++ b/software/software/software_functions.py @@ -299,9 +299,14 @@ class ReleaseData(object): :param state: Indicates Applied, Available, or Committed :return: Release ID """ - tree = ElementTree.parse(filename) - root = tree.getroot() + with open(filename, "r") as f: + text = f.read() + + return self.parse_metadata_string(text, state) + + def parse_metadata_string(self, text, state): + root = ElementTree.fromstring(text) # # # PATCH_0001 diff --git a/software/software/tests/test_software_function.py b/software/software/tests/test_software_function.py new file mode 100644 index 00000000..a22014a9 --- /dev/null +++ b/software/software/tests/test_software_function.py @@ -0,0 +1,187 @@ +# +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright (c) 2024 Wind River Systems, Inc. +# +import unittest +from software.release_data import SWReleaseCollection +from software.software_functions import ReleaseData + +metadata = """ + + 23.09_RR_ALL_NODES + 23.09 + Debian patch test + Reboot required patch + Sample instructions + Sample warning + TST + Y + Y + + + 1 + + 0db647647b009c5cc02410d461de0870049bdeb66caf1bdc1ccd189ac83b8e92 + bae3ff59c5f59c95aa8d3ccf8c1364c4c869cd428f7b5032a00a8b777cc132f7 + + + 38453dcb1aeb5bb9394ed02c0e6b8f2f913d00a827c89faf98cb63dff503b8e2 + 2f742b1b719f19b302c306604659ccf4aa61a1fdb7742ac79b009c79af18c79b + + + + + +""" + +metadata2 = """ + + 23.09_NRR_INSVC + 23.09 + Debian patch test + In service patch + Sample instructions2 + Sample warning2 + DEV + N + N + 23.09_NRR_INSVC_example-cgcs-patch-restart + + + 1 + + 0db647647b009c5cc02410d461de0870049bdeb66caf1bdc1ccd189ac83b8e92 + bae3ff59c5f59c95aa8d3ccf8c1364c4c869cd428f7b5032a00a8b777cc132f7 + + + 0b53576092a189133d56eac49ae858c1218f480a4a859eaca2b47f2604a4e0e7 + 2f742b1b719f19b302c306604659ccf4aa61a1fdb7742ac79b009c79af18c79b + + + + + +""" + +expected_values = [ + { + "release_id": "23.09_NRR_INSVC", + "version": "23.09", + "state": "deployed", + "summary": "Debian patch test", + "description": "In service patch", + "install_instructions": "Sample instructions2", + "warnings": "Sample warning2", + "status": "DEV", + "unremovable": "N", + "restart_script": "23.09_NRR_INSVC_example-cgcs-patch-restart", + "commit_id": "0b53576092a189133d56eac49ae858c1218f480a4a859eaca2b47f2604a4e0e7", + "checksum": "2f742b1b719f19b302c306604659ccf4aa61a1fdb7742ac79b009c79af18c79b", + }, + { + "release_id": "23.09_RR_ALL_NODES", + "version": "23.09", + "state": "available", + "summary": "Debian patch test", + "description": "Reboot required patch", + "install_instructions": "Sample instructions", + "warnings": "Sample warning", + "status": "TST", + "unremovable": "Y", + "restart_script": None, + "commit_id": "38453dcb1aeb5bb9394ed02c0e6b8f2f913d00a827c89faf98cb63dff503b8e2", + "checksum": "2f742b1b719f19b302c306604659ccf4aa61a1fdb7742ac79b009c79af18c79b", + } +] + +package_dir = {"23.09": "/var/www/page/feed/rel_23.09"} + + +class TestSoftwareFunction(unittest.TestCase): + + def setUp(self): + pass + + def tearDown(self): + pass + + @property + def release_collection(self): + rd = ReleaseData() + rd.parse_metadata_string(metadata, "available") + rd2 = ReleaseData() + rd2.parse_metadata_string(metadata2, "deployed") + rd.add_release(rd2) + + rc = SWReleaseCollection(rd) + return rc + + def test_SWReleaseCollection_iterate_releases(self): + idx = 0 + for r in self.release_collection.iterate_releases(): + val = expected_values[idx] + idx += 1 + self.assertEqual(val["release_id"], r.id) + self.assertEqual(val["version"], r.sw_version) + self.assertEqual(val["state"], r.state) + self.assertEqual(val["summary"], r.summary) + self.assertEqual(val["description"], r.description) + self.assertEqual(val["install_instructions"], r.install_instructions) + self.assertEqual(val["warnings"], r.warnings) + self.assertEqual(val["status"], r.status) + self.assertEqual(val["unremovable"], r.unremovable) + if val["restart_script"] is None: + self.assertIsNone(r.restart_script) + else: + self.assertEqual(val["restart_script"], r.restart_script) + self.assertEqual(val["commit_id"], r.commit_id) + self.assertEqual(val["checksum"], r.commit_checksum) + + def test_SWReleaseCollection_get_release_by_id(self): + rd = ReleaseData() + rd.parse_metadata_string(metadata, "available") + rd2 = ReleaseData() + rd2.parse_metadata_string(metadata2, "deployed") + rd.add_release(rd2) + + rc = SWReleaseCollection(rd) + + idx = 0 + rid = expected_values[idx]["release_id"] + r = rc.get_release_by_id(rid) + val = expected_values[idx] + self.assertEqual(val["release_id"], r.id) + self.assertEqual(val["version"], r.sw_version) + self.assertEqual(val["state"], r.state) + self.assertEqual(val["summary"], r.summary) + self.assertEqual(val["description"], r.description) + self.assertEqual(val["install_instructions"], r.install_instructions) + self.assertEqual(val["warnings"], r.warnings) + self.assertEqual(val["status"], r.status) + self.assertEqual(val["unremovable"], r.unremovable) + if val["restart_script"] is None: + self.assertIsNone(r.restart_script) + else: + self.assertEqual(val["restart_script"], r.restart_script) + self.assertEqual(val["commit_id"], r.commit_id) + self.assertEqual(val["checksum"], r.commit_checksum) + + def test_SWReleaseCollection_iterate_release_by_state(self): + val = expected_values[0] + for r in self.release_collection.iterate_releases_by_state('deployed'): + self.assertEqual(val["release_id"], r.id) + self.assertEqual(val["version"], r.sw_version) + self.assertEqual(val["state"], r.state) + self.assertEqual(val["summary"], r.summary) + self.assertEqual(val["description"], r.description) + self.assertEqual(val["install_instructions"], r.install_instructions) + self.assertEqual(val["warnings"], r.warnings) + self.assertEqual(val["status"], r.status) + self.assertEqual(val["unremovable"], r.unremovable) + if val["restart_script"] is None: + self.assertIsNone(r.restart_script) + else: + self.assertEqual(val["restart_script"], r.restart_script) + self.assertEqual(val["commit_id"], r.commit_id) + self.assertEqual(val["checksum"], r.commit_checksum)