starlingx
/
update
Code Issues Proposed changes

Merge "Create release_data wrapper classes"

This commit is contained in:
Zuul 2024-02-16 15:14:07 +00:00 committed by Gerrit Code Review
parent 5b37af0ef6 0378788233
commit ce3dc63c3e
6 changed files with 468 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
software/software/constants.py
View File

@ -1,5 +1,5 @@
"""
Copyright (c) 2023 Wind River Systems, Inc.
Copyright (c) 2023-2024 Wind River Systems, Inc.
SPDX-License-Identifier: Apache-2.0
@ -26,12 +26,15 @@ SOFTWARE_CONFIG_FILE_LOCAL = "/etc/software/software.conf"
AVAILABLE_DIR = "%s/metadata/available" % SOFTWARE_STORAGE_DIR
UNAVAILABLE_DIR = "%s/metadata/unavailable" % SOFTWARE_STORAGE_DIR
DEPLOYING_DIR = "%s/metadata/deploying" % SOFTWARE_STORAGE_DIR
DEPLOYED_DIR = "%s/metadata/deployed" % SOFTWARE_STORAGE_DIR
REMOVING_DIR = "%s/metadata/removing" % SOFTWARE_STORAGE_DIR
# TODO(bqian) states to be removed once current references are removed
DEPLOYING_START_DIR = "%s/metadata/deploying_start" % SOFTWARE_STORAGE_DIR
DEPLOYING_HOST_DIR = "%s/metadata/deploying_host" % SOFTWARE_STORAGE_DIR
DEPLOYING_ACTIVATE_DIR = "%s/metadata/deploying_activate" % SOFTWARE_STORAGE_DIR
DEPLOYING_COMPLETE_DIR = "%s/metadata/deploying_complete" % SOFTWARE_STORAGE_DIR
DEPLOYED_DIR = "%s/metadata/deployed" % SOFTWARE_STORAGE_DIR
REMOVING_DIR = "%s/metadata/removing" % SOFTWARE_STORAGE_DIR
ABORTING_DIR = "%s/metadata/aborting" % SOFTWARE_STORAGE_DIR
COMMITTED_DIR = "%s/metadata/committed" % SOFTWARE_STORAGE_DIR
SEMANTICS_DIR = "%s/semantics" % SOFTWARE_STORAGE_DIR
@ -40,33 +43,57 @@ DEPLOY_STATE_METADATA_DIR = \
[
AVAILABLE_DIR,
UNAVAILABLE_DIR,
DEPLOYING_DIR,
DEPLOYED_DIR,
REMOVING_DIR,
# TODO(bqian) states to be removed once current references are removed
DEPLOYING_START_DIR,
DEPLOYING_HOST_DIR,
DEPLOYING_ACTIVATE_DIR,
DEPLOYING_COMPLETE_DIR,
DEPLOYED_DIR,
REMOVING_DIR,
ABORTING_DIR,
COMMITTED_DIR,
]
ABORTING = 'aborting'
# new release state needs to be added to VALID_RELEASE_STATES list
AVAILABLE = 'available'
COMMITTED = 'committed'
UNAVAILABLE = 'unavailable'
DEPLOYING = 'deploying'
DEPLOYED = 'deployed'
REMOVING = 'removing'
UNKNOWN = 'n/a'
# TODO(bqian) states to be removed once current references are removed
ABORTING = 'aborting'
COMMITTED = 'committed'
DEPLOYING_ACTIVATE = 'deploying-activate'
DEPLOYING_COMPLETE = 'deploying-complete'
DEPLOYING_HOST = 'deploying-host'
DEPLOYING_START = 'deploying-start'
REMOVING = 'removing'
UNAVAILABLE = 'unavailable'
UNKNOWN = 'n/a'
VALID_DEPLOY_START_STATES = [
AVAILABLE,
DEPLOYED,
]
VALID_RELEASE_STATES = [AVAILABLE, UNAVAILABLE, DEPLOYING, DEPLOYED,
REMOVING]
RELEASE_STATE_TO_DIR_MAP = {AVAILABLE: AVAILABLE_DIR,
UNAVAILABLE: UNAVAILABLE_DIR,
DEPLOYING: DEPLOYING_DIR,
DEPLOYED: DEPLOYED_DIR,
REMOVING: REMOVING_DIR}
# valid release state transition below could still be changed as
# development continue
RELEASE_STATE_VALID_TRANSITION = {
AVAILABLE: [DEPLOYING],
DEPLOYING: [DEPLOYED],
DEPLOYED: [REMOVING, UNAVAILABLE]
}
STATUS_DEVELOPEMENT = 'DEV'
STATUS_OBSOLETE = 'OBS'
STATUS_RELEASED = 'REL'

13
software/software/exceptions.py
View File

@ -117,6 +117,19 @@ class ReleaseVersionDoNotExist(SoftwareError):
pass
class FileSystemError(SoftwareError):
"""
A failure during a linux file operation.
Likely fixable by a root user.
"""
pass
class InternalError(Exception):
"""This is an internal error aka bug"""
pass
class SoftwareServiceError(Exception):
"""
This is a service error, such as file system issue or configuration

212
software/software/release_data.py Normal file
View File

@ -0,0 +1,212 @@
#
# SPDX-License-Identifier: Apache-2.0
#
# Copyright (c) 2024 Wind River Systems, Inc.
#
import os
import shutil
from software import constants
from software.exceptions import FileSystemError
from software.exceptions import InternalError
from software.software_functions import LOG
class SWRelease(object):
'''wrapper class to group matching metadata and contents'''
def __init__(self, rel_id, metadata, contents):
self._id = rel_id
self._metadata = metadata
self._contents = contents
@property
def metadata(self):
return self._metadata
@property
def contents(self):
return self._contents
@property
def id(self):
return self._id
@property
def state(self):
return self.metadata['state']
@staticmethod
def is_valid_state_transition(from_state, to_state):
if to_state not in constants.VALID_RELEASE_STATES:
msg = "Invalid state %s." % to_state
LOG.error(msg)
# this is a bug
raise InternalError(msg)
if from_state in constants.RELEASE_STATE_VALID_TRANSITION:
if to_state in constants.RELEASE_STATE_VALID_TRANSITION[from_state]:
return True
return False
@staticmethod
def ensure_state_transition(to_state):
to_dir = constants.RELEASE_STATE_TO_DIR_MAP[to_state]
if not os.path.isdir(to_dir):
try:
os.makedirs(to_dir, mode=0o755, exist_ok=True)
except FileExistsError:
error = "Cannot create directory %s" % to_dir
raise FileSystemError(error)
def update_state(self, state):
if SWRelease.is_valid_state_transition(self.state, state):
LOG.info("%s state from %s to %s" % (self.id, self.state, state))
SWRelease.ensure_state_transition(state)
to_dir = constants.RELEASE_STATE_TO_DIR_MAP[state]
from_dir = constants.RELEASE_STATE_TO_DIR_MAP[self.state]
try:
shutil.move("%s/%s-metadata.xml" % (from_dir, self.id),
"%s/%s-metadata.xml" % (to_dir, self.id))
except shutil.Error:
msg = "Failed to move the metadata for %s" % self.id
LOG.exception(msg)
raise FileSystemError(msg)
self.metadata['state'] = state
else:
# this is a bug
error = "Invalid state transition %s, current is %s, target state is %s" % \
(self.id, self.state, state)
LOG.info(error)
raise InternalError(error)
@property
def sw_version(self):
return self.metadata['sw_version']
def _get_latest_commit(self):
num_commits = self.contents['number_of_commits']
if int(num_commits) > 0:
commit_tag = "commit%s" % num_commits
return self.contents[commit_tag]
else:
# may consider raise InvalidRelease exception in this case after
# iso metadata comes with commit id
LOG.warning("Commit data not found in metadata. Release %s" %
self.id)
return None
@property
def commit_id(self):
commit = self._get_latest_commit()
if commit is not None:
return commit['commit']
else:
# may consider raise InvalidRelease exception when iso comes with
# latest commit
return None
def _get_by_key(self, key, default=None):
if key in self._metadata:
return self._metadata[key]
else:
return default
@property
def summary(self):
return self._get_by_key('summary')
@property
def description(self):
return self._get_by_key('description')
@property
def install_instructions(self):
return self._get_by_key('install_instructions')
@property
def warnings(self):
return self._get_by_key('warnings')
@property
def status(self):
return self._get_by_key('status')
@property
def unremovable(self):
return self._get_by_key('unremovable')
@property
def reboot_required(self):
return self._get_by_key('reboot_required')
@property
def restart_script(self):
return self._get_by_key('restart_script')
@property
def commit_checksum(self):
commit = self._get_latest_commit()
if commit is not None:
return commit['checksum']
else:
# may consider raise InvalidRelease exception when iso comes with
# latest commit
return None
class SWReleaseCollection(object):
'''SWReleaseCollection encapsulates aggregated software release collection
managed by USM.
'''
def __init__(self, release_data):
self._sw_releases = {}
for rel_id in release_data.metadata:
rel_data = release_data.metadata[rel_id]
contents = release_data.contents[rel_id]
sw_release = SWRelease(rel_id, rel_data, contents)
self._sw_releases[rel_id] = sw_release
def get_release_by_id(self, rel_id):
if rel_id in self._sw_releases:
return self._sw_releases[rel_id]
return None
def get_release_by_commit_id(self, commit_id):
for _, sw_release in self._sw_releases:
if sw_release.commit_id == commit_id:
return sw_release
return None
def iterate_releases_by_state(self, state):
'''return iteration of releases matching specified state.
sorted by id in ascending order
'''
sorted_list = sorted(self._sw_releases)
for rel_id in sorted_list:
rel_data = self._sw_releases[rel_id]
if rel_data.metadata['state'] == state:
yield rel_data
def iterate_releases(self):
'''return iteration of all releases sorted by id in ascending order'''
sorted_list = sorted(self._sw_releases)
for rel_id in sorted_list:
yield self._sw_releases[rel_id]
def update_state(self, list_of_releases, state):
for release_id in list_of_releases:
release = self.get_release_by_id(release_id)
if release is not None:
if SWRelease.is_valid_state_transition(release.state, state):
SWRelease.ensure_state_transition(state)
else:
LOG.error("release %s not found" % release_id)
for release_id in list_of_releases:
release = self.get_release_by_id(release_id)
if release is not None:
release.update_state(state)

14
software/software/software_controller.py
View File

@ -33,6 +33,7 @@ from software.constants import DEPLOY_STATES
from software.base import PatchService
from software.exceptions import APTOSTreeCommandFail
from software.db import api as db_api
from software.exceptions import InternalError
from software.exceptions import MetadataFail
from software.exceptions import UpgradeNotSupported
from software.exceptions import OSTreeCommandFail
@ -43,6 +44,7 @@ from software.exceptions import ReleaseInvalidRequest
from software.exceptions import ReleaseValidationFailure
from software.exceptions import ReleaseMismatchFailure
from software.exceptions import ReleaseIsoDeleteFailure
from software.release_data import SWReleaseCollection
from software.software_functions import collect_current_load_for_hosts
from software.software_functions import parse_release_metadata
from software.software_functions import configure_logging
@ -653,6 +655,13 @@ class PatchController(PatchService):
else:
self.write_state_file()
@property
def release_collection(self):
# for this stage, the SWReleaseCollection behaves as a broker which
# does not hold any release data. it only last one request
swrc = SWReleaseCollection(self.release_data)
return swrc
def update_config(self):
cfg.read_config()
@ -2035,12 +2044,15 @@ class PatchController(PatchService):
ret["error"] += "Please fix above issues then retry the deploy.\n"
return ret
collect_current_load_for_hosts()
if self._deploy_upgrade_start(to_release):
collect_current_load_for_hosts()
dbapi = db_api.get_instance()
dbapi.create_deploy(SW_VERSION, to_release, True)
dbapi.update_deploy(DEPLOY_STATES.DATA_MIGRATION)
sw_rel = self.release_collection.get_release_by_id(deployment)
if sw_rel is None:
raise InternalError("%s cannot be found" % to_release)
sw_rel.update_state(constants.DEPLOYING)
msg_info = "Deployment for %s started" % deployment
else:
msg_error = "Deployment for %s failed to start" % deployment

9
software/software/software_functions.py
View File

@ -299,9 +299,14 @@ class ReleaseData(object):
:param state: Indicates Applied, Available, or Committed
:return: Release ID
"""
tree = ElementTree.parse(filename)
root = tree.getroot()
with open(filename, "r") as f:
text = f.read()
return self.parse_metadata_string(text, state)
def parse_metadata_string(self, text, state):
root = ElementTree.fromstring(text)
#
# <patch>
# <id>PATCH_0001</id>

187
software/software/tests/test_software_function.py Normal file
View File

@ -0,0 +1,187 @@
#
# SPDX-License-Identifier: Apache-2.0
#
# Copyright (c) 2024 Wind River Systems, Inc.
#
import unittest
from software.release_data import SWReleaseCollection
from software.software_functions import ReleaseData
metadata = """<?xml version="1.0" ?>
<patch>
<id>23.09_RR_ALL_NODES</id>
<sw_version>23.09</sw_version>
<summary>Debian patch test</summary>
<description>Reboot required patch</description>
<install_instructions>Sample instructions</install_instructions>
<warnings>Sample warning</warnings>
<status>TST</status>
<unremovable>Y</unremovable>
<reboot_required>Y</reboot_required>
<contents>
<ostree>
<number_of_commits>1</number_of_commits>
<base>
<commit>0db647647b009c5cc02410d461de0870049bdeb66caf1bdc1ccd189ac83b8e92</commit>
<checksum>bae3ff59c5f59c95aa8d3ccf8c1364c4c869cd428f7b5032a00a8b777cc132f7</checksum>
</base>
<commit1>
<commit>38453dcb1aeb5bb9394ed02c0e6b8f2f913d00a827c89faf98cb63dff503b8e2</commit>
<checksum>2f742b1b719f19b302c306604659ccf4aa61a1fdb7742ac79b009c79af18c79b</checksum>
</commit1>
</ostree>
</contents>
<requires/>
<semantics/>
</patch>"""
metadata2 = """<?xml version="1.0" ?>
<patch>
<id>23.09_NRR_INSVC</id>
<sw_version>23.09</sw_version>
<summary>Debian patch test</summary>
<description>In service patch</description>
<install_instructions>Sample instructions2</install_instructions>
<warnings>Sample warning2</warnings>
<status>DEV</status>
<unremovable>N</unremovable>
<reboot_required>N</reboot_required>
<restart_script>23.09_NRR_INSVC_example-cgcs-patch-restart</restart_script>
<contents>
<ostree>
<number_of_commits>1</number_of_commits>
<base>
<commit>0db647647b009c5cc02410d461de0870049bdeb66caf1bdc1ccd189ac83b8e92</commit>
<checksum>bae3ff59c5f59c95aa8d3ccf8c1364c4c869cd428f7b5032a00a8b777cc132f7</checksum>
</base>
<commit1>
<commit>0b53576092a189133d56eac49ae858c1218f480a4a859eaca2b47f2604a4e0e7</commit>
<checksum>2f742b1b719f19b302c306604659ccf4aa61a1fdb7742ac79b009c79af18c79b</checksum>
</commit1>
</ostree>
</contents>
<requires/>
<semantics/>
</patch>"""
expected_values = [
{
"release_id": "23.09_NRR_INSVC",
"version": "23.09",
"state": "deployed",
"summary": "Debian patch test",
"description": "In service patch",
"install_instructions": "Sample instructions2",
"warnings": "Sample warning2",
"status": "DEV",
"unremovable": "N",
"restart_script": "23.09_NRR_INSVC_example-cgcs-patch-restart",
"commit_id": "0b53576092a189133d56eac49ae858c1218f480a4a859eaca2b47f2604a4e0e7",
"checksum": "2f742b1b719f19b302c306604659ccf4aa61a1fdb7742ac79b009c79af18c79b",
},
{
"release_id": "23.09_RR_ALL_NODES",
"version": "23.09",
"state": "available",
"summary": "Debian patch test",
"description": "Reboot required patch",
"install_instructions": "Sample instructions",
"warnings": "Sample warning",
"status": "TST",
"unremovable": "Y",
"restart_script": None,
"commit_id": "38453dcb1aeb5bb9394ed02c0e6b8f2f913d00a827c89faf98cb63dff503b8e2",
"checksum": "2f742b1b719f19b302c306604659ccf4aa61a1fdb7742ac79b009c79af18c79b",
}
]
package_dir = {"23.09": "/var/www/page/feed/rel_23.09"}
class TestSoftwareFunction(unittest.TestCase):
def setUp(self):
pass
def tearDown(self):
pass
@property
def release_collection(self):
rd = ReleaseData()
rd.parse_metadata_string(metadata, "available")
rd2 = ReleaseData()
rd2.parse_metadata_string(metadata2, "deployed")
rd.add_release(rd2)
rc = SWReleaseCollection(rd)
return rc
def test_SWReleaseCollection_iterate_releases(self):
idx = 0
for r in self.release_collection.iterate_releases():
val = expected_values[idx]
idx += 1
self.assertEqual(val["release_id"], r.id)
self.assertEqual(val["version"], r.sw_version)
self.assertEqual(val["state"], r.state)
self.assertEqual(val["summary"], r.summary)
self.assertEqual(val["description"], r.description)
self.assertEqual(val["install_instructions"], r.install_instructions)
self.assertEqual(val["warnings"], r.warnings)
self.assertEqual(val["status"], r.status)
self.assertEqual(val["unremovable"], r.unremovable)
if val["restart_script"] is None:
self.assertIsNone(r.restart_script)
else:
self.assertEqual(val["restart_script"], r.restart_script)
self.assertEqual(val["commit_id"], r.commit_id)
self.assertEqual(val["checksum"], r.commit_checksum)
def test_SWReleaseCollection_get_release_by_id(self):
rd = ReleaseData()
rd.parse_metadata_string(metadata, "available")
rd2 = ReleaseData()
rd2.parse_metadata_string(metadata2, "deployed")
rd.add_release(rd2)
rc = SWReleaseCollection(rd)
idx = 0
rid = expected_values[idx]["release_id"]
r = rc.get_release_by_id(rid)
val = expected_values[idx]
self.assertEqual(val["release_id"], r.id)
self.assertEqual(val["version"], r.sw_version)
self.assertEqual(val["state"], r.state)
self.assertEqual(val["summary"], r.summary)
self.assertEqual(val["description"], r.description)
self.assertEqual(val["install_instructions"], r.install_instructions)
self.assertEqual(val["warnings"], r.warnings)
self.assertEqual(val["status"], r.status)
self.assertEqual(val["unremovable"], r.unremovable)
if val["restart_script"] is None:
self.assertIsNone(r.restart_script)
else:
self.assertEqual(val["restart_script"], r.restart_script)
self.assertEqual(val["commit_id"], r.commit_id)
self.assertEqual(val["checksum"], r.commit_checksum)
def test_SWReleaseCollection_iterate_release_by_state(self):
val = expected_values[0]
for r in self.release_collection.iterate_releases_by_state('deployed'):
self.assertEqual(val["release_id"], r.id)
self.assertEqual(val["version"], r.sw_version)
self.assertEqual(val["state"], r.state)
self.assertEqual(val["summary"], r.summary)
self.assertEqual(val["description"], r.description)
self.assertEqual(val["install_instructions"], r.install_instructions)
self.assertEqual(val["warnings"], r.warnings)
self.assertEqual(val["status"], r.status)
self.assertEqual(val["unremovable"], r.unremovable)
if val["restart_script"] is None:
self.assertIsNone(r.restart_script)
else:
self.assertEqual(val["restart_script"], r.restart_script)
self.assertEqual(val["commit_id"], r.commit_id)
self.assertEqual(val["checksum"], r.commit_checksum)