diff --git a/software/software/constants.py b/software/software/constants.py index 8d9f953e..d85f6ca8 100644 --- a/software/software/constants.py +++ b/software/software/constants.py @@ -19,6 +19,8 @@ from tsconfig.tsconfig import SW_VERSION ADDRESS_VERSION_IPV4 = 4 ADDRESS_VERSION_IPV6 = 6 CONTROLLER_FLOATING_HOSTNAME = "controller" +CONTROLLER_0_HOSTNAME = '%s-0' % CONTROLLER_FLOATING_HOSTNAME +CONTROLLER_1_HOSTNAME = '%s-1' % CONTROLLER_FLOATING_HOSTNAME DISTRIBUTED_CLOUD_ROLE_SYSTEMCONTROLLER = 'systemcontroller' SYSTEM_CONTROLLER_REGION = 'SystemController' @@ -74,6 +76,7 @@ PATCH_EXTENSION = ".patch" SUPPORTED_UPLOAD_FILE_EXT = [ISO_EXTENSION, SIG_EXTENSION, PATCH_EXTENSION] SCRATCH_DIR = "/scratch" RELEASE_GA_NAME = "starlingx-%s" +MAJOR_RELEASE = "%s.0" # Precheck constants LICENSE_FILE = "/etc/platform/.license" @@ -94,3 +97,11 @@ LAST_IN_SYNC = "last_in_sync" SYSTEM_MODE_SIMPLEX = "simplex" SYSTEM_MODE_DUPLEX = "duplex" + +# Personalities +CONTROLLER = 'controller' +STORAGE = 'storage' +WORKER = 'worker' + +AVAILABILITY_ONLINE = 'online' +ADMIN_LOCKED = 'locked' diff --git a/software/software/software_controller.py b/software/software/software_controller.py index 66975fbf..e96c9f86 100644 --- a/software/software/software_controller.py +++ b/software/software/software_controller.py @@ -55,6 +55,7 @@ from software.release_data import reload_release_data from software.release_data import get_SWReleaseCollection from software.software_functions import collect_current_load_for_hosts from software.software_functions import create_deploy_hosts +from software.software_functions import deploy_host_validations from software.software_functions import parse_release_metadata from software.software_functions import configure_logging from software.software_functions import mount_iso_load @@ -757,7 +758,7 @@ class SWMessageDeployStateChanged(messages.PatchMessage): (self.deploy_state, self.agent)) sc.deploy_state_changed(self.deploy_state) else: - LOG.info("Received %s deploy state changed to %s, agent %s" % + LOG.info("Received %s deploy host state changed to %s, agent %s" % (self.hostname, self.host_state, self.agent)) sc.host_deploy_state_changed(self.hostname, self.host_state) @@ -2715,6 +2716,7 @@ class PatchController(PatchService): if deploy_host is None: raise HostNotFound(hostname) + deploy_host_validations(hostname) deploy_state = DeployState.get_instance() deploy_host_state = DeployHostState(hostname) deploy_state.deploy_host() diff --git a/software/software/software_entities.py b/software/software/software_entities.py index cba8b8b1..974d66bd 100644 --- a/software/software/software_entities.py +++ b/software/software/software_entities.py @@ -210,7 +210,7 @@ class DeployHosts(ABC): pass @abstractmethod - def update(self, hostname: str, state: str): + def update(self, hostname: str, state: DEPLOY_HOST_STATES): """ Update a deploy-host entry diff --git a/software/software/software_functions.py b/software/software/software_functions.py index 350c147e..82a960e2 100644 --- a/software/software/software_functions.py +++ b/software/software/software_functions.py @@ -39,6 +39,8 @@ import software.constants as constants from software import states import software.utils as utils from software.sysinv_utils import get_ihost_list +from software.sysinv_utils import get_system_info +from software.sysinv_utils import is_host_locked_and_online try: @@ -1215,14 +1217,17 @@ def create_deploy_hosts(): Create deploy-hosts entities based on hostnames from sysinv. """ + db_api_instance = get_instance() + db_api_instance.begin_update() try: - db_api_instance = get_instance() for ihost in get_ihost_list(): db_api_instance.create_deploy_host(ihost.hostname) LOG.info("Deploy-hosts entities created successfully.") except Exception as err: LOG.exception("Error in deploy-hosts entities creation") raise err + finally: + db_api_instance.end_update() def collect_current_load_for_hosts(): @@ -1330,3 +1335,86 @@ def set_host_target_load(hostname, major_release): LOG.exception("Error setting target_load to %s for %s: %s" % ( major_release, hostname, str(e))) raise + + +def validate_host_state_to_deploy_host(hostname): + """ + Check if the deployment host state for the hostname is pending. + + If the validation fails raise SoftwareServiceError exception. + + :param hostname: Hostname of the host to be deployed + """ + + host_state = get_instance().get_deploy_host_by_hostname(hostname).get("state") + if host_state != states.DEPLOY_HOST_STATES.PENDING.value: + msg = (f"Host state is {host_state} and should be " + f"{states.DEPLOY_HOST_STATES.PENDING.value}") + raise SoftwareServiceError(msg) + +def deploy_host_validations(hostname): + """ + Check the conditions below: + Host state is pending. + If system mode is duplex, check if provided hostname satisfy the right deployment order. + Host is locked and online. + + If one of the validations fail, raise SoftwareServiceError exception, except if system + is a simplex. + + :param hostname: Hostname of the host to be deployed + """ + validate_host_state_to_deploy_host(hostname) + _, system_mode = get_system_info() + simplex = (system_mode == constants.SYSTEM_MODE_SIMPLEX) + if simplex: + LOG.info("System mode is simplex. Skipping deploy order validation...") + else: + validate_host_deploy_order(hostname) + if not is_host_locked_and_online(hostname): + msg = f"Host {hostname} must be {constants.ADMIN_LOCKED}." + raise SoftwareServiceError(msg) + + +def validate_host_deploy_order(hostname): + """ + Check if the host to be deployed satisfy the major release deployment right + order of controller-1 -> controller-0 -> storages -> computes + and for patch release: controllers -> storages -> computes + + Case one of the validations failed raise SoftwareError exception + + :param hostname: Hostname of the host to be deployed. + """ + db_api_instance = get_instance() + controllers_list = [constants.CONTROLLER_1_HOSTNAME, constants.CONTROLLER_0_HOSTNAME] + storage_list = [] + workers_list = [] + is_patch_release = False + deploy = db_api_instance.get_deploy_all()[0] + to_release = deploy.get("from_release") + if to_release != (constants.MAJOR_RELEASE % utils.get_major_release_version(to_release)): + is_patch_release = True + for host in get_ihost_list(): + if host.personality == constants.STORAGE: + storage_list.append(host.hostname) + if host.personality == constants.WORKER: + workers_list.append(host.hostname) + + ordered_storage_list = sorted(storage_list, key=lambda x: int(x.split("-")[1])) + ordered_list = controllers_list + ordered_storage_list + workers_list + + for host in db_api_instance.get_deploy_host(): + if host.get("state") == states.DEPLOY_HOST_STATES.DEPLOYED.value: + ordered_list.remove(host.get("hostname")) + if not ordered_list: + raise SoftwareServiceError("All hosts are already in deployed state.") + # If there is only workers nodes there is no order to deploy + if hostname == ordered_list[0] or (ordered_list[0] in workers_list and hostname in workers_list): + return + # If deployment is a patch release bypass the controllers order + elif is_patch_release and ordered_list[0] in controllers_list and hostname in controllers_list: + return + else: + raise SoftwareServiceError(f"{hostname.capitalize()} do not satisfy the right order of deployment " + f"should be {ordered_list[0]}") diff --git a/software/software/sysinv_utils.py b/software/software/sysinv_utils.py index fe49dbbd..fa532529 100644 --- a/software/software/sysinv_utils.py +++ b/software/software/sysinv_utils.py @@ -54,6 +54,22 @@ def get_ihost_list(): raise +def is_host_locked_and_online(host): + for ihost in get_ihost_list(): + if (host == ihost.hostname and ihost.availability == constants.AVAILABILITY_ONLINE and + ihost.administrative == constants.ADMIN_LOCKED): + return True + return False + + +def get_system_info(): + """Returns system type and system mode""" + token, endpoint = utils.get_endpoints_token() + sysinv_client = get_sysinv_client(token=token, endpoint=endpoint) + system_info = sysinv_client.isystem.list()[0] + return system_info.system_type, system_info.system_mode + + def get_dc_role(): try: token, endpoint = utils.get_endpoints_token() diff --git a/software/software/tests/test_software_function.py b/software/software/tests/test_software_function.py index a8660f4a..c976a9d2 100644 --- a/software/software/tests/test_software_function.py +++ b/software/software/tests/test_software_function.py @@ -4,8 +4,14 @@ # Copyright (c) 2024 Wind River Systems, Inc. # import unittest +from unittest.mock import MagicMock +from unittest.mock import patch + +from software import states +from software.exceptions import SoftwareServiceError from software.release_data import SWReleaseCollection from software.software_functions import ReleaseData +from software.software_functions import validate_host_state_to_deploy_host metadata = """ @@ -185,3 +191,17 @@ class TestSoftwareFunction(unittest.TestCase): self.assertEqual(val["restart_script"], r.restart_script) self.assertEqual(val["commit_id"], r.commit_id) self.assertEqual(val["checksum"], r.commit_checksum) + + + @patch('software.db.api.SoftwareAPI') + def test_validate_host_state_to_deploy_host_raises_exception_if_deploy_host_state_is_wrong(self, software_api_mock): + # Arrange + deploy_host_state = states.DEPLOY_HOST_STATES.DEPLOYED.value + deploy_by_hostname = MagicMock(return_value={"state": deploy_host_state}) + software_api_mock.return_value = MagicMock(get_deploy_host_by_hostname=deploy_by_hostname) + with self.assertRaises(SoftwareServiceError) as error: + # Actions + validate_host_state_to_deploy_host(hostname="abc") + # Assertions + error_msg = "Host state is deployed and should be pending" + self.assertEqual(str(error.exception), error_msg) diff --git a/software/software/utilities/update_deploy_state.py b/software/software/utilities/update_deploy_state.py index 13328d57..96880410 100644 --- a/software/software/utilities/update_deploy_state.py +++ b/software/software/utilities/update_deploy_state.py @@ -53,7 +53,7 @@ def update_deploy_state(server_addr, server_port, agent, deploy_state=None, host "agent": agent, "deploy-state": deploy_state, "hostname": host, - "host_state": host_state + "host-state": host_state } msg_txt = json.dumps(msg)