Software deploy host validation

This commit validates pre-conditions when software
deploy-host <hostname> is issued.

The pre-conditions are:
The host state is in pending state.
The specified host target is locked and online.
Nodes deployed to major release in the order below in DX system:
Controller-1 -> Controller-0 -> Storage nodes -> Compute nodes.
Nodes deployed to patch release in the order below in DX system:
Controllers -> Storages nodes -> Compute nodes.

Test Plan:

PASS: Deploy host to controller-1 validation.
PASS: Validated target locked and online checker.
PASS: Validated start-done deploy state checker.

Story: 2010676
Task: 49795

Change-Id: I8fe8faa85c594472bb6c8c021416205bf4a61fbb
Signed-off-by: Luis Eduardo Bonatti <LuizEduardo.Bonatti@windriver.com>

software/software/constants.py

@@ -19,6 +19,8 @@ from tsconfig.tsconfig import SW_VERSION
 ADDRESS_VERSION_IPV4 = 4
 ADDRESS_VERSION_IPV6 = 6
 CONTROLLER_FLOATING_HOSTNAME = "controller"
+CONTROLLER_0_HOSTNAME = '%s-0' % CONTROLLER_FLOATING_HOSTNAME
+CONTROLLER_1_HOSTNAME = '%s-1' % CONTROLLER_FLOATING_HOSTNAME
 
 DISTRIBUTED_CLOUD_ROLE_SYSTEMCONTROLLER = 'systemcontroller'
 SYSTEM_CONTROLLER_REGION = 'SystemController'
@@ -74,6 +76,7 @@ PATCH_EXTENSION = ".patch"
 SUPPORTED_UPLOAD_FILE_EXT = [ISO_EXTENSION, SIG_EXTENSION, PATCH_EXTENSION]
 SCRATCH_DIR = "/scratch"
 RELEASE_GA_NAME = "starlingx-%s"
+MAJOR_RELEASE = "%s.0"
 
 # Precheck constants
 LICENSE_FILE = "/etc/platform/.license"
@@ -94,3 +97,11 @@ LAST_IN_SYNC = "last_in_sync"
 
 SYSTEM_MODE_SIMPLEX = "simplex"
 SYSTEM_MODE_DUPLEX = "duplex"
+
+# Personalities
+CONTROLLER = 'controller'
+STORAGE = 'storage'
+WORKER = 'worker'
+
+AVAILABILITY_ONLINE = 'online'
+ADMIN_LOCKED = 'locked'

software/software/software_controller.py

@@ -55,6 +55,7 @@ from software.release_data import reload_release_data
 from software.release_data import get_SWReleaseCollection
 from software.software_functions import collect_current_load_for_hosts
 from software.software_functions import create_deploy_hosts
+from software.software_functions import deploy_host_validations
 from software.software_functions import parse_release_metadata
 from software.software_functions import configure_logging
 from software.software_functions import mount_iso_load
@@ -757,7 +758,7 @@ class SWMessageDeployStateChanged(messages.PatchMessage):
                      (self.deploy_state, self.agent))
             sc.deploy_state_changed(self.deploy_state)
         else:
-            LOG.info("Received %s deploy state changed to %s, agent %s" %
+            LOG.info("Received %s deploy host state changed to %s, agent %s" %
                      (self.hostname, self.host_state, self.agent))
             sc.host_deploy_state_changed(self.hostname, self.host_state)
 
@@ -2715,6 +2716,7 @@ class PatchController(PatchService):
         if deploy_host is None:
             raise HostNotFound(hostname)
 
+        deploy_host_validations(hostname)
         deploy_state = DeployState.get_instance()
         deploy_host_state = DeployHostState(hostname)
         deploy_state.deploy_host()

software/software/software_entities.py

@@ -210,7 +210,7 @@ class DeployHosts(ABC):
         pass
 
     @abstractmethod
-    def update(self, hostname: str, state: str):
+    def update(self, hostname: str, state: DEPLOY_HOST_STATES):
         """
         Update a deploy-host entry
 

software/software/software_functions.py

@@ -39,6 +39,8 @@ import software.constants as constants
 from software import states
 import software.utils as utils
 from software.sysinv_utils import get_ihost_list
+from software.sysinv_utils import get_system_info
+from software.sysinv_utils import is_host_locked_and_online
 
 
 try:
@@ -1215,14 +1217,17 @@ def create_deploy_hosts():
     Create deploy-hosts entities based on hostnames
     from sysinv.
     """
+    db_api_instance = get_instance()
+    db_api_instance.begin_update()
     try:
-        db_api_instance = get_instance()
         for ihost in get_ihost_list():
             db_api_instance.create_deploy_host(ihost.hostname)
         LOG.info("Deploy-hosts entities created successfully.")
     except Exception as err:
         LOG.exception("Error in deploy-hosts entities creation")
         raise err
+    finally:
+        db_api_instance.end_update()
 
 
 def collect_current_load_for_hosts():
@@ -1330,3 +1335,86 @@ def set_host_target_load(hostname, major_release):
         LOG.exception("Error setting target_load to %s for %s: %s" % (
             major_release, hostname, str(e)))
         raise
+
+
+def validate_host_state_to_deploy_host(hostname):
+    """
+    Check if the deployment host state for the hostname is pending.
+
+    If the validation fails raise SoftwareServiceError exception.
+
+    :param hostname: Hostname of the host to be deployed
+    """
+
+    host_state = get_instance().get_deploy_host_by_hostname(hostname).get("state")
+    if host_state != states.DEPLOY_HOST_STATES.PENDING.value:
+        msg = (f"Host state is {host_state} and should be "
+               f"{states.DEPLOY_HOST_STATES.PENDING.value}")
+        raise SoftwareServiceError(msg)
+
+def deploy_host_validations(hostname):
+    """
+    Check the conditions below:
+    Host state is pending.
+    If system mode is duplex, check if provided hostname satisfy the right deployment order.
+    Host is locked and online.
+
+    If one of the validations fail, raise SoftwareServiceError exception, except if system
+    is a simplex.
+
+    :param hostname: Hostname of the host to be deployed
+    """
+    validate_host_state_to_deploy_host(hostname)
+    _, system_mode = get_system_info()
+    simplex = (system_mode == constants.SYSTEM_MODE_SIMPLEX)
+    if simplex:
+        LOG.info("System mode is simplex. Skipping deploy order validation...")
+    else:
+        validate_host_deploy_order(hostname)
+    if not is_host_locked_and_online(hostname):
+        msg = f"Host {hostname} must be {constants.ADMIN_LOCKED}."
+        raise SoftwareServiceError(msg)
+
+
+def validate_host_deploy_order(hostname):
+    """
+    Check if the host to be deployed satisfy the major release deployment right
+    order of controller-1 -> controller-0 -> storages -> computes
+    and for patch release: controllers -> storages -> computes
+
+    Case one of the validations failed raise SoftwareError exception
+
+    :param hostname: Hostname of the host to be deployed.
+    """
+    db_api_instance = get_instance()
+    controllers_list = [constants.CONTROLLER_1_HOSTNAME, constants.CONTROLLER_0_HOSTNAME]
+    storage_list = []
+    workers_list = []
+    is_patch_release = False
+    deploy = db_api_instance.get_deploy_all()[0]
+    to_release = deploy.get("from_release")
+    if to_release != (constants.MAJOR_RELEASE % utils.get_major_release_version(to_release)):
+        is_patch_release = True
+    for host in get_ihost_list():
+        if host.personality == constants.STORAGE:
+            storage_list.append(host.hostname)
+        if host.personality == constants.WORKER:
+            workers_list.append(host.hostname)
+
+    ordered_storage_list = sorted(storage_list, key=lambda x: int(x.split("-")[1]))
+    ordered_list = controllers_list + ordered_storage_list + workers_list
+
+    for host in db_api_instance.get_deploy_host():
+        if host.get("state") == states.DEPLOY_HOST_STATES.DEPLOYED.value:
+            ordered_list.remove(host.get("hostname"))
+    if not ordered_list:
+        raise SoftwareServiceError("All hosts are already in deployed state.")
+    # If there is only workers nodes there is no order to deploy
+    if hostname == ordered_list[0] or (ordered_list[0] in workers_list and hostname in workers_list):
+        return
+    # If deployment is a patch release bypass the controllers order
+    elif is_patch_release and ordered_list[0] in controllers_list and hostname in controllers_list:
+        return
+    else:
+        raise SoftwareServiceError(f"{hostname.capitalize()} do not satisfy the right order of deployment "
+                                   f"should be {ordered_list[0]}")

software/software/sysinv_utils.py

@@ -54,6 +54,22 @@ def get_ihost_list():
         raise
 
 
+def is_host_locked_and_online(host):
+    for ihost in get_ihost_list():
+        if (host == ihost.hostname and ihost.availability == constants.AVAILABILITY_ONLINE and
+                ihost.administrative == constants.ADMIN_LOCKED):
+            return True
+    return False
+
+
+def get_system_info():
+    """Returns system type and system mode"""
+    token, endpoint = utils.get_endpoints_token()
+    sysinv_client = get_sysinv_client(token=token, endpoint=endpoint)
+    system_info = sysinv_client.isystem.list()[0]
+    return system_info.system_type, system_info.system_mode
+
+
 def get_dc_role():
     try:
         token, endpoint = utils.get_endpoints_token()

software/software/tests/test_software_function.py

@@ -4,8 +4,14 @@
 # Copyright (c) 2024 Wind River Systems, Inc.
 #
 import unittest
+from unittest.mock import MagicMock
+from unittest.mock import patch
+
+from software import states
+from software.exceptions import SoftwareServiceError
 from software.release_data import SWReleaseCollection
 from software.software_functions import ReleaseData
+from software.software_functions import validate_host_state_to_deploy_host
 
 metadata = """<?xml version="1.0" ?>
 <patch>
@@ -185,3 +191,17 @@ class TestSoftwareFunction(unittest.TestCase):
                 self.assertEqual(val["restart_script"], r.restart_script)
             self.assertEqual(val["commit_id"], r.commit_id)
             self.assertEqual(val["checksum"], r.commit_checksum)
+
+
+    @patch('software.db.api.SoftwareAPI')
+    def test_validate_host_state_to_deploy_host_raises_exception_if_deploy_host_state_is_wrong(self, software_api_mock):
+        # Arrange
+        deploy_host_state = states.DEPLOY_HOST_STATES.DEPLOYED.value
+        deploy_by_hostname = MagicMock(return_value={"state": deploy_host_state})
+        software_api_mock.return_value = MagicMock(get_deploy_host_by_hostname=deploy_by_hostname)
+        with self.assertRaises(SoftwareServiceError) as error:
+        # Actions
+            validate_host_state_to_deploy_host(hostname="abc")
+        # Assertions
+        error_msg = "Host state is deployed and should be pending"
+        self.assertEqual(str(error.exception), error_msg)

software/software/utilities/update_deploy_state.py

@@ -53,7 +53,7 @@ def update_deploy_state(server_addr, server_port, agent, deploy_state=None, host
         "agent": agent,
         "deploy-state": deploy_state,
         "hostname": host,
-        "host_state": host_state
+        "host-state": host_state
     }
 
     msg_txt = json.dumps(msg)