sw-deploy precheck and start API

sw-deploy precheck and start synchronous
APIs implementation.

Test Plan:
PASSED: On a DX system, when active alarm is present
perform precheck on an in-service patch, precheck fails.
PASSED: On a DX system, there is no active alarm and
system is in healthy state, perform precheck on an
in-service patch, precheck is successful.
PASSED: On a DX system, there is no active alarm and
system is in healthy state, perform precheck on in-service
patch, precheck is successful.
PASSED: On a DX system, try to perform precheck on
an in-service patch when there is another minor release
which is in 'deploy-start' state, precheck fails.
PASSED: On a DX system, perfrom software deploy start on
an in-service patch,with change in release state,
start API is successful.
PASSED: On DX system, perform software deploy start
on an in-service patch, with no change in release state,
start API fails.
to do: On SX system, perform end to end sw-deploy strategy
update using in-service patch.

Story: 2011045
Task: 49911

Change-Id: I265c8a1f9fbadf04275e0af788614d094c23315c
Signed-off-by: Vanathi.Selvaraju <vanathi.selvaraju@windriver.com>

nfv/nfv-common/nfv_common/tasks/_task_future.py

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2015-2016 Wind River Systems, Inc.
+# Copyright (c) 2015-2016,2024 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -57,7 +57,9 @@ class TaskFuture(object):
         if timeout_in_secs is None:
             # WARNING: Any change to the default timeout must be reflected in
             # the timeouts used for any work being done.
-            timeout_in_secs = 20
+            # The timeout to be changed back to 20sec when the start
+            # software-api is async for patch-release
+            timeout_in_secs = 600
 
         elif 0 >= timeout_in_secs:
             timeout_in_secs = None  # No timeout wanted, wait forever

nfv/nfv-common/nfv_common/thread/_thread.py

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2015-2016 Wind River Systems, Inc.
+# Copyright (c) 2015-2016,2024 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -94,7 +94,7 @@ class Thread(object):
                         self._stall_timestamp_ms = \
                             timers.get_monotonic_timestamp_in_ms()
 
-                    DLOG.error("Thread %s stalled, progress_marker=%s, "
+                    DLOG.warn("Thread %s stalled, progress_marker=%s, "
                                "elapsed_secs=%s." % (self._name,
                                                      self._progress_marker.value,
                                                      self.stall_elapsed_secs))

nfv/nfv-plugins/nfv_plugins/nfvi_plugins/config.ini

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2015-2018 Wind River Systems, Inc.
+# Copyright (c) 2015-2018,2024 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -125,3 +125,4 @@ glance.upload_image_data_by_file=180
 glance.upload_image_data_by_url=180
 sysinv=60
 patching.apply_patch=180
+usm=60

nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_infrastructure_api.py

@@ -2297,17 +2297,13 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
 
             future.work(usm.sw_deploy_precheck, self._platform_token, release)
             future.result = (yield)
-
             if not future.result.is_complete():
                 DLOG.error("USM software deploy precheck did not complete.")
                 return
 
-            upgrade_obj = nfvi.objects.v1.Upgrade(
+            precheck_data = future.result.data['error']
-                release,
-                None,
-                None)
 
-            response['result-data'] = upgrade_obj
+            response['result-data'] = precheck_data
             response['completed'] = True
 
         except exceptions.OpenStackRestAPIException as e:
@@ -2328,7 +2324,7 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
             callback.send(response)
             callback.close()
 
-    def upgrade_start(self, future, release, callback):
+    def sw_deploy_start(self, future, release, callback):
         """
         Start a USM software deploy
         """
@@ -2337,9 +2333,7 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
         response['reason'] = ''
 
         try:
-            upgrade_data = future.result.data
             future.set_timeouts(config.CONF.get('nfvi-timeouts', None))
-
             if self._platform_token is None or \
                     self._platform_token.is_expired():
                 future.work(openstack.get_token, self._platform_directory)
@@ -2354,17 +2348,22 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
 
             future.work(usm.sw_deploy_start, self._platform_token, release)
             future.result = (yield)
-
             if not future.result.is_complete():
                 DLOG.error("USM software deploy start did not complete.")
                 return
 
-            upgrade_obj = nfvi.objects.v1.Upgrade(
+            # todo (vselvara): remove the state check here once the api is changed
-                release,
+            # to async. state check to be done in _strategy_steps.py
-                upgrade_data,
+            future.work(usm.sw_deploy_get_release, self._platform_token, release)
-                None)
+            future.result = (yield)
+            if not future.result.is_complete():
+                DLOG.error("USM software deploy get release did not complete.")
+                return
 
-            response['result-data'] = upgrade_obj
+            release_data = future.result.data
+            start_state_data = release_data["metadata"].get(release, None)
+
+            response['result-data'] = start_state_data
             response['completed'] = True
 
         except exceptions.OpenStackRestAPIException as e:

nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/rest_api.py

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2015-2023 Wind River Systems, Inc.
+# Copyright (c) 2015-2024 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -473,7 +473,7 @@ def rest_api_request(token,
                      api_cmd,
                      api_cmd_headers=None,
                      api_cmd_payload=None,
-                     timeout_in_secs=20,
+                     timeout_in_secs=600,
                      file_to_post=None):
     """
     Make a rest-api request using the given token

nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/usm.py

@@ -56,7 +56,7 @@ def _api_post(token, url, payload, headers=None):
                                 url,
                                 headers,
                                 json.dumps(payload),
-                                timeout_in_secs=REST_API_REQUEST_TIMEOUT)
+                                timeout_in_secs=600)
     return response
 
 
@@ -88,7 +88,8 @@ def sw_deploy_precheck(token, release, force=False):
     Ask USM to precheck before a deployment
     """
 
-    uri = f"deploy_precheck/{release}/force" if force else f"deploy_precheck/{release}"
+    uri = (f"deploy_precheck/{release}/force?region_name=RegionOne" if
+            force else f"deploy_precheck/{release}?region_name=RegionOne")
     url = _usm_api_cmd(token, uri)
     response = _api_post(token, url, {})
     return response

nfv/nfv-tests/nfv_unit_tests/tests/test_sw_deploy_strategy.py

@@ -205,11 +205,9 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
             'total_stages': 1,
             'stages': [
                 {'name': 'sw-upgrade-start',
-                 'total_steps': 4,
+                 'total_steps': 3,
                  'steps': [
                      {'name': 'query-alarms'},
-                     {'name': 'sw-deploy-precheck',
-                      'release': release},
                      {'name': 'start-upgrade',
                       'release': release},
                      {'name': 'system-stabilize',
@@ -253,11 +251,9 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
             'total_stages': 1,
             'stages': [
                 {'name': 'sw-upgrade-start',
-                 'total_steps': 4,
+                 'total_steps': 3,
                  'steps': [
                      {'name': 'query-alarms'},
-                     {'name': 'sw-deploy-precheck',
-                      'release': release},
                      {'name': 'start-upgrade',
                       'release': release},
                      {'name': 'system-stabilize',
@@ -302,13 +298,11 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
             'total_stages': 1,
             'stages': [
                 {'name': 'sw-upgrade-start',
-                 'total_steps': 6,
+                 'total_steps': 5,
                  'steps': [
                      {'name': 'query-alarms'},
                      {'name': 'swact-hosts',
                       'entity_names': ['controller-1']},
-                     {'name': 'sw-deploy-precheck',
-                      'release': release},
                      {'name': 'start-upgrade',
                       'release': release},
                      {'name': 'system-stabilize',
@@ -994,10 +988,9 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
             'stages': [
                 {
                     'name': 'sw-upgrade-start',
-                    'total_steps': 4,
+                    'total_steps': 3,
                     'steps': [
                         {'name': 'query-alarms'},
-                        {'name': 'sw-deploy-precheck', 'release': release},
                         {'name': 'start-upgrade', 'release': release},
                         {'name': 'system-stabilize', 'timeout': 60},
                     ],
@@ -1060,10 +1053,9 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
             'stages': [
                 {
                     'name': 'sw-upgrade-start',
-                    'total_steps': 4,
+                    'total_steps': 3,
                     'steps': [
                         {'name': 'query-alarms'},
-                        {'name': 'sw-deploy-precheck', 'release': release},
                         {'name': 'start-upgrade', 'release': release},
                         {'name': 'system-stabilize', 'timeout': 60},
                     ],
@@ -1130,10 +1122,9 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
             'stages': [
                 {
                     'name': 'sw-upgrade-start',
-                    'total_steps': 4,
+                    'total_steps': 3,
                     'steps': [
                         {'name': 'query-alarms'},
-                        {'name': 'sw-deploy-precheck', 'release': release},
                         {'name': 'start-upgrade', 'release': release},
                         {'name': 'system-stabilize', 'timeout': 60},
                     ],
@@ -1205,10 +1196,9 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
             'stages': [
                 {
                     'name': 'sw-upgrade-start',
-                    'total_steps': 4,
+                    'total_steps': 3,
                     'steps': [
                         {'name': 'query-alarms'},
-                        {'name': 'sw-deploy-precheck', 'release': release},
                         {'name': 'start-upgrade', 'release': release},
                         {'name': 'system-stabilize', 'timeout': 60},
                     ],
@@ -1292,10 +1282,9 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
             'stages': [
                 {
                     'name': 'sw-upgrade-start',
-                    'total_steps': 4,
+                    'total_steps': 3,
                     'steps': [
                         {'name': 'query-alarms'},
-                        {'name': 'sw-deploy-precheck', 'release': release},
                         {'name': 'start-upgrade', 'release': release},
                         {'name': 'system-stabilize', 'timeout': 60},
                     ],
@@ -1389,10 +1378,9 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
             'stages': [
                 {
                     'name': 'sw-upgrade-start',
-                    'total_steps': 4,
+                    'total_steps': 3,
                     'steps': [
                         {'name': 'query-alarms'},
-                        {'name': 'sw-deploy-precheck', 'release': release},
                         {'name': 'start-upgrade', 'release': release},
                         {'name': 'system-stabilize', 'timeout': 60},
                     ],

nfv/nfv-tests/nfv_unit_tests/tests/test_sw_upgrade_strategy.py

@@ -1400,11 +1400,9 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
             'total_stages': 8,
             'stages': [
                 {'name': 'sw-upgrade-start',
-                 'total_steps': 4,
+                 'total_steps': 3,
                  'steps': [
                      {'name': 'query-alarms'},
-                     {'name': 'sw-deploy-precheck',
-                      'release': strategy.nfvi_upgrade.release},
                      {'name': 'start-upgrade',
                       'release': strategy.nfvi_upgrade.release},
                      {'name': 'system-stabilize',
@@ -1570,13 +1568,11 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
             'total_stages': 6,
             'stages': [
                 {'name': 'sw-upgrade-start',
-                 'total_steps': 6,
+                 'total_steps': 5,
                  'steps': [
                      {'name': 'query-alarms'},
                      {'name': 'swact-hosts',
                       'entity_names': ['controller-1']},
-                     {'name': 'sw-deploy-precheck',
-                      'release': strategy.nfvi_upgrade.release},
                      {'name': 'start-upgrade',
                       'release': strategy.nfvi_upgrade.release},
                      {'name': 'system-stabilize',

nfv/nfv-vim/nfv_vim/strategy/_strategy.py

@@ -10,6 +10,7 @@ import weakref
 from nfv_common import debug
 from nfv_common.helpers import Constant
 from nfv_common.helpers import Constants
+from nfv_common.helpers import coroutine
 from nfv_common.helpers import get_local_host_name
 from nfv_common.helpers import Singleton
 from nfv_common import strategy
@@ -1804,6 +1805,48 @@ class SwUpgradeStrategy(
         """
         self._nfvi_upgrade = nfvi_upgrade
 
+    @coroutine
+    def _sw_deploy_precheck_callback(self):
+        """
+        Software deploy precheck callback
+        """
+        response = (yield)
+        DLOG.debug("sw-deploy precheck callback response=%s." % response)
+
+        if response['completed']:
+            if not response['result-data']:
+                DLOG.debug("sw-deploy precheck completed %s" % response['result-data'])
+            else:
+                reason = "sw-deploy precheck failed"
+                # todo: (vselvara) to display the entire erorr response
+                # reason = response['result-data']
+                DLOG.warn(reason)
+                self._state = strategy.STRATEGY_STATE.BUILD_FAILED
+                self.build_phase.result = strategy.STRATEGY_PHASE_RESULT.FAILED
+                self.build_phase.result_reason = reason
+                self.sw_update_obj.strategy_build_complete(
+                        False, self.build_phase.result_reason)
+                self.save()
+        else:
+            reason = "sw-deploy precheck operation not completed"
+            DLOG.warn(reason)
+            self._state = strategy.STRATEGY_STATE.BUILD_FAILED
+            self.build_phase.result = strategy.STRATEGY_PHASE_RESULT.FAILED
+            self.build_phase.result_reason = reason
+            self.sw_update_obj.strategy_build_complete(
+                    False, self.build_phase.result_reason)
+            self.save()
+        return
+
+    def swdeploy_precheck(self, release):
+        """
+        Software deploy precheck
+        """
+        from nfv_vim import nfvi
+
+        DLOG.info("Software deploy precheck for %s." % release)
+        nfvi._nfvi_infrastructure_module.nfvi_sw_deploy_precheck(release, self._sw_deploy_precheck_callback())
+
     def build(self):
         """
         Build the strategy
@@ -1812,6 +1855,7 @@ class SwUpgradeStrategy(
 
         stage = strategy.StrategyStage(strategy.STRATEGY_STAGE_NAME.SW_UPGRADE_QUERY)
         stage.add_step(strategy.QueryAlarmsStep(ignore_alarms=self._ignore_alarms))
+        self.swdeploy_precheck(release=self._release)
         stage.add_step(strategy.QueryUpgradeStep(release=self._release))
         self.build_phase.add_stage(stage)
 
@@ -1852,7 +1896,6 @@ class SwUpgradeStrategy(
         if self.nfvi_upgrade.is_available:
             # sw-deploy start must be done on controller-0
             self._swact_fix(stage, HOST_NAME.CONTROLLER_1)
-            stage.add_step(strategy.SwDeployPrecheckStep(release=self._release))
             stage.add_step(strategy.UpgradeStartStep(release=self._release))
             stage.add_step(strategy.SystemStabilizeStep())
         # sw-deploy host must first be on controller-1

nfv/nfv-vim/nfv_vim/strategy/_strategy_steps.py

@@ -24,6 +24,11 @@ KUBE_CERT_UPDATE_TRUSTBOTHCAS = "trust-both-cas"
 KUBE_CERT_UPDATE_TRUSTNEWCA = "trust-new-ca"
 KUBE_CERT_UPDATE_UPDATECERTS = "update-certs"
 
+# sw-deploy strategy constants
+# todo: (vselvara) based on the new modified states
+SW_DEPLOY_START = 'deploying-start'
+SW_DEPLOY_REMOVE = 'removing'
+
 
 @six.add_metaclass(Singleton)
 class StrategyStepNames(Constants):
@@ -1107,14 +1112,20 @@ class UpgradeStartStep(strategy.StrategyStep):
         """
         response = (yield)
         DLOG.debug("Start-Upgrade callback response=%s." % response)
-
         if response['completed']:
             # TODO(jkraitbe): Consider updating self.strategy.nfvi_upgrade.hosts_info
-            result = strategy.STRATEGY_STEP_RESULT.SUCCESS
+            if (response['result-data']['state'] == SW_DEPLOY_START or
-            self.stage.step_complete(result, "")
+                    response['result-data']['state'] == SW_DEPLOY_REMOVE):
+                result = strategy.STRATEGY_STEP_RESULT.SUCCESS
+                self.stage.step_complete(result, "")
+            else:
+                reason = "Software deploy not started %s" % response['result-data']['state']
+                result = strategy.STRATEGY_STEP_RESULT.FAILED, reason
+                self.stage.step_complete(result, "")
         else:
             # TODO(jkraitbe): Add error message
-            result = strategy.STRATEGY_STEP_RESULT.FAILED
+            reason = "Software deploy start not completed"
+            result = strategy.STRATEGY_STEP_RESULT.FAILED, reason
             self.stage.step_complete(result, "")
 
     def apply(self):