diff --git a/nfv/nfv-common/nfv_common/tasks/_task_future.py b/nfv/nfv-common/nfv_common/tasks/_task_future.py index eb45b6c2..e52c0ec2 100755 --- a/nfv/nfv-common/nfv_common/tasks/_task_future.py +++ b/nfv/nfv-common/nfv_common/tasks/_task_future.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2015-2016 Wind River Systems, Inc. +# Copyright (c) 2015-2016,2024 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -57,7 +57,7 @@ class TaskFuture(object): if timeout_in_secs is None: # WARNING: Any change to the default timeout must be reflected in # the timeouts used for any work being done. - timeout_in_secs = 20 + timeout_in_secs = 600 elif 0 >= timeout_in_secs: timeout_in_secs = None # No timeout wanted, wait forever diff --git a/nfv/nfv-common/nfv_common/thread/_thread.py b/nfv/nfv-common/nfv_common/thread/_thread.py index 4462bfa1..f12cac64 100755 --- a/nfv/nfv-common/nfv_common/thread/_thread.py +++ b/nfv/nfv-common/nfv_common/thread/_thread.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2015-2016 Wind River Systems, Inc. +# Copyright (c) 2015-2016,2024 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -94,7 +94,7 @@ class Thread(object): self._stall_timestamp_ms = \ timers.get_monotonic_timestamp_in_ms() - DLOG.error("Thread %s stalled, progress_marker=%s, " + DLOG.warn("Thread %s stalled, progress_marker=%s, " "elapsed_secs=%s." % (self._name, self._progress_marker.value, self.stall_elapsed_secs)) diff --git a/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/config.ini b/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/config.ini index 2c022f9d..dc221f90 100644 --- a/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/config.ini +++ b/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/config.ini @@ -1,5 +1,5 @@ # -# Copyright (c) 2015-2018 Wind River Systems, Inc. +# Copyright (c) 2015-2018,2024 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -125,3 +125,4 @@ glance.upload_image_data_by_file=180 glance.upload_image_data_by_url=180 sysinv=60 patching.apply_patch=180 +usm=60 diff --git a/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_infrastructure_api.py b/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_infrastructure_api.py index f155ebfd..2d704d5a 100755 --- a/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_infrastructure_api.py +++ b/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_infrastructure_api.py @@ -2297,17 +2297,13 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI): future.work(usm.sw_deploy_precheck, self._platform_token, release) future.result = (yield) - if not future.result.is_complete(): DLOG.error("USM software deploy precheck did not complete.") return - upgrade_obj = nfvi.objects.v1.Upgrade( - release, - None, - None) + precheck_data = future.result.data['error'] - response['result-data'] = upgrade_obj + response['result-data'] = precheck_data response['completed'] = True except exceptions.OpenStackRestAPIException as e: @@ -2328,7 +2324,7 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI): callback.send(response) callback.close() - def upgrade_start(self, future, release, callback): + def sw_deploy_start(self, future, release, callback): """ Start a USM software deploy """ @@ -2337,9 +2333,7 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI): response['reason'] = '' try: - upgrade_data = future.result.data future.set_timeouts(config.CONF.get('nfvi-timeouts', None)) - if self._platform_token is None or \ self._platform_token.is_expired(): future.work(openstack.get_token, self._platform_directory) @@ -2354,17 +2348,19 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI): future.work(usm.sw_deploy_start, self._platform_token, release) future.result = (yield) - if not future.result.is_complete(): DLOG.error("USM software deploy start did not complete.") return + future.work(usm.sw_deploy_get_release, self._platform_token, release) + future.result = (yield) + if not future.result.is_complete(): + DLOG.error("USM software deploy get release did not complete.") + return - upgrade_obj = nfvi.objects.v1.Upgrade( - release, - upgrade_data, - None) + release_data = future.result.data + start_state_data = release_data["metadata"].get(release, None) - response['result-data'] = upgrade_obj + response['result-data'] = start_state_data response['completed'] = True except exceptions.OpenStackRestAPIException as e: diff --git a/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/rest_api.py b/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/rest_api.py index a06d2630..79e466d3 100755 --- a/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/rest_api.py +++ b/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/rest_api.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2015-2023 Wind River Systems, Inc. +# Copyright (c) 2015-2024 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -473,7 +473,7 @@ def rest_api_request(token, api_cmd, api_cmd_headers=None, api_cmd_payload=None, - timeout_in_secs=20, + timeout_in_secs=600, file_to_post=None): """ Make a rest-api request using the given token diff --git a/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/usm.py b/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/usm.py index 912e683f..d101eff8 100755 --- a/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/usm.py +++ b/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/usm.py @@ -56,7 +56,7 @@ def _api_post(token, url, payload, headers=None): url, headers, json.dumps(payload), - timeout_in_secs=REST_API_REQUEST_TIMEOUT) + timeout_in_secs=600) return response @@ -88,7 +88,8 @@ def sw_deploy_precheck(token, release, force=False): Ask USM to precheck before a deployment """ - uri = f"deploy_precheck/{release}/force" if force else f"deploy_precheck/{release}" + uri = (f"deploy_precheck/{release}/force?region_name=RegionOne" if + force else f"deploy_precheck/{release}?region_name=RegionOne") url = _usm_api_cmd(token, uri) response = _api_post(token, url, {}) return response diff --git a/nfv/nfv-tests/nfv_unit_tests/tests/test_sw_deploy_strategy.py b/nfv/nfv-tests/nfv_unit_tests/tests/test_sw_deploy_strategy.py index 9cf63869..abfdfd1a 100755 --- a/nfv/nfv-tests/nfv_unit_tests/tests/test_sw_deploy_strategy.py +++ b/nfv/nfv-tests/nfv_unit_tests/tests/test_sw_deploy_strategy.py @@ -205,11 +205,9 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): 'total_stages': 1, 'stages': [ {'name': 'sw-upgrade-start', - 'total_steps': 4, + 'total_steps': 3, 'steps': [ {'name': 'query-alarms'}, - {'name': 'sw-deploy-precheck', - 'release': release}, {'name': 'start-upgrade', 'release': release}, {'name': 'system-stabilize', @@ -253,11 +251,9 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): 'total_stages': 1, 'stages': [ {'name': 'sw-upgrade-start', - 'total_steps': 4, + 'total_steps': 3, 'steps': [ {'name': 'query-alarms'}, - {'name': 'sw-deploy-precheck', - 'release': release}, {'name': 'start-upgrade', 'release': release}, {'name': 'system-stabilize', @@ -302,13 +298,11 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): 'total_stages': 1, 'stages': [ {'name': 'sw-upgrade-start', - 'total_steps': 6, + 'total_steps': 5, 'steps': [ {'name': 'query-alarms'}, {'name': 'swact-hosts', 'entity_names': ['controller-1']}, - {'name': 'sw-deploy-precheck', - 'release': release}, {'name': 'start-upgrade', 'release': release}, {'name': 'system-stabilize', @@ -994,10 +988,9 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): 'stages': [ { 'name': 'sw-upgrade-start', - 'total_steps': 4, + 'total_steps': 3, 'steps': [ {'name': 'query-alarms'}, - {'name': 'sw-deploy-precheck', 'release': release}, {'name': 'start-upgrade', 'release': release}, {'name': 'system-stabilize', 'timeout': 60}, ], @@ -1060,10 +1053,9 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): 'stages': [ { 'name': 'sw-upgrade-start', - 'total_steps': 4, + 'total_steps': 3, 'steps': [ {'name': 'query-alarms'}, - {'name': 'sw-deploy-precheck', 'release': release}, {'name': 'start-upgrade', 'release': release}, {'name': 'system-stabilize', 'timeout': 60}, ], @@ -1130,10 +1122,9 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): 'stages': [ { 'name': 'sw-upgrade-start', - 'total_steps': 4, + 'total_steps': 3, 'steps': [ {'name': 'query-alarms'}, - {'name': 'sw-deploy-precheck', 'release': release}, {'name': 'start-upgrade', 'release': release}, {'name': 'system-stabilize', 'timeout': 60}, ], @@ -1205,10 +1196,9 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): 'stages': [ { 'name': 'sw-upgrade-start', - 'total_steps': 4, + 'total_steps': 3, 'steps': [ {'name': 'query-alarms'}, - {'name': 'sw-deploy-precheck', 'release': release}, {'name': 'start-upgrade', 'release': release}, {'name': 'system-stabilize', 'timeout': 60}, ], @@ -1292,10 +1282,9 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): 'stages': [ { 'name': 'sw-upgrade-start', - 'total_steps': 4, + 'total_steps': 3, 'steps': [ {'name': 'query-alarms'}, - {'name': 'sw-deploy-precheck', 'release': release}, {'name': 'start-upgrade', 'release': release}, {'name': 'system-stabilize', 'timeout': 60}, ], @@ -1389,10 +1378,9 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): 'stages': [ { 'name': 'sw-upgrade-start', - 'total_steps': 4, + 'total_steps': 3, 'steps': [ {'name': 'query-alarms'}, - {'name': 'sw-deploy-precheck', 'release': release}, {'name': 'start-upgrade', 'release': release}, {'name': 'system-stabilize', 'timeout': 60}, ], diff --git a/nfv/nfv-tests/nfv_unit_tests/tests/test_sw_upgrade_strategy.py b/nfv/nfv-tests/nfv_unit_tests/tests/test_sw_upgrade_strategy.py index ed730044..d3e3c434 100755 --- a/nfv/nfv-tests/nfv_unit_tests/tests/test_sw_upgrade_strategy.py +++ b/nfv/nfv-tests/nfv_unit_tests/tests/test_sw_upgrade_strategy.py @@ -1400,11 +1400,9 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): 'total_stages': 8, 'stages': [ {'name': 'sw-upgrade-start', - 'total_steps': 4, + 'total_steps': 3, 'steps': [ {'name': 'query-alarms'}, - {'name': 'sw-deploy-precheck', - 'release': strategy.nfvi_upgrade.release}, {'name': 'start-upgrade', 'release': strategy.nfvi_upgrade.release}, {'name': 'system-stabilize', @@ -1570,13 +1568,11 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): 'total_stages': 6, 'stages': [ {'name': 'sw-upgrade-start', - 'total_steps': 6, + 'total_steps': 5, 'steps': [ {'name': 'query-alarms'}, {'name': 'swact-hosts', 'entity_names': ['controller-1']}, - {'name': 'sw-deploy-precheck', - 'release': strategy.nfvi_upgrade.release}, {'name': 'start-upgrade', 'release': strategy.nfvi_upgrade.release}, {'name': 'system-stabilize', diff --git a/nfv/nfv-vim/nfv_vim/strategy/_strategy.py b/nfv/nfv-vim/nfv_vim/strategy/_strategy.py index ffc56fcc..302b4e03 100755 --- a/nfv/nfv-vim/nfv_vim/strategy/_strategy.py +++ b/nfv/nfv-vim/nfv_vim/strategy/_strategy.py @@ -10,6 +10,7 @@ import weakref from nfv_common import debug from nfv_common.helpers import Constant from nfv_common.helpers import Constants +from nfv_common.helpers import coroutine from nfv_common.helpers import get_local_host_name from nfv_common.helpers import Singleton from nfv_common import strategy @@ -1804,6 +1805,48 @@ class SwUpgradeStrategy( """ self._nfvi_upgrade = nfvi_upgrade + @coroutine + def _sw_deploy_precheck_callback(self): + """ + Software deploy precheck callback + """ + response = (yield) + DLOG.debug("sw-deploy precheck callback response=%s." % response) + + if response['completed']: + if not response['result-data']: + DLOG.debug("sw-deploy precheck completed %s" % response['result-data']) + else: + reason = "sw-deploy precheck failed" + # todo: (vselvara) to display the entire erorr response + # reason = response['result-data'] + DLOG.warn(reason) + self._state = strategy.STRATEGY_STATE.BUILD_FAILED + self.build_phase.result = strategy.STRATEGY_PHASE_RESULT.FAILED + self.build_phase.result_reason = reason + self.sw_update_obj.strategy_build_complete( + False, self.build_phase.result_reason) + self.save() + else: + reason = "sw-deploy precheck operation not completed" + DLOG.warn(reason) + self._state = strategy.STRATEGY_STATE.BUILD_FAILED + self.build_phase.result = strategy.STRATEGY_PHASE_RESULT.FAILED + self.build_phase.result_reason = reason + self.sw_update_obj.strategy_build_complete( + False, self.build_phase.result_reason) + self.save() + return + + def swdeploy_precheck(self, release): + """ + Software deploy precheck + """ + from nfv_vim import nfvi + + DLOG.info("Software deploy precheck for %s." % release) + nfvi._nfvi_infrastructure_module.nfvi_sw_deploy_precheck(release, self._sw_deploy_precheck_callback()) + def build(self): """ Build the strategy @@ -1812,6 +1855,7 @@ class SwUpgradeStrategy( stage = strategy.StrategyStage(strategy.STRATEGY_STAGE_NAME.SW_UPGRADE_QUERY) stage.add_step(strategy.QueryAlarmsStep(ignore_alarms=self._ignore_alarms)) + self.swdeploy_precheck(release=self._release) stage.add_step(strategy.QueryUpgradeStep(release=self._release)) self.build_phase.add_stage(stage) @@ -1852,7 +1896,6 @@ class SwUpgradeStrategy( if self.nfvi_upgrade.is_available: # sw-deploy start must be done on controller-0 self._swact_fix(stage, HOST_NAME.CONTROLLER_1) - stage.add_step(strategy.SwDeployPrecheckStep(release=self._release)) stage.add_step(strategy.UpgradeStartStep(release=self._release)) stage.add_step(strategy.SystemStabilizeStep()) # sw-deploy host must first be on controller-1 diff --git a/nfv/nfv-vim/nfv_vim/strategy/_strategy_steps.py b/nfv/nfv-vim/nfv_vim/strategy/_strategy_steps.py index a2446bd6..3afe0f2e 100755 --- a/nfv/nfv-vim/nfv_vim/strategy/_strategy_steps.py +++ b/nfv/nfv-vim/nfv_vim/strategy/_strategy_steps.py @@ -24,6 +24,11 @@ KUBE_CERT_UPDATE_TRUSTBOTHCAS = "trust-both-cas" KUBE_CERT_UPDATE_TRUSTNEWCA = "trust-new-ca" KUBE_CERT_UPDATE_UPDATECERTS = "update-certs" +# sw-deploy strategy constants +# todo: (vselvara) based on the new modified states +SW_DEPLOY_START = 'deploying-start' +SW_DEPLOY_REMOVE = 'removing' + @six.add_metaclass(Singleton) class StrategyStepNames(Constants): @@ -1107,14 +1112,20 @@ class UpgradeStartStep(strategy.StrategyStep): """ response = (yield) DLOG.debug("Start-Upgrade callback response=%s." % response) - if response['completed']: # TODO(jkraitbe): Consider updating self.strategy.nfvi_upgrade.hosts_info - result = strategy.STRATEGY_STEP_RESULT.SUCCESS - self.stage.step_complete(result, "") + if (response['result-data']['state'] == SW_DEPLOY_START or + response['result-data']['state'] == SW_DEPLOY_REMOVE): + result = strategy.STRATEGY_STEP_RESULT.SUCCESS + self.stage.step_complete(result, "") + else: + reason = "Software deploy not started %s" % response['result-data']['state'] + result = strategy.STRATEGY_STEP_RESULT.FAILED, reason + self.stage.step_complete(result, "") else: # TODO(jkraitbe): Add error message - result = strategy.STRATEGY_STEP_RESULT.FAILED + reason = "Software deploy start not completed" + result = strategy.STRATEGY_STEP_RESULT.FAILED, reason self.stage.step_complete(result, "") def apply(self):