diff --git a/nfv/nfv-client/nfv_client/openstack/sw_update.py b/nfv/nfv-client/nfv_client/openstack/sw_update.py index 9d3b29c8..c36634e9 100755 --- a/nfv/nfv-client/nfv_client/openstack/sw_update.py +++ b/nfv/nfv-client/nfv_client/openstack/sw_update.py @@ -299,18 +299,11 @@ def create_strategy(token_id, elif sw_update.STRATEGY_NAME_SYSTEM_CONFIG_UPDATE == strategy_name: api_cmd_payload['controller-apply-type'] = controller_apply_type api_cmd_payload['default-instance-action'] = default_instance_action - # TODO(jkraitbe): Backend for sw-deploy will continue as old sw-upgrade for now elif sw_update.STRATEGY_NAME_SW_UPGRADE == strategy_name: # controller-apply-type and default-instance-action passed to strategy api_cmd_payload['controller-apply-type'] = controller_apply_type api_cmd_payload['default-instance-action'] = default_instance_action - # required: 'release' passed to strategy as 'release' api_cmd_payload['release'] = kwargs['release'] - if 'start_upgrade' in kwargs and kwargs['start_upgrade']: - api_cmd_payload['start-upgrade'] = True - if 'complete_upgrade' in kwargs and kwargs['complete_upgrade']: - api_cmd_payload['complete-upgrade'] = True - api_cmd_payload['storage-apply-type'] = storage_apply_type api_cmd_payload['worker-apply-type'] = worker_apply_type if max_parallel_worker_hosts is not None: diff --git a/nfv/nfv-client/nfv_client/shell.py b/nfv/nfv-client/nfv_client/shell.py index 4ec121c2..fa1b5cc5 100755 --- a/nfv/nfv-client/nfv_client/shell.py +++ b/nfv/nfv-client/nfv_client/shell.py @@ -51,12 +51,7 @@ def get_extra_create_args(cmd_area, args): # no additional kwargs for patch return {} elif sw_update.CMD_NAME_SW_DEPLOY == cmd_area: - # TODO(jkraitbe): Args will be updated to use new release parameter - # upgrade supports: complete_upgrade - return { - 'release': args.release, - 'complete_upgrade': args.complete_upgrade - } + return {'release': args.release} elif sw_update.CMD_NAME_FW_UPDATE == cmd_area: # no additional kwargs for firmware update return {} @@ -451,7 +446,7 @@ def setup_sw_deploy_parser(commands): # alarm restrictions, defaults to strict create_strategy_cmd = setup_create_cmd( sub_cmds, - [sw_update.APPLY_TYPE_SERIAL, # hard coded to serial + [sw_update.APPLY_TYPE_SERIAL, sw_update.APPLY_TYPE_IGNORE], [sw_update.APPLY_TYPE_SERIAL, # storage supports serial and parallel sw_update.APPLY_TYPE_PARALLEL, @@ -470,18 +465,10 @@ def setup_sw_deploy_parser(commands): # add sw-deploy specific arguments to the create command # The get_extra_create_args method is updated to align with these - # Disable support for --start-upgrade as it was not completed - # create_strategy_cmd.add_argument('--start-upgrade', - # action='store_true', - # help=argparse.SUPPRESS) # sw-deploy create requires 'release' parameter create_strategy_cmd.add_argument('release', help='software release for deployment') - create_strategy_cmd.add_argument('--complete-upgrade', - action='store_true', - help=argparse.SUPPRESS) - # define the delete command _ = setup_delete_cmd(sub_cmds) # define the apply command diff --git a/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_infrastructure_api.py b/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_infrastructure_api.py index 0c7ac08b..f155ebfd 100755 --- a/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_infrastructure_api.py +++ b/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_infrastructure_api.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2015-2023 Wind River Systems, Inc. +# Copyright (c) 2015-2024 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -23,6 +23,7 @@ from nfv_plugins.nfvi_plugins.openstack import mtc from nfv_plugins.nfvi_plugins.openstack import openstack from nfv_plugins.nfvi_plugins.openstack import rest_api from nfv_plugins.nfvi_plugins.openstack import sysinv +from nfv_plugins.nfvi_plugins.openstack import usm from nfv_plugins.nfvi_plugins.openstack.objects import OPENSTACK_SERVICE @@ -2202,9 +2203,9 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI): callback.send(response) callback.close() - def get_upgrade(self, future, callback): + def get_upgrade(self, future, release, callback): """ - Get information about the upgrade from the plugin + Get information about the software deploy from the plugin """ response = dict() response['completed'] = False @@ -2225,26 +2226,30 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI): self._platform_token = future.result.data - future.work(sysinv.get_upgrade, self._platform_token) + future.work(usm.sw_deploy_get_release, self._platform_token, release) future.result = (yield) if not future.result.is_complete(): - DLOG.error("SysInv get-upgrade did not complete.") + DLOG.error("USM software deploy get release did not complete.") return - upgrade_data_list = future.result.data - if 1 < len(upgrade_data_list): - DLOG.critical("Too many upgrades retrieved, num_upgrades=%i" - % len(upgrade_data_list)) + release_data = future.result.data + release_info = release_data["metadata"].get(release, None) - upgrade_obj = None + future.work(usm.sw_deploy_host_list, self._platform_token) + future.result = (yield) - for upgrade_data in upgrade_data_list['upgrades']: - upgrade_obj = nfvi.objects.v1.Upgrade( - upgrade_data['state'], - upgrade_data['from_release'], - upgrade_data['to_release']) - break + if not future.result.is_complete(): + DLOG.error("USM software deploy host list did not complete.") + return + + hosts_info_data = future.result.data + + upgrade_obj = nfvi.objects.v1.Upgrade( + release, + release_info, + hosts_info_data["data"], + ) response['result-data'] = upgrade_obj response['completed'] = True @@ -2267,9 +2272,9 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI): callback.send(response) callback.close() - def upgrade_start(self, future, callback): + def sw_deploy_precheck(self, future, release, callback): """ - Start an upgrade + Precheck a USM software deploy """ response = dict() response['completed'] = False @@ -2290,18 +2295,74 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI): self._platform_token = future.result.data - future.work(sysinv.upgrade_start, self._platform_token) + future.work(usm.sw_deploy_precheck, self._platform_token, release) future.result = (yield) if not future.result.is_complete(): - DLOG.error("SysInv upgrade-start did not complete.") + DLOG.error("USM software deploy precheck did not complete.") return - upgrade_data = future.result.data upgrade_obj = nfvi.objects.v1.Upgrade( - upgrade_data['state'], - upgrade_data['from_release'], - upgrade_data['to_release']) + release, + None, + None) + + response['result-data'] = upgrade_obj + response['completed'] = True + + except exceptions.OpenStackRestAPIException as e: + if httplib.UNAUTHORIZED == e.http_status_code: + response['error-code'] = nfvi.NFVI_ERROR_CODE.TOKEN_EXPIRED + if self._platform_token is not None: + self._platform_token.set_expired() + + else: + DLOG.exception("Caught exception while trying to precheck " + "USM software deploy, error=%s." % e) + + except Exception as e: + DLOG.exception("Caught exception while trying to precheck USM software deploy, " + "error=%s." % e) + + finally: + callback.send(response) + callback.close() + + def upgrade_start(self, future, release, callback): + """ + Start a USM software deploy + """ + response = dict() + response['completed'] = False + response['reason'] = '' + + try: + upgrade_data = future.result.data + future.set_timeouts(config.CONF.get('nfvi-timeouts', None)) + + if self._platform_token is None or \ + self._platform_token.is_expired(): + future.work(openstack.get_token, self._platform_directory) + future.result = (yield) + + if not future.result.is_complete() or \ + future.result.data is None: + DLOG.error("OpenStack get-token did not complete.") + return + + self._platform_token = future.result.data + + future.work(usm.sw_deploy_start, self._platform_token, release) + future.result = (yield) + + if not future.result.is_complete(): + DLOG.error("USM software deploy start did not complete.") + return + + upgrade_obj = nfvi.objects.v1.Upgrade( + release, + upgrade_data, + None) response['result-data'] = upgrade_obj response['completed'] = True @@ -2314,19 +2375,19 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI): else: DLOG.exception("Caught exception while trying to start " - "upgrade, error=%s." % e) + "USM software deploy, error=%s." % e) except Exception as e: - DLOG.exception("Caught exception while trying to start upgrade, " + DLOG.exception("Caught exception while trying to start USM software deploy, " "error=%s." % e) finally: callback.send(response) callback.close() - def upgrade_activate(self, future, callback): + def upgrade_activate(self, future, release, callback): """ - Activate an upgrade + Activate a USM software deployement """ response = dict() response['completed'] = False @@ -2347,18 +2408,18 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI): self._platform_token = future.result.data - future.work(sysinv.upgrade_activate, self._platform_token) + future.work(usm.sw_deploy_activate, self._platform_token, release) future.result = (yield) if not future.result.is_complete(): - DLOG.error("SysInv upgrade-activate did not complete.") + DLOG.error("USM software deploy activate did not complete.") return upgrade_data = future.result.data upgrade_obj = nfvi.objects.v1.Upgrade( - upgrade_data['state'], - upgrade_data['from_release'], - upgrade_data['to_release']) + release, + upgrade_data, + None) response['result-data'] = upgrade_obj response['completed'] = True @@ -2371,19 +2432,19 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI): else: DLOG.exception("Caught exception while trying to activate " - "upgrade, error=%s." % e) + "USM software deploy, error=%s." % e) except Exception as e: - DLOG.exception("Caught exception while trying to activate upgrade, " + DLOG.exception("Caught exception while trying to activate USM software deploy, " "error=%s." % e) finally: callback.send(response) callback.close() - def upgrade_complete(self, future, callback): + def upgrade_complete(self, future, release, callback): """ - Complete an upgrade + Complete a USM software deployement """ response = dict() response['completed'] = False @@ -2404,18 +2465,18 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI): self._platform_token = future.result.data - future.work(sysinv.upgrade_complete, self._platform_token) + future.work(usm.sw_deploy_complete, self._platform_token, release) future.result = (yield) if not future.result.is_complete(): - DLOG.error("SysInv upgrade-complete did not complete.") + DLOG.error("USM software deploy complete did not complete.") return upgrade_data = future.result.data upgrade_obj = nfvi.objects.v1.Upgrade( - upgrade_data['state'], - upgrade_data['from_release'], - upgrade_data['to_release']) + release, + upgrade_data, + None) response['result-data'] = upgrade_obj response['completed'] = True @@ -2428,10 +2489,10 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI): else: DLOG.exception("Caught exception while trying to complete " - "upgrade, error=%s." % e) + "USM software deploy, error=%s." % e) except Exception as e: - DLOG.exception("Caught exception while trying to complete upgrade, " + DLOG.exception("Caught exception while trying to complete USM software deploy, " "error=%s." % e) finally: @@ -3594,7 +3655,7 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI): self._platform_token = future.result.data - future.work(sysinv.upgrade_host, self._platform_token, host_uuid) + future.work(usm.sw_deploy_execute, self._platform_token, host_name) future.result = (yield) if not future.result.is_complete(): diff --git a/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/objects.py b/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/objects.py index 7397061c..575b943f 100755 --- a/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/objects.py +++ b/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/objects.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2015-2018 Wind River Systems, Inc. +# Copyright (c) 2015-2024 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -40,6 +40,7 @@ class PlatformServices(Constants): SYSINV = Constant('sysinv') PATCHING = Constant('patching') FM = Constant('fm') + USM = Constant('usm') # Platform Services Constant diff --git a/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/usm.py b/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/usm.py new file mode 100755 index 00000000..912e683f --- /dev/null +++ b/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/usm.py @@ -0,0 +1,138 @@ +# +# Copyright (c) 2024 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +import json +import os + +from nfv_common import debug +from nfv_plugins.nfvi_plugins.openstack.objects import PLATFORM_SERVICE +from nfv_plugins.nfvi_plugins.openstack.rest_api import rest_api_request + + +REST_API_REQUEST_TIMEOUT = 60 + +DLOG = debug.debug_get_logger('nfv_plugins.nfvi_plugins.openstack.usm') + + +def _usm_api_cmd(token, endpoint): + base_url = token.get_service_url(PLATFORM_SERVICE.USM) + if base_url is None: + raise ValueError("PlatformService USM URL is invalid") + + url = os.path.join(base_url, "v1/software", endpoint) + return url + + +def _api_cmd_headers(): + api_cmd_headers = dict() + api_cmd_headers['Content-Type'] = "application/json" + api_cmd_headers['User-Agent'] = "vim/1.0" + return api_cmd_headers + + +def _api_get(token, url): + """ + Perform a generic GET for a particular API endpoint + """ + response = rest_api_request(token, + "GET", + url, + timeout_in_secs=REST_API_REQUEST_TIMEOUT) + return response + + +def _api_post(token, url, payload, headers=None): + """ + Generic POST to an endpoint with a payload + """ + if headers is None: + headers = _api_cmd_headers() + + response = rest_api_request(token, + "POST", + url, + headers, + json.dumps(payload), + timeout_in_secs=REST_API_REQUEST_TIMEOUT) + return response + + +def sw_deploy_get_release(token, release): + """ + Query USM for information about a specific upgrade + """ + + uri = f"show/{release}" + url = _usm_api_cmd(token, uri) + response = _api_get(token, url) + return response + + +def sw_deploy_host_list(token): + """ + Query USM for information about a hosts during a deployment + """ + + # TODO(jkraitbe): This API will change in the future + uri = "host_list" + url = _usm_api_cmd(token, uri) + response = _api_get(token, url) + return response + + +def sw_deploy_precheck(token, release, force=False): + """ + Ask USM to precheck before a deployment + """ + + uri = f"deploy_precheck/{release}/force" if force else f"deploy_precheck/{release}" + url = _usm_api_cmd(token, uri) + response = _api_post(token, url, {}) + return response + + +def sw_deploy_start(token, release, force=False): + """ + Ask USM to start a deployment + """ + + uri = f"deploy_start/{release}/force" if force else f"deploy_start/{release}" + url = _usm_api_cmd(token, uri) + response = _api_post(token, url, {}) + return response + + +def sw_deploy_execute(token, host_name): + """ + Ask USM to execute a deployment on a host + """ + + uri = f"deploy_host/{host_name}" + url = _usm_api_cmd(token, uri) + response = _api_post(token, url, {}) + return response + + +def sw_deploy_activate(token, release): + """ + Ask USM activate a deployment + """ + + uri = f"deploy_activate/{release}" + url = _usm_api_cmd(token, uri) + response = _api_post(token, url, {}) + return response + + +def sw_deploy_complete(token, release): + """ + Ask USM complete a deployment + """ + + uri = f"deploy_complete/{release}" + url = _usm_api_cmd(token, uri) + response = _api_post(token, url, {}) + return response diff --git a/nfv/nfv-tests/nfv_unit_tests/tests/sw_update_testcase.py b/nfv/nfv-tests/nfv_unit_tests/tests/sw_update_testcase.py index 90a1c51c..64db9781 100755 --- a/nfv/nfv-tests/nfv_unit_tests/tests/sw_update_testcase.py +++ b/nfv/nfv-tests/nfv_unit_tests/tests/sw_update_testcase.py @@ -1,11 +1,12 @@ # -# Copyright (c) 2020-2023 Wind River Systems, Inc. +# Copyright (c) 2020-2024 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # import fixtures +import json import pprint import uuid @@ -29,6 +30,7 @@ from nfv_unit_tests.tests import utils # unit test comparison between json structures DEBUG_PRINTING = False DEBUG_DEPTH = 3 +DEBUG_WITH_JSON = False def validate_strategy_persists(strategy): @@ -44,9 +46,15 @@ def validate_strategy_persists(strategy): if DEBUG_PRINTING: if strategy.as_dict() != new_strategy.as_dict(): print("==================== Strategy ====================") - pprint.pprint(strategy.as_dict(), depth=DEBUG_DEPTH) + if DEBUG_WITH_JSON: + print(json.dumps(strategy.as_dict(), indent=2)) + else: + pprint.pprint(strategy.as_dict(), depth=DEBUG_DEPTH) print("============== Converted Strategy ================") - pprint.pprint(new_strategy.as_dict(), depth=DEBUG_DEPTH) + if DEBUG_WITH_JSON: + print(json.dumps(new_strategy.as_dict(), indent=2)) + else: + pprint.pprint(new_strategy.as_dict(), depth=DEBUG_DEPTH) assert strategy.as_dict() == new_strategy.as_dict(), \ "Strategy changed when converting to/from dict" @@ -59,9 +67,15 @@ def validate_phase(phase, expected_results): """ if DEBUG_PRINTING: print("====================== Phase Results ========================") - pprint.pprint(phase, depth=DEBUG_DEPTH) + if DEBUG_WITH_JSON: + print(json.dumps(phase, indent=2)) + else: + pprint.pprint(phase, depth=DEBUG_DEPTH) print("===================== Expected Results ======================") - pprint.pprint(expected_results, depth=DEBUG_DEPTH) + if DEBUG_WITH_JSON: + print(json.dumps(expected_results, indent=2)) + else: + pprint.pprint(expected_results, depth=DEBUG_DEPTH) for key in expected_results: if key == 'stages': @@ -75,10 +89,14 @@ def validate_phase(phase, expected_results): apply_step = apply_stage[stages_key][step_number] for step_key in step: assert apply_step[step_key] == step[step_key], \ - "for [%s][%d][%s][%d][%s] found: %s but expected: %s" % \ + "for [%s][%d][%s][%d][%s] found: %s but expected: %s\n" \ + "\n===== Found Step =====\n%s\n" \ + "\n===== Expected Step =====\n%s" % \ (key, stage_number, stages_key, step_number, step_key, - apply_step[step_key], step[step_key]) + apply_step[step_key], step[step_key], + json.dumps(apply_step, indent=2), + json.dumps(step, indent=2)) step_number += 1 else: assert apply_stage[stages_key] == stage[stages_key], \ @@ -112,6 +130,20 @@ def fake_host_name_controller_0(): return 'controller-0' +def fake_host_name_flipper(before, after, n): + """Flip host after n-calls""" + + hosts = [before, after] + index = 0 + + def _inner(): + nonlocal index + index += 1 + return hosts[index // n % 2] + + return _inner + + def fake_callback(): return diff --git a/nfv/nfv-tests/nfv_unit_tests/tests/test_nfv_client.py b/nfv/nfv-tests/nfv_unit_tests/tests/test_nfv_client.py index 8c466f6a..00e03601 100755 --- a/nfv/nfv-tests/nfv_unit_tests/tests/test_nfv_client.py +++ b/nfv/nfv-tests/nfv_unit_tests/tests/test_nfv_client.py @@ -252,7 +252,7 @@ class TestCLISwDeployStrategy(TestNFVClientShell, self.set_strategy('sw-deploy-strategy') def required_create_fields(self): - """sw-deploy strategy requires 'release' parameter for create""" + """Kube Upgrade requires a to-version for create""" return ['starlingx-24.03.1'] diff --git a/nfv/nfv-tests/nfv_unit_tests/tests/test_sw_deploy_strategy.py b/nfv/nfv-tests/nfv_unit_tests/tests/test_sw_deploy_strategy.py new file mode 100755 index 00000000..9cf63869 --- /dev/null +++ b/nfv/nfv-tests/nfv_unit_tests/tests/test_sw_deploy_strategy.py @@ -0,0 +1,1472 @@ +# +# Copyright (c) 2016-2024 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +from unittest import mock +import uuid + +from nfv_common import strategy as common_strategy +from nfv_vim import nfvi + +from nfv_vim.objects import HOST_PERSONALITY +from nfv_vim.objects import SW_UPDATE_ALARM_RESTRICTION +from nfv_vim.objects import SW_UPDATE_APPLY_TYPE +from nfv_vim.objects import SW_UPDATE_INSTANCE_ACTION +from nfv_vim.objects import SwUpgrade +from nfv_vim.strategy._strategy import SwUpgradeStrategy + +from nfv_unit_tests.tests import sw_update_testcase + + +# utility method for the formatting of unlock-hosts stage as dict +# workers default to 5 retries with 120 second delay between attempts +# std controllers and storage have 0 retries +def _unlock_hosts_stage_as_dict(host_names, retry_count=5, retry_delay=120): + return { + 'name': 'unlock-hosts', + 'entity_names': host_names, + 'retry_count': retry_count, + 'retry_delay': retry_delay, + 'timeout': 1800, + } + + +@mock.patch('nfv_vim.event_log._instance._event_issue', + sw_update_testcase.fake_event_issue) +@mock.patch('nfv_vim.objects._sw_update.SwUpdate.save', + sw_update_testcase.fake_save) +@mock.patch('nfv_vim.objects._sw_update.timers.timers_create_timer', + sw_update_testcase.fake_timer) +@mock.patch('nfv_vim.nfvi.nfvi_compute_plugin_disabled', + sw_update_testcase.fake_nfvi_compute_plugin_disabled) +class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): + + def create_sw_deploy_strategy(self, + controller_apply_type=SW_UPDATE_APPLY_TYPE.IGNORE, + storage_apply_type=SW_UPDATE_APPLY_TYPE.IGNORE, + worker_apply_type=SW_UPDATE_APPLY_TYPE.IGNORE, + max_parallel_worker_hosts=10, + alarm_restrictions=SW_UPDATE_ALARM_RESTRICTION.STRICT, + default_instance_action=SW_UPDATE_INSTANCE_ACTION.STOP_START, + release="123.1", + nfvi_upgrade=None, + single_controller=False + ): + """ + Create a software update strategy + """ + strategy = SwUpgradeStrategy( + uuid=str(uuid.uuid4()), + controller_apply_type=controller_apply_type, + storage_apply_type=storage_apply_type, + worker_apply_type=worker_apply_type, + max_parallel_worker_hosts=max_parallel_worker_hosts, + default_instance_action=default_instance_action, + alarm_restrictions=alarm_restrictions, + release=release, + ignore_alarms=[], + single_controller=single_controller, + ) + strategy.nfvi_upgrade = nfvi_upgrade + return strategy + + def _gen_aiosx_hosts_and_strategy( + self, + openstack=True, + # aio-sx must be stop_start + default_instance_action=SW_UPDATE_INSTANCE_ACTION.STOP_START, + worker_apply_type=SW_UPDATE_APPLY_TYPE.SERIAL, + instances=None, + **kwargs, + ): + self.create_host('controller-0', aio=True, openstack_installed=openstack) + + controller_hosts = [] + for host in list(self._host_table.values()): + if HOST_PERSONALITY.WORKER in host.personality: + controller_hosts.append(host) + + for args in instances or []: + self.create_instance(*args) + + strategy = self.create_sw_deploy_strategy( + single_controller=True, + default_instance_action=default_instance_action, + worker_apply_type=worker_apply_type, + **kwargs, + ) + + return controller_hosts, strategy + + def _gen_aiodx_hosts_and_strategy( + self, + openstack=True, + default_instance_action=SW_UPDATE_INSTANCE_ACTION.STOP_START, + worker_apply_type=SW_UPDATE_APPLY_TYPE.SERIAL, + instances=None, + **kwargs, + ): + self.create_host('controller-0', aio=True, openstack_installed=openstack) + self.create_host('controller-1', aio=True, openstack_installed=openstack) + + controller_hosts = [] + for host in list(self._host_table.values()): + if HOST_PERSONALITY.WORKER in host.personality: + controller_hosts.append(host) + + for args in instances or []: + self.create_instance(*args) + + strategy = self.create_sw_deploy_strategy( + default_instance_action=default_instance_action, + worker_apply_type=worker_apply_type, + **kwargs, + ) + + return controller_hosts, strategy + + def _gen_standard_hosts_and_strategy( + self, + openstack=True, + default_instance_action=SW_UPDATE_INSTANCE_ACTION.STOP_START, + controller_apply_type=SW_UPDATE_APPLY_TYPE.SERIAL, + storage_apply_type=SW_UPDATE_APPLY_TYPE.PARALLEL, + worker_apply_type=SW_UPDATE_APPLY_TYPE.PARALLEL, + instances=None, + **kwargs, + ): + self.create_host('controller-0', openstack_installed=openstack) + self.create_host('controller-1', openstack_installed=openstack) + self.create_host('compute-0') + self.create_host('compute-1') + self.create_host('compute-2') + self.create_host('storage-0') + self.create_host('storage-1') + self.create_host('storage-2') + + controller_hosts = [] + storage_hosts = [] + worker_hosts = [] + + for host in list(self._host_table.values()): + if HOST_PERSONALITY.CONTROLLER in host.personality: + controller_hosts.append(host) + + elif HOST_PERSONALITY.STORAGE in host.personality: + storage_hosts.append(host) + + elif HOST_PERSONALITY.WORKER in host.personality: + worker_hosts.append(host) + + for args in instances or []: + self.create_instance(*args) + + strategy = self.create_sw_deploy_strategy( + default_instance_action=default_instance_action, + controller_apply_type=controller_apply_type, + worker_apply_type=worker_apply_type, + storage_apply_type=storage_apply_type, + **kwargs, + ) + + return controller_hosts, storage_hosts, worker_hosts, strategy + + # ~~~ SW-DEPLOY Start ~~~ + + @mock.patch('nfv_vim.strategy._strategy.get_local_host_name', + sw_update_testcase.fake_host_name_controller_0) + def test_sw_deploy_strategy_start_on_controller_0_aiosx(self): + """ + Test the sw_upgrade strategy start stage controller-0 + - sx + Verify: + - pass + """ + + release = "888.8" + _, strategy = self._gen_aiosx_hosts_and_strategy( + release=release, + nfvi_upgrade=nfvi.objects.v1.Upgrade( + release, + { + 'state': 'available', + 'reboot_required': 'N', + }, + None, + ) + ) + + strategy._add_upgrade_start_stage() + + apply_phase = strategy.apply_phase.as_dict() + + expected_results = { + 'total_stages': 1, + 'stages': [ + {'name': 'sw-upgrade-start', + 'total_steps': 4, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'sw-deploy-precheck', + 'release': release}, + {'name': 'start-upgrade', + 'release': release}, + {'name': 'system-stabilize', + 'timeout': 60}, + ] + } + ] + } + + sw_update_testcase.validate_strategy_persists(strategy) + sw_update_testcase.validate_phase(apply_phase, expected_results) + + @mock.patch('nfv_vim.strategy._strategy.get_local_host_name', + sw_update_testcase.fake_host_name_controller_0) + def test_sw_deploy_strategy_start_on_controller_0__aiodx(self): + """ + Test the sw_upgrade strategy start stages on controller-0: + - dx + Verify: + - pass + """ + + release = "888.8" + _, strategy = self._gen_aiosx_hosts_and_strategy( + release=release, + nfvi_upgrade=nfvi.objects.v1.Upgrade( + release, + { + 'state': 'available', + 'reboot_required': 'N', + }, + None, + ) + ) + + strategy._add_upgrade_start_stage() + + apply_phase = strategy.apply_phase.as_dict() + + expected_results = { + 'total_stages': 1, + 'stages': [ + {'name': 'sw-upgrade-start', + 'total_steps': 4, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'sw-deploy-precheck', + 'release': release}, + {'name': 'start-upgrade', + 'release': release}, + {'name': 'system-stabilize', + 'timeout': 60}, + ] + } + ] + } + + sw_update_testcase.validate_strategy_persists(strategy) + sw_update_testcase.validate_phase(apply_phase, expected_results) + + # pylint: disable=no-member + @mock.patch('nfv_vim.strategy._strategy.get_local_host_name', + sw_update_testcase.fake_host_name_flipper('controller-1', 'controller-0', n=2)) + def test_sw_deploy_strategy_start_on_controller_1_aiodx(self): + """ + Test the sw_upgrade strategy start stages on controller-1: + - dx + Verify: + - pass + """ + + release = "888.8" + _, strategy = self._gen_aiodx_hosts_and_strategy( + release=release, + nfvi_upgrade=nfvi.objects.v1.Upgrade( + release, + { + 'state': 'available', + 'reboot_required': 'N', + }, + None, + ) + ) + + strategy._add_upgrade_start_stage() + + apply_phase = strategy.apply_phase.as_dict() + + expected_results = { + 'total_stages': 1, + 'stages': [ + {'name': 'sw-upgrade-start', + 'total_steps': 6, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'swact-hosts', + 'entity_names': ['controller-1']}, + {'name': 'sw-deploy-precheck', + 'release': release}, + {'name': 'start-upgrade', + 'release': release}, + {'name': 'system-stabilize', + 'timeout': 60}, + {'name': 'swact-hosts', + 'entity_names': ['controller-0']}, + ] + } + ] + } + + sw_update_testcase.validate_strategy_persists(strategy) + sw_update_testcase.validate_phase(apply_phase, expected_results) + + # ~~~ SW-DEPLOY Complete ~~~ + + @mock.patch('nfv_vim.strategy._strategy.get_local_host_name', + sw_update_testcase.fake_host_name_controller_0) + def test_sw_deploy_strategy_complete_on_controller_0_aiosx(self): + """ + Test the sw_upgrade strategy complete stage controller-0 + - sx + Verify: + - pass + """ + + release = "888.8" + _, strategy = self._gen_aiosx_hosts_and_strategy( + release=release, + nfvi_upgrade=nfvi.objects.v1.Upgrade( + release, + { + 'state': 'available', + 'reboot_required': 'N', + }, + None, + ) + ) + + strategy._add_upgrade_complete_stage() + + apply_phase = strategy.apply_phase.as_dict() + + expected_results = { + 'total_stages': 1, + 'stages': [ + {'name': 'sw-upgrade-complete', + 'total_steps': 4, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'activate-upgrade', + 'release': release}, + {'name': 'complete-upgrade', + 'release': release}, + {'name': 'system-stabilize', + 'timeout': 60}, + ] + } + ] + } + + sw_update_testcase.validate_strategy_persists(strategy) + sw_update_testcase.validate_phase(apply_phase, expected_results) + + @mock.patch('nfv_vim.strategy._strategy.get_local_host_name', + sw_update_testcase.fake_host_name_controller_0) + def test_sw_deploy_strategy_complete_on_controller_0__aiodx(self): + """ + Test the sw_upgrade strategy complete stages on controller-0: + - dx + Verify: + - pass + """ + + release = "888.8" + _, strategy = self._gen_aiodx_hosts_and_strategy( + release=release, + nfvi_upgrade=nfvi.objects.v1.Upgrade( + release, + { + 'state': 'available', + 'reboot_required': 'N', + }, + None, + ) + ) + + strategy._add_upgrade_complete_stage() + + apply_phase = strategy.apply_phase.as_dict() + + expected_results = { + 'total_stages': 1, + 'stages': [ + {'name': 'sw-upgrade-complete', + 'total_steps': 4, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'activate-upgrade', + 'release': release}, + {'name': 'complete-upgrade', + 'release': release}, + {'name': 'system-stabilize', + 'timeout': 60}, + ] + } + ] + } + + sw_update_testcase.validate_strategy_persists(strategy) + sw_update_testcase.validate_phase(apply_phase, expected_results) + + # pylint: disable=no-member + @mock.patch('nfv_vim.strategy._strategy.get_local_host_name', + sw_update_testcase.fake_host_name_flipper('controller-1', 'controller-0', n=2)) + def test_sw_deploy_strategy_complete_on_controller_1_aiodx(self): + """ + Test the sw_upgrade strategy complete stages on controller-1: + - dx + Verify: + - pass + """ + + release = "888.8" + _, strategy = self._gen_aiodx_hosts_and_strategy( + release=release, + nfvi_upgrade=nfvi.objects.v1.Upgrade( + release, + { + 'state': 'available', + 'reboot_required': 'N', + }, + None, + ) + ) + + strategy._add_upgrade_complete_stage() + + apply_phase = strategy.apply_phase.as_dict() + + expected_results = { + 'total_stages': 1, + 'stages': [ + {'name': 'sw-upgrade-complete', + 'total_steps': 5, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'swact-hosts', + 'entity_names': ['controller-1']}, + {'name': 'activate-upgrade', + 'release': release}, + {'name': 'complete-upgrade', + 'release': release}, + {'name': 'system-stabilize', + 'timeout': 60}, + ] + } + ] + } + + sw_update_testcase.validate_strategy_persists(strategy) + sw_update_testcase.validate_phase(apply_phase, expected_results) + + # ~~~ AIO-SX NRR ~~~ + + def test_sw_deploy_strategy_aiosx_controllers_serial_nrr(self): + """ + Test the sw_deploy strategy add controller strategy stages: + - aio-sx host + - serial apply + - no reboot required + - stop_start instances + - no instances + Verify: + - Pass + """ + + controller_hosts, strategy = self._gen_aiosx_hosts_and_strategy() + + success, reason = strategy._add_worker_strategy_stages( + worker_hosts=controller_hosts, + reboot=False) + + assert success is True, reason + + apply_phase = strategy.apply_phase.as_dict() + expected_results = { + 'total_stages': 1, + 'stages': [ + { + 'name': 'sw-upgrade-worker-hosts', + 'total_steps': 3, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'upgrade-hosts', 'entity_names': ['controller-0']}, + {'name': 'system-stabilize', 'timeout': 30}, + ] + }, + ], + } + + sw_update_testcase.validate_strategy_persists(strategy) + sw_update_testcase.validate_phase(apply_phase, expected_results) + + def test_sw_deploy_strategy_aiosx_controllers_parallel_nrr(self): + """ + Test the sw_deploy strategy add controller strategy stages: + - aio-sx host + - parallel apply + - no reboot required + - stop_start instances + - no instances + Verify: + - Pass + """ + + controller_hosts, strategy = self._gen_aiosx_hosts_and_strategy( + worker_apply_type=SW_UPDATE_APPLY_TYPE.PARALLEL, + ) + + success, reason = strategy._add_worker_strategy_stages( + worker_hosts=controller_hosts, + reboot=False) + + assert success is True, reason + + apply_phase = strategy.apply_phase.as_dict() + expected_results = { + 'total_stages': 1, + 'stages': [ + { + 'name': 'sw-upgrade-worker-hosts', + 'total_steps': 3, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'upgrade-hosts', 'entity_names': ['controller-0']}, + {'name': 'system-stabilize', 'timeout': 30}, + ] + }, + ], + } + + sw_update_testcase.validate_strategy_persists(strategy) + sw_update_testcase.validate_phase(apply_phase, expected_results) + + def test_sw_deploy_strategy_aiosx_controllers_parallel_nrr_no_openstack(self): + """ + Test the sw_deploy strategy add controller strategy stages: + - aio-sx host + - parallel apply + - no reboot required + - stop_start instances + - no instances + - no openstack + Verify: + - Pass + """ + + controller_hosts, strategy = self._gen_aiosx_hosts_and_strategy( + openstack=False, + worker_apply_type=SW_UPDATE_APPLY_TYPE.PARALLEL, + ) + + success, reason = strategy._add_worker_strategy_stages( + worker_hosts=controller_hosts, + reboot=False) + + assert success is True, reason + + apply_phase = strategy.apply_phase.as_dict() + expected_results = { + 'total_stages': 1, + 'stages': [ + { + 'name': 'sw-upgrade-worker-hosts', + 'total_steps': 3, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'upgrade-hosts', 'entity_names': ['controller-0']}, + {'name': 'system-stabilize', 'timeout': 30}, + ] + }, + ], + } + + sw_update_testcase.validate_strategy_persists(strategy) + sw_update_testcase.validate_phase(apply_phase, expected_results) + + def test_sw_deploy_strategy_aiosx_controllers_parallel_nrr_instances_migrate(self): + """ + Test the sw_deploy strategy add controller strategy stages: + - aio-sx host + - parallel apply + - no reboot required + - migrate instances + - instances + Verify: + - Fail + """ + + controller_hosts, strategy = self._gen_aiosx_hosts_and_strategy( + default_instance_action=SW_UPDATE_INSTANCE_ACTION.MIGRATE, + worker_apply_type=SW_UPDATE_APPLY_TYPE.PARALLEL, + instances=[('small', 'test_instance_0', 'controller-0')], + ) + + success, reason = strategy._add_worker_strategy_stages( + worker_hosts=controller_hosts, + reboot=False) + + assert success is False + assert reason == 'cannot migrate instances in a single controller configuration' + + sw_update_testcase.validate_strategy_persists(strategy) + + def test_sw_deploy_strategy_aiosx_controllers_parallel_nrr_instances_stop_start(self): + """ + Test the sw_deploy strategy add controller strategy stages: + - aio-sx host + - parallel apply + - no reboot required + - stop_start instances + - instances + Verify: + - Pass + """ + + controller_hosts, strategy = self._gen_aiosx_hosts_and_strategy( + worker_apply_type=SW_UPDATE_APPLY_TYPE.PARALLEL, + instances=[('small', 'test_instance_0', 'controller-0')], + ) + + success, reason = strategy._add_worker_strategy_stages( + worker_hosts=controller_hosts, + reboot=False) + + assert success is True, reason + + apply_phase = strategy.apply_phase.as_dict() + expected_results = { + 'total_stages': 1, + 'stages': [ + { + 'name': 'sw-upgrade-worker-hosts', + 'total_steps': 3, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'upgrade-hosts', 'entity_names': ['controller-0']}, + {'name': 'system-stabilize', 'timeout': 30}, + ] + }, + ], + } + + sw_update_testcase.validate_strategy_persists(strategy) + sw_update_testcase.validate_phase(apply_phase, expected_results) + + # ~~~ AIO-SX RR ~~~ + + def test_sw_deploy_strategy_aiosx_controllers_serial_rr(self): + """ + Test the sw_deploy strategy add controller strategy stages: + - aio-sx host + - serial apply + - reboot required + - stop_start instances + - no instances + Verify: + - Pass + """ + + controller_hosts, strategy = self._gen_aiosx_hosts_and_strategy() + + success, reason = strategy._add_worker_strategy_stages( + worker_hosts=controller_hosts, + reboot=True) + + assert success is True, reason + + apply_phase = strategy.apply_phase.as_dict() + expected_results = { + 'total_stages': 1, + 'stages': [ + { + 'name': 'sw-upgrade-worker-hosts', + 'total_steps': 6, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'lock-hosts', 'entity_names': ['controller-0']}, + {'name': 'upgrade-hosts', 'entity_names': ['controller-0']}, + {'name': 'system-stabilize', 'timeout': 15}, + {'name': 'unlock-hosts', + 'entity_names': ['controller-0'], 'retry_count': 0, 'retry_delay': 120}, + {'name': 'wait-alarms-clear', 'timeout': 2400}, + ] + }, + ], + } + + sw_update_testcase.validate_strategy_persists(strategy) + sw_update_testcase.validate_phase(apply_phase, expected_results) + + def test_sw_deploy_strategy_aiosx_controllers_parallel_rr(self): + """ + Test the sw_deploy strategy add controller strategy stages: + - aio-sx host + - parallel apply + - reboot required + - stop_start instances + - no instances + Verify: + - Pass + """ + + controller_hosts, strategy = self._gen_aiosx_hosts_and_strategy( + worker_apply_type=SW_UPDATE_APPLY_TYPE.PARALLEL, + ) + + success, reason = strategy._add_worker_strategy_stages( + worker_hosts=controller_hosts, + reboot=True) + + assert success is True, reason + + apply_phase = strategy.apply_phase.as_dict() + expected_results = { + 'total_stages': 1, + 'stages': [ + { + 'name': 'sw-upgrade-worker-hosts', + 'total_steps': 6, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'lock-hosts', 'entity_names': ['controller-0']}, + {'name': 'upgrade-hosts', 'entity_names': ['controller-0']}, + {'name': 'system-stabilize', 'timeout': 15}, + {'name': 'unlock-hosts', + 'entity_names': ['controller-0'], 'retry_count': 0, 'retry_delay': 120}, + {'name': 'wait-alarms-clear', 'timeout': 2400}, + ] + }, + ], + } + + sw_update_testcase.validate_strategy_persists(strategy) + sw_update_testcase.validate_phase(apply_phase, expected_results) + + def test_sw_deploy_strategy_aiosx_controllers_parallel_rr_no_openstack(self): + """ + Test the sw_deploy strategy add controller strategy stages: + - aio-sx host + - parallel apply + - reboot required + - stop_start instances + - no instances + - no openstack + Verify: + - Pass + """ + + controller_hosts, strategy = self._gen_aiosx_hosts_and_strategy( + openstack=False, + worker_apply_type=SW_UPDATE_APPLY_TYPE.PARALLEL, + ) + + success, reason = strategy._add_worker_strategy_stages( + worker_hosts=controller_hosts, + reboot=True) + + assert success is True, reason + + apply_phase = strategy.apply_phase.as_dict() + expected_results = { + 'total_stages': 1, + 'stages': [ + { + 'name': 'sw-upgrade-worker-hosts', + 'total_steps': 6, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'lock-hosts', 'entity_names': ['controller-0']}, + {'name': 'upgrade-hosts', 'entity_names': ['controller-0']}, + {'name': 'system-stabilize', 'timeout': 15}, + {'name': 'unlock-hosts', + 'entity_names': ['controller-0'], 'retry_count': 0, 'retry_delay': 120}, + {'name': 'wait-alarms-clear', 'timeout': 2400}, + ] + }, + ], + } + + sw_update_testcase.validate_strategy_persists(strategy) + sw_update_testcase.validate_phase(apply_phase, expected_results) + + def test_sw_deploy_strategy_aiosx_controllers_parallel_rr_instances_migrate(self): + """ + Test the sw_deploy strategy add controller strategy stages: + - aio-sx host + - parallel apply + - reboot required + - migrate instances + - instances + Verify: + - Pass + """ + + controller_hosts, strategy = self._gen_aiosx_hosts_and_strategy( + default_instance_action=SW_UPDATE_INSTANCE_ACTION.MIGRATE, + worker_apply_type=SW_UPDATE_APPLY_TYPE.PARALLEL, + instances=[('small', 'test_instance_0', 'controller-0')], + ) + + success, reason = strategy._add_worker_strategy_stages( + worker_hosts=controller_hosts, + reboot=True) + + assert success is False + assert reason == 'cannot migrate instances in a single controller configuration' + + sw_update_testcase.validate_strategy_persists(strategy) + + def test_sw_deploy_strategy_aiosx_controllers_parallel_rr_instances_stop_start(self): + """ + Test the sw_deploy strategy add controller strategy stages: + - aio-sx host + - parallel apply + - reboot required + - stop_start instances + - instances + Verify: + - Pass + """ + + controller_hosts, strategy = self._gen_aiosx_hosts_and_strategy( + worker_apply_type=SW_UPDATE_APPLY_TYPE.PARALLEL, + instances=[('small', 'test_instance_0', 'controller-0')], + ) + + success, reason = strategy._add_worker_strategy_stages( + worker_hosts=controller_hosts, + reboot=True) + + assert success is True, reason + + apply_phase = strategy.apply_phase.as_dict() + expected_results = { + 'total_stages': 1, + 'stages': [ + { + 'name': 'sw-upgrade-worker-hosts', + 'total_steps': 8, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'stop-instances', 'entity_names': ['test_instance_0']}, + {'name': 'lock-hosts', 'entity_names': ['controller-0']}, + {'name': 'upgrade-hosts', 'entity_names': ['controller-0']}, + {'name': 'system-stabilize', 'timeout': 15}, + {'name': 'unlock-hosts', + 'entity_names': ['controller-0'], 'retry_count': 0, 'retry_delay': 120}, + {'name': 'start-instances', 'entity_names': ['test_instance_0']}, + {'name': 'wait-alarms-clear', 'timeout': 2400}, + ] + }, + ], + } + + sw_update_testcase.validate_strategy_persists(strategy) + sw_update_testcase.validate_phase(apply_phase, expected_results) + + def test_sw_deploy_strategy_aiosx_already_deployed(self): + """ + Test the sw_deploy strategy when patch already deployed: + - patch already committed + Verify: + - Fail + """ + + _, strategy = self._gen_aiosx_hosts_and_strategy( + nfvi_upgrade=nfvi.objects.v1.Upgrade( + '13.01', + {'state': 'deployed'}, + None, + ) + ) + + fake_upgrade_obj = SwUpgrade() + strategy.sw_update_obj = fake_upgrade_obj + + strategy.build_complete(common_strategy.STRATEGY_RESULT.SUCCESS, "") + expected_reason = "Software release is already deployed or committed." + bpr = strategy.build_phase + + assert strategy._state == common_strategy.STRATEGY_STATE.BUILD_FAILED + assert bpr.result == common_strategy.STRATEGY_PHASE_RESULT.FAILED + assert bpr.result_reason == expected_reason, strategy.build_phase.result_reason + + def test_sw_deploy_strategy_aiosx_already_committed(self): + """ + Test the sw_deploy strategy when patch already committed: + - patch already committed + Verify: + - Fail + """ + + _, strategy = self._gen_aiosx_hosts_and_strategy( + nfvi_upgrade=nfvi.objects.v1.Upgrade( + '13.01', + {'state': 'committed'}, + None, + ) + ) + + fake_upgrade_obj = SwUpgrade() + strategy.sw_update_obj = fake_upgrade_obj + + strategy.build_complete(common_strategy.STRATEGY_RESULT.SUCCESS, "") + expected_reason = "Software release is already deployed or committed." + bpr = strategy.build_phase + + assert strategy._state == common_strategy.STRATEGY_STATE.BUILD_FAILED + assert bpr.result == common_strategy.STRATEGY_PHASE_RESULT.FAILED + assert bpr.result_reason == expected_reason, strategy.build_phase.result_reason + + def test_sw_deploy_strategy_aiosx_release_does_not_exist(self): + """ + Test the sw_deploy strategy when patch does not exist: + - patch does not exist + Verify: + - Fail + """ + + _, strategy = self._gen_aiosx_hosts_and_strategy( + nfvi_upgrade=nfvi.objects.v1.Upgrade( + '13.01', + None, + None, + ) + ) + + fake_upgrade_obj = SwUpgrade() + strategy.sw_update_obj = fake_upgrade_obj + + strategy.build_complete(common_strategy.STRATEGY_RESULT.SUCCESS, "") + expected_reason = "Software release does not exist." + bpr = strategy.build_phase + + assert strategy._state == common_strategy.STRATEGY_STATE.BUILD_FAILED + assert bpr.result == common_strategy.STRATEGY_PHASE_RESULT.FAILED + assert bpr.result_reason == expected_reason, strategy.build_phase.result_reason + +# ~~~~~~ Full Apply Phase ~~~~~~~ + + def test_sw_deploy_strategy_aiosx_apply_phase_nrr(self): + """ + Test the sw_deploy strategy apply phase: + - aio-sx + - nrr + Verify: + - Pass + """ + + release = '888.8' + _, strategy = self._gen_aiosx_hosts_and_strategy( + release=release, + nfvi_upgrade=nfvi.objects.v1.Upgrade( + release, + { + 'state': 'available', + 'reboot_required': 'N', + }, + None, + ) + ) + + fake_upgrade_obj = SwUpgrade() + strategy.sw_update_obj = fake_upgrade_obj + + strategy.build_complete(common_strategy.STRATEGY_RESULT.SUCCESS, "") + apply_phase = strategy.apply_phase.as_dict() + + expected_results = { + 'total_stages': 3, + 'stages': [ + { + 'name': 'sw-upgrade-start', + 'total_steps': 4, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'sw-deploy-precheck', 'release': release}, + {'name': 'start-upgrade', 'release': release}, + {'name': 'system-stabilize', 'timeout': 60}, + ], + }, + { + 'name': 'sw-upgrade-worker-hosts', + 'total_steps': 3, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'upgrade-hosts', 'entity_names': ['controller-0']}, + {'name': 'system-stabilize', 'timeout': 30}, + ] + }, + { + 'name': 'sw-upgrade-complete', + 'total_steps': 4, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'activate-upgrade', 'release': release}, + {'name': 'complete-upgrade', 'release': release}, + {'name': 'system-stabilize', 'timeout': 60}, + ], + }, + ], + } + + sw_update_testcase.validate_strategy_persists(strategy) + sw_update_testcase.validate_phase(apply_phase, expected_results) + + def test_sw_deploy_strategy_aiosx_apply_phase_rr(self): + """ + Test the sw_deploy strategy apply phase: + - aio-sx + - rr + Verify: + - Pass + """ + + release = '888.8' + _, strategy = self._gen_aiosx_hosts_and_strategy( + release=release, + nfvi_upgrade=nfvi.objects.v1.Upgrade( + release, + { + 'state': 'available', + 'reboot_required': 'Y', + }, + None, + ) + ) + + fake_upgrade_obj = SwUpgrade() + strategy.sw_update_obj = fake_upgrade_obj + + strategy.build_complete(common_strategy.STRATEGY_RESULT.SUCCESS, "") + apply_phase = strategy.apply_phase.as_dict() + + expected_results = { + 'total_stages': 3, + 'stages': [ + { + 'name': 'sw-upgrade-start', + 'total_steps': 4, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'sw-deploy-precheck', 'release': release}, + {'name': 'start-upgrade', 'release': release}, + {'name': 'system-stabilize', 'timeout': 60}, + ], + }, + { + 'name': 'sw-upgrade-worker-hosts', + 'total_steps': 6, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'lock-hosts', 'entity_names': ['controller-0']}, + {'name': 'upgrade-hosts', 'entity_names': ['controller-0']}, + {'name': 'system-stabilize', 'timeout': 15}, + {'name': 'unlock-hosts', + 'entity_names': ['controller-0'], 'retry_count': 0, 'retry_delay': 120}, + {'name': 'wait-alarms-clear', 'timeout': 2400}, + ] + }, + { + 'name': 'sw-upgrade-complete', + 'total_steps': 4, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'activate-upgrade', 'release': release}, + {'name': 'complete-upgrade', 'release': release}, + {'name': 'system-stabilize', 'timeout': 60}, + ], + }, + ], + } + + sw_update_testcase.validate_strategy_persists(strategy) + sw_update_testcase.validate_phase(apply_phase, expected_results) + + def test_sw_deploy_strategy_aiodx_apply_phase_nrr(self): + """ + Test the sw_deploy strategy apply phase: + - aio-dx + - nrr + Verify: + - Pass + """ + + release = '888.8' + _, strategy = self._gen_aiodx_hosts_and_strategy( + release=release, + nfvi_upgrade=nfvi.objects.v1.Upgrade( + release, + { + 'state': 'available', + 'reboot_required': 'N', + }, + None, + ) + ) + + fake_upgrade_obj = SwUpgrade() + strategy.sw_update_obj = fake_upgrade_obj + + strategy.build_complete(common_strategy.STRATEGY_RESULT.SUCCESS, "") + apply_phase = strategy.apply_phase.as_dict() + + expected_results = { + 'total_stages': 4, + 'stages': [ + { + 'name': 'sw-upgrade-start', + 'total_steps': 4, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'sw-deploy-precheck', 'release': release}, + {'name': 'start-upgrade', 'release': release}, + {'name': 'system-stabilize', 'timeout': 60}, + ], + }, + { + 'name': 'sw-upgrade-worker-hosts', + 'total_steps': 3, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'upgrade-hosts', 'entity_names': ['controller-1']}, + {'name': 'system-stabilize', 'timeout': 30}, + ] + }, + { + 'name': 'sw-upgrade-worker-hosts', + 'total_steps': 3, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'upgrade-hosts', 'entity_names': ['controller-0']}, + {'name': 'system-stabilize', 'timeout': 30}, + ] + }, + { + 'name': 'sw-upgrade-complete', + 'total_steps': 4, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'activate-upgrade', 'release': release}, + {'name': 'complete-upgrade', 'release': release}, + {'name': 'system-stabilize', 'timeout': 60}, + ], + }, + ], + } + + sw_update_testcase.validate_strategy_persists(strategy) + sw_update_testcase.validate_phase(apply_phase, expected_results) + + def test_sw_deploy_strategy_aiodx_apply_phase_rr(self): + """ + Test the sw_deploy strategy apply phase: + - aio-dx + - rr + Verify: + - Pass + """ + + release = '888.8' + _, strategy = self._gen_aiodx_hosts_and_strategy( + release=release, + nfvi_upgrade=nfvi.objects.v1.Upgrade( + release, + { + 'state': 'available', + 'reboot_required': 'Y', + }, + None, + ) + ) + + fake_upgrade_obj = SwUpgrade() + strategy.sw_update_obj = fake_upgrade_obj + + strategy.build_complete(common_strategy.STRATEGY_RESULT.SUCCESS, "") + apply_phase = strategy.apply_phase.as_dict() + + expected_results = { + 'total_stages': 4, + 'stages': [ + { + 'name': 'sw-upgrade-start', + 'total_steps': 4, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'sw-deploy-precheck', 'release': release}, + {'name': 'start-upgrade', 'release': release}, + {'name': 'system-stabilize', 'timeout': 60}, + ], + }, + { + 'name': 'sw-upgrade-worker-hosts', + 'total_steps': 7, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'swact-hosts', 'entity_names': ['controller-1']}, + {'name': 'lock-hosts', 'entity_names': ['controller-1']}, + {'name': 'upgrade-hosts', 'entity_names': ['controller-1']}, + {'name': 'system-stabilize', 'timeout': 15}, + {'name': 'unlock-hosts', + 'entity_names': ['controller-1'], 'retry_count': 0, 'retry_delay': 120}, + {'name': 'wait-alarms-clear', 'timeout': 2400}, + ] + }, + { + 'name': 'sw-upgrade-worker-hosts', + 'total_steps': 7, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'swact-hosts', 'entity_names': ['controller-0']}, + {'name': 'lock-hosts', 'entity_names': ['controller-0']}, + {'name': 'upgrade-hosts', 'entity_names': ['controller-0']}, + {'name': 'system-stabilize', 'timeout': 15}, + {'name': 'unlock-hosts', + 'entity_names': ['controller-0'], 'retry_count': 0, 'retry_delay': 120}, + {'name': 'wait-alarms-clear', 'timeout': 2400}, + ] + }, + { + 'name': 'sw-upgrade-complete', + 'total_steps': 4, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'activate-upgrade', 'release': release}, + {'name': 'complete-upgrade', 'release': release}, + {'name': 'system-stabilize', 'timeout': 60}, + ], + }, + ], + } + + sw_update_testcase.validate_strategy_persists(strategy) + sw_update_testcase.validate_phase(apply_phase, expected_results) + + def test_sw_deploy_strategy_standard_apply_phase_nrr(self): + """ + Test the sw_deploy strategy apply phase: + - standard + - nrr + - parallel storage + - parallel workers + Verify: + - Pass + """ + + release = '888.8' + _, _, _, strategy = self._gen_standard_hosts_and_strategy( + release=release, + nfvi_upgrade=nfvi.objects.v1.Upgrade( + release, + { + 'state': 'available', + 'reboot_required': 'N', + }, + None, + ) + ) + + fake_upgrade_obj = SwUpgrade() + strategy.sw_update_obj = fake_upgrade_obj + + strategy.build_complete(common_strategy.STRATEGY_RESULT.SUCCESS, "") + apply_phase = strategy.apply_phase.as_dict() + + expected_results = { + 'total_stages': 6, + 'stages': [ + { + 'name': 'sw-upgrade-start', + 'total_steps': 4, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'sw-deploy-precheck', 'release': release}, + {'name': 'start-upgrade', 'release': release}, + {'name': 'system-stabilize', 'timeout': 60}, + ], + }, + { + 'name': 'sw-upgrade-controllers', + 'total_steps': 3, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'upgrade-hosts', 'entity_names': ['controller-1']}, + {'name': 'system-stabilize', 'timeout': 30}, + ] + }, + { + 'name': 'sw-upgrade-controllers', + 'total_steps': 3, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'upgrade-hosts', 'entity_names': ['controller-0']}, + {'name': 'system-stabilize', 'timeout': 30}, + ] + }, + { + 'name': 'sw-upgrade-storage-hosts', + 'total_steps': 3, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'upgrade-hosts', + 'entity_names': ['storage-0', 'storage-1', 'storage-2']}, + {'name': 'system-stabilize', 'timeout': 30}, + ] + }, + { + 'name': 'sw-upgrade-worker-hosts', + 'total_steps': 3, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'upgrade-hosts', + 'entity_names': ['compute-0', 'compute-1', 'compute-2']}, + {'name': 'system-stabilize', 'timeout': 30}, + ] + }, + { + 'name': 'sw-upgrade-complete', + 'total_steps': 4, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'activate-upgrade', 'release': release}, + {'name': 'complete-upgrade', 'release': release}, + {'name': 'system-stabilize', 'timeout': 60}, + ], + }, + ], + } + + sw_update_testcase.validate_strategy_persists(strategy) + sw_update_testcase.validate_phase(apply_phase, expected_results) + + def test_sw_deploy_strategy_standard_apply_phase_rr(self): + """ + Test the sw_deploy strategy apply phase: + - standard + - rr + - parallel storage + - parallel workers + Verify: + - Pass + """ + + release = '888.8' + _, _, _, strategy = self._gen_standard_hosts_and_strategy( + release=release, + nfvi_upgrade=nfvi.objects.v1.Upgrade( + release, + { + 'state': 'available', + 'reboot_required': 'Y', + }, + None, + ) + ) + + fake_upgrade_obj = SwUpgrade() + strategy.sw_update_obj = fake_upgrade_obj + + strategy.build_complete(common_strategy.STRATEGY_RESULT.SUCCESS, "") + apply_phase = strategy.apply_phase.as_dict() + + expected_results = { + 'total_stages': 6, + 'stages': [ + { + 'name': 'sw-upgrade-start', + 'total_steps': 4, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'sw-deploy-precheck', 'release': release}, + {'name': 'start-upgrade', 'release': release}, + {'name': 'system-stabilize', 'timeout': 60}, + ], + }, + { + 'name': 'sw-upgrade-controllers', + 'total_steps': 7, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'swact-hosts', 'entity_names': ['controller-1']}, + {'name': 'lock-hosts', 'entity_names': ['controller-1']}, + {'name': 'upgrade-hosts', 'entity_names': ['controller-1']}, + {'name': 'system-stabilize', 'timeout': 15}, + {'name': 'unlock-hosts', + 'entity_names': ['controller-1'], 'retry_count': 0, 'retry_delay': 120}, + {'name': 'wait-alarms-clear', 'timeout': 2400}, + ] + }, + { + 'name': 'sw-upgrade-controllers', + 'total_steps': 7, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'swact-hosts', 'entity_names': ['controller-0']}, + {'name': 'lock-hosts', 'entity_names': ['controller-0']}, + {'name': 'upgrade-hosts', 'entity_names': ['controller-0']}, + {'name': 'system-stabilize', 'timeout': 15}, + {'name': 'unlock-hosts', + 'entity_names': ['controller-0'], 'retry_count': 0, 'retry_delay': 120}, + {'name': 'wait-alarms-clear', 'timeout': 2400}, + ] + }, + { + 'name': 'sw-upgrade-storage-hosts', + 'total_steps': 6, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'lock-hosts', + 'entity_names': ['storage-0', 'storage-1', 'storage-2']}, + {'name': 'upgrade-hosts', + 'entity_names': ['storage-0', 'storage-1', 'storage-2']}, + {'name': 'system-stabilize', 'timeout': 15}, + {'name': 'unlock-hosts', + 'entity_names': ['storage-0', 'storage-1', 'storage-2']}, + {'name': 'wait-data-sync', 'timeout': 1800}, + ] + }, + { + 'name': 'sw-upgrade-worker-hosts', + 'total_steps': 6, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'lock-hosts', + 'entity_names': ['compute-0', 'compute-1', 'compute-2']}, + {'name': 'upgrade-hosts', + 'entity_names': ['compute-0', 'compute-1', 'compute-2']}, + {'name': 'system-stabilize', 'timeout': 15}, + {'name': 'unlock-hosts', + 'entity_names': ['compute-0', 'compute-1', 'compute-2']}, + {'name': 'wait-alarms-clear', 'timeout': 600}, + ] + }, + { + 'name': 'sw-upgrade-complete', + 'total_steps': 4, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'activate-upgrade', 'release': release}, + {'name': 'complete-upgrade', 'release': release}, + {'name': 'system-stabilize', 'timeout': 60}, + ], + }, + ], + } + + sw_update_testcase.validate_strategy_persists(strategy) + sw_update_testcase.validate_phase(apply_phase, expected_results) diff --git a/nfv/nfv-tests/nfv_unit_tests/tests/test_sw_upgrade_strategy.py b/nfv/nfv-tests/nfv_unit_tests/tests/test_sw_upgrade_strategy.py index a453f480..ed730044 100755 --- a/nfv/nfv-tests/nfv_unit_tests/tests/test_sw_upgrade_strategy.py +++ b/nfv/nfv-tests/nfv_unit_tests/tests/test_sw_upgrade_strategy.py @@ -14,15 +14,15 @@ from nfv_vim.objects import HOST_NAME from nfv_vim.objects import HOST_PERSONALITY from nfv_vim.objects import SW_UPDATE_ALARM_RESTRICTION from nfv_vim.objects import SW_UPDATE_APPLY_TYPE +from nfv_vim.objects import SW_UPDATE_INSTANCE_ACTION from nfv_vim.objects import SwUpgrade from nfv_vim.strategy._strategy import strategy_rebuild_from_dict from nfv_vim.strategy._strategy import SwUpgradeStrategy -from nfv_vim.nfvi.objects.v1 import UPGRADE_STATE - from nfv_unit_tests.tests import sw_update_testcase +# TODO(jkraitbe): Update this when retry count is decicded. # utility method for the formatting of unlock-hosts stage as dict # workers default to 5 retries with 120 second delay between attempts # std controllers and storage have 0 retries @@ -30,9 +30,9 @@ def _unlock_hosts_stage_as_dict(host_names, retry_count=5, retry_delay=120): return { 'name': 'unlock-hosts', 'entity_names': host_names, - 'retry_count': retry_count, - 'retry_delay': retry_delay, - 'timeout': 1800, + # 'retry_count': retry_count, + # 'retry_delay': retry_delay, + # 'timeout': 1800, } @@ -47,13 +47,13 @@ def _unlock_hosts_stage_as_dict(host_names, retry_count=5, retry_delay=120): class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): def create_sw_upgrade_strategy(self, + controller_apply_type=SW_UPDATE_APPLY_TYPE.SERIAL, storage_apply_type=SW_UPDATE_APPLY_TYPE.IGNORE, worker_apply_type=SW_UPDATE_APPLY_TYPE.IGNORE, max_parallel_worker_hosts=10, alarm_restrictions=SW_UPDATE_ALARM_RESTRICTION.STRICT, + default_instance_action=SW_UPDATE_INSTANCE_ACTION.MIGRATE, release='starlingx-24.03.1', - start_upgrade=False, - complete_upgrade=False, nfvi_upgrade=None, single_controller=False ): @@ -62,19 +62,76 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): """ strategy = SwUpgradeStrategy( uuid=str(uuid.uuid4()), + controller_apply_type=controller_apply_type, storage_apply_type=storage_apply_type, worker_apply_type=worker_apply_type, max_parallel_worker_hosts=max_parallel_worker_hosts, + default_instance_action=default_instance_action, alarm_restrictions=alarm_restrictions, release=release, - start_upgrade=start_upgrade, - complete_upgrade=complete_upgrade, ignore_alarms=[], single_controller=single_controller, ) + + if nfvi_upgrade is True: + nfvi_upgrade = nfvi.objects.v1.Upgrade( + release, + { + 'state': 'available', + 'reboot_required': 'Y', + }, + None, + ) + strategy.nfvi_upgrade = nfvi_upgrade return strategy + def _gen_aiosx_hosts_and_strategy( + self, + openstack=True, + default_instance_action=SW_UPDATE_INSTANCE_ACTION.STOP_START, + worker_apply_type=SW_UPDATE_APPLY_TYPE.SERIAL, + **kwargs, + ): + self.create_host('controller-0', aio=True, openstack_installed=openstack) + + controller_hosts = [] + for host in list(self._host_table.values()): + if HOST_PERSONALITY.WORKER in host.personality: + controller_hosts.append(host) + + strategy = self.create_sw_upgrade_strategy( + single_controller=True, + default_instance_action=default_instance_action, + worker_apply_type=worker_apply_type, + **kwargs, + ) + + return controller_hosts, strategy + + def _gen_aiodx_hosts_and_strategy( + self, + openstack=True, + default_instance_action=SW_UPDATE_INSTANCE_ACTION.MIGRATE, + worker_apply_type=SW_UPDATE_APPLY_TYPE.SERIAL, + **kwargs, + ): + self.create_host('controller-0', aio=True, openstack_installed=openstack) + self.create_host('controller-1', aio=True, openstack_installed=openstack) + + controller_hosts = [] + for host in list(self._host_table.values()): + if HOST_PERSONALITY.WORKER in host.personality: + controller_hosts.append(host) + + strategy = self.create_sw_upgrade_strategy( + default_instance_action=default_instance_action, + worker_apply_type=worker_apply_type, + **kwargs, + ) + + return controller_hosts, strategy + @mock.patch('nfv_vim.strategy._strategy.get_local_host_name', sw_update_testcase.fake_host_name_controller_1) def test_sw_upgrade_strategy_worker_stages_ignore(self): @@ -116,7 +173,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): worker_hosts=sorted_worker_hosts, reboot=True) - assert success is True, "Strategy creation failed" + assert success is True, f"Strategy creation failed: {reason}" apply_phase = strategy.apply_phase.as_dict() @@ -176,45 +233,55 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): 'total_stages': 3, 'stages': [ {'name': 'sw-upgrade-worker-hosts', - 'total_steps': 5, + 'total_steps': 8, 'steps': [ {'name': 'query-alarms'}, + {'name': 'disable-host-services'}, + {'name': 'migrate-instances-from-host', + 'host_names': ['compute-2', 'compute-3'], + 'entity_names': []}, {'name': 'lock-hosts', 'entity_names': ['compute-2', 'compute-3']}, {'name': 'upgrade-hosts', 'entity_names': ['compute-2', 'compute-3']}, + {'name': 'system-stabilize', + 'timeout': 15}, _unlock_hosts_stage_as_dict(['compute-2', 'compute-3']), {'name': 'wait-alarms-clear', 'timeout': 600} ] }, {'name': 'sw-upgrade-worker-hosts', - 'total_steps': 7, + 'total_steps': 8, 'steps': [ {'name': 'query-alarms'}, {'name': 'disable-host-services'}, - {'name': 'migrate-instances', + {'name': 'migrate-instances-from-host', 'entity_names': ['test_instance_0']}, {'name': 'lock-hosts', 'entity_names': ['compute-0']}, {'name': 'upgrade-hosts', 'entity_names': ['compute-0']}, + {'name': 'system-stabilize', + 'timeout': 15}, _unlock_hosts_stage_as_dict(['compute-0']), {'name': 'wait-alarms-clear', 'timeout': 600} ] }, {'name': 'sw-upgrade-worker-hosts', - 'total_steps': 7, + 'total_steps': 8, 'steps': [ {'name': 'query-alarms'}, {'name': 'disable-host-services'}, - {'name': 'migrate-instances', + {'name': 'migrate-instances-from-host', 'entity_names': ['test_instance_1']}, {'name': 'lock-hosts', 'entity_names': ['compute-1']}, {'name': 'upgrade-hosts', 'entity_names': ['compute-1']}, + {'name': 'system-stabilize', + 'timeout': 15}, _unlock_hosts_stage_as_dict(['compute-1']), {'name': 'wait-alarms-clear', 'timeout': 600} @@ -279,24 +346,30 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): 'total_stages': 4, 'stages': [ {'name': 'sw-upgrade-worker-hosts', - 'total_steps': 5, + 'total_steps': 8, 'steps': [ {'name': 'query-alarms'}, + {'name': 'disable-host-services'}, + {'name': 'migrate-instances-from-host', + 'host_names': ['compute-1', 'compute-5'], + 'entity_names': []}, {'name': 'lock-hosts', 'entity_names': ['compute-1', 'compute-5']}, {'name': 'upgrade-hosts', 'entity_names': ['compute-1', 'compute-5']}, + {'name': 'system-stabilize', + 'timeout': 15}, _unlock_hosts_stage_as_dict(['compute-1', 'compute-5']), {'name': 'wait-alarms-clear', 'timeout': 600} ] }, {'name': 'sw-upgrade-worker-hosts', - 'total_steps': 7, + 'total_steps': 8, 'steps': [ {'name': 'query-alarms'}, {'name': 'disable-host-services'}, - {'name': 'migrate-instances', + {'name': 'migrate-instances-from-host', 'entity_names': ['test_instance_0', 'test_instance_2', 'test_instance_3']}, @@ -304,6 +377,8 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): 'entity_names': ['compute-0', 'compute-2', 'compute-3']}, {'name': 'upgrade-hosts', 'entity_names': ['compute-0', 'compute-2', 'compute-3']}, + {'name': 'system-stabilize', + 'timeout': 15}, _unlock_hosts_stage_as_dict( ['compute-0', 'compute-2', 'compute-3']), {'name': 'wait-alarms-clear', @@ -311,11 +386,11 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): ] }, {'name': 'sw-upgrade-worker-hosts', - 'total_steps': 7, + 'total_steps': 8, 'steps': [ {'name': 'query-alarms'}, {'name': 'disable-host-services'}, - {'name': 'migrate-instances', + {'name': 'migrate-instances-from-host', 'entity_names': ['test_instance_4', 'test_instance_6', 'test_instance_7']}, @@ -323,6 +398,8 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): 'entity_names': ['compute-4', 'compute-6', 'compute-7']}, {'name': 'upgrade-hosts', 'entity_names': ['compute-4', 'compute-6', 'compute-7']}, + {'name': 'system-stabilize', + 'timeout': 15}, _unlock_hosts_stage_as_dict( ['compute-4', 'compute-6', 'compute-7']), {'name': 'wait-alarms-clear', @@ -330,17 +407,19 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): ] }, {'name': 'sw-upgrade-worker-hosts', - 'total_steps': 7, + 'total_steps': 8, 'steps': [ {'name': 'query-alarms'}, {'name': 'disable-host-services'}, - {'name': 'migrate-instances', + {'name': 'migrate-instances-from-host', 'entity_names': ['test_instance_8', 'test_instance_9']}, {'name': 'lock-hosts', 'entity_names': ['compute-8', 'compute-9']}, {'name': 'upgrade-hosts', 'entity_names': ['compute-8', 'compute-9']}, + {'name': 'system-stabilize', + 'timeout': 15}, _unlock_hosts_stage_as_dict( ['compute-8', 'compute-9']), {'name': 'wait-alarms-clear', @@ -391,6 +470,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): strategy = self.create_sw_upgrade_strategy( worker_apply_type=SW_UPDATE_APPLY_TYPE.PARALLEL, + default_instance_action=SW_UPDATE_INSTANCE_ACTION.MIGRATE, max_parallel_worker_hosts=3 ) @@ -404,21 +484,45 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): 'total_stages': 5, 'stages': [ {'name': 'sw-upgrade-worker-hosts', - 'total_steps': 8, + 'total_steps': 9, 'steps': [ {'name': 'query-alarms'}, + {'name': 'swact-hosts', + 'entity_names': ['controller-0']}, {'name': 'disable-host-services'}, - {'name': 'migrate-instances', + {'name': 'migrate-instances-from-host', + 'host_names': ['controller-0'], 'entity_names': ['test_instance_6']}, - {'name': 'swact-hosts', - 'entity_names': ['controller-0']}, {'name': 'lock-hosts', 'entity_names': ['controller-0']}, {'name': 'upgrade-hosts', 'entity_names': ['controller-0']}, + {'name': 'system-stabilize', + 'timeout': 15}, _unlock_hosts_stage_as_dict(['controller-0']), - {'name': 'wait-data-sync', - 'timeout': 14400} + {'name': 'wait-alarms-clear', + 'timeout': 2400}, + ] + }, + {'name': 'sw-upgrade-worker-hosts', + 'total_steps': 9, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'swact-hosts', + 'entity_names': ['controller-1']}, + {'name': 'disable-host-services'}, + {'name': 'migrate-instances-from-host', + 'host_names': ['controller-1'], + 'entity_names': ['test_instance_7']}, + {'name': 'lock-hosts', + 'entity_names': ['controller-1']}, + {'name': 'upgrade-hosts', + 'entity_names': ['controller-1']}, + {'name': 'system-stabilize', + 'timeout': 15}, + _unlock_hosts_stage_as_dict(['controller-1']), + {'name': 'wait-alarms-clear', + 'timeout': 2400}, ] }, {'name': 'sw-upgrade-worker-hosts', @@ -426,38 +530,27 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): 'steps': [ {'name': 'query-alarms'}, {'name': 'disable-host-services'}, - {'name': 'migrate-instances', - 'entity_names': ['test_instance_7']}, - {'name': 'swact-hosts', - 'entity_names': ['controller-1']}, - {'name': 'lock-hosts', - 'entity_names': ['controller-1']}, - {'name': 'upgrade-hosts', - 'entity_names': ['controller-1']}, - _unlock_hosts_stage_as_dict(['controller-1']), - {'name': 'wait-data-sync', - 'timeout': 14400} - ] - }, - {'name': 'sw-upgrade-worker-hosts', - 'total_steps': 5, - 'steps': [ - {'name': 'query-alarms'}, + {'name': 'migrate-instances-from-host', + 'host_names': ['compute-1'], + 'entity_names': []}, {'name': 'lock-hosts', 'entity_names': ['compute-1']}, {'name': 'upgrade-hosts', 'entity_names': ['compute-1']}, + {'name': 'system-stabilize', + 'timeout': 15}, _unlock_hosts_stage_as_dict(['compute-1']), {'name': 'wait-alarms-clear', - 'timeout': 600} + 'timeout': 600}, ] }, {'name': 'sw-upgrade-worker-hosts', - 'total_steps': 7, + 'total_steps': 8, 'steps': [ {'name': 'query-alarms'}, {'name': 'disable-host-services'}, - {'name': 'migrate-instances', + {'name': 'migrate-instances-from-host', + 'host_names': ['compute-0', 'compute-2', 'compute-3'], 'entity_names': ['test_instance_0', 'test_instance_2', 'test_instance_3']}, @@ -465,6 +558,8 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): 'entity_names': ['compute-0', 'compute-2', 'compute-3']}, {'name': 'upgrade-hosts', 'entity_names': ['compute-0', 'compute-2', 'compute-3']}, + {'name': 'system-stabilize', + 'timeout': 15}, _unlock_hosts_stage_as_dict( ['compute-0', 'compute-2', 'compute-3']), {'name': 'wait-alarms-clear', @@ -472,16 +567,19 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): ] }, {'name': 'sw-upgrade-worker-hosts', - 'total_steps': 7, + 'total_steps': 8, 'steps': [ {'name': 'query-alarms'}, {'name': 'disable-host-services'}, - {'name': 'migrate-instances', + {'name': 'migrate-instances-from-host', + 'host_names': ['compute-4'], 'entity_names': ['test_instance_4']}, {'name': 'lock-hosts', 'entity_names': ['compute-4']}, {'name': 'upgrade-hosts', 'entity_names': ['compute-4']}, + {'name': 'system-stabilize', + 'timeout': 15}, _unlock_hosts_stage_as_dict(['compute-4']), {'name': 'wait-alarms-clear', 'timeout': 600} @@ -564,13 +662,19 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): 'total_stages': 13, 'stages': [ {'name': 'sw-upgrade-worker-hosts', - 'total_steps': 5, + 'total_steps': 8, 'steps': [ {'name': 'query-alarms'}, + {'name': 'disable-host-services'}, + {'name': 'migrate-instances-from-host', + 'host_names': stage_hosts[0], + 'entity_names': []}, {'name': 'lock-hosts', 'entity_names': stage_hosts[0]}, {'name': 'upgrade-hosts', 'entity_names': stage_hosts[0]}, + {'name': 'system-stabilize', + 'timeout': 15}, _unlock_hosts_stage_as_dict(stage_hosts[0]), {'name': 'wait-alarms-clear', 'timeout': 600} @@ -582,16 +686,19 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): for x in range(1, len(stage_hosts)): expected_results['stages'].append( {'name': 'sw-upgrade-worker-hosts', - 'total_steps': 7, + 'total_steps': 8, 'steps': [ {'name': 'query-alarms'}, {'name': 'disable-host-services'}, - {'name': 'migrate-instances', + {'name': 'migrate-instances-from-host', + 'host_names': stage_hosts[x], 'entity_names': stage_instances[x]}, {'name': 'lock-hosts', 'entity_names': stage_hosts[x]}, {'name': 'upgrade-hosts', 'entity_names': stage_hosts[x]}, + {'name': 'system-stabilize', + 'timeout': 15}, _unlock_hosts_stage_as_dict(stage_hosts[x]), {'name': 'wait-alarms-clear', 'timeout': 600} @@ -648,61 +755,75 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): 'total_stages': 4, 'stages': [ {'name': 'sw-upgrade-worker-hosts', - 'total_steps': 5, + 'total_steps': 7, 'steps': [ {'name': 'query-alarms'}, + {'name': 'migrate-instances-from-host', + 'host_names': ['compute-2']}, {'name': 'lock-hosts', 'entity_names': ['compute-2']}, {'name': 'upgrade-hosts', 'entity_names': ['compute-2']}, + {'name': 'system-stabilize', + 'timeout': 15}, _unlock_hosts_stage_as_dict(['compute-2']), {'name': 'wait-alarms-clear', - 'timeout': 600} + 'timeout': 600}, ] }, {'name': 'sw-upgrade-worker-hosts', - 'total_steps': 5, + 'total_steps': 7, 'steps': [ {'name': 'query-alarms'}, + {'name': 'migrate-instances-from-host', + 'host_names': ['compute-3']}, {'name': 'lock-hosts', 'entity_names': ['compute-3']}, {'name': 'upgrade-hosts', 'entity_names': ['compute-3']}, + {'name': 'system-stabilize', + 'timeout': 15}, _unlock_hosts_stage_as_dict(['compute-3']), {'name': 'wait-alarms-clear', - 'timeout': 600} + 'timeout': 600}, ] }, {'name': 'sw-upgrade-worker-hosts', - 'total_steps': 6, + 'total_steps': 7, 'steps': [ {'name': 'query-alarms'}, - {'name': 'migrate-instances', + {'name': 'migrate-instances-from-host', + 'host_names': ['compute-0'], 'entity_names': ['test_instance_0']}, {'name': 'lock-hosts', 'entity_names': ['compute-0']}, {'name': 'upgrade-hosts', 'entity_names': ['compute-0']}, + {'name': 'system-stabilize', + 'timeout': 15}, _unlock_hosts_stage_as_dict(['compute-0']), {'name': 'wait-alarms-clear', - 'timeout': 600} + 'timeout': 600}, ] }, {'name': 'sw-upgrade-worker-hosts', - 'total_steps': 6, + 'total_steps': 7, 'steps': [ {'name': 'query-alarms'}, - {'name': 'migrate-instances', + {'name': 'migrate-instances-from-host', + 'host_names': ['compute-1'], 'entity_names': ['test_instance_1']}, {'name': 'lock-hosts', 'entity_names': ['compute-1']}, {'name': 'upgrade-hosts', 'entity_names': ['compute-1']}, + {'name': 'system-stabilize', + 'timeout': 15}, _unlock_hosts_stage_as_dict(['compute-1']), {'name': 'wait-alarms-clear', - 'timeout': 600} + 'timeout': 600}, ] - } + }, ] } @@ -744,54 +865,22 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): expected_results = { 'total_stages': 4, 'stages': [ - {'name': 'sw-upgrade-worker-hosts', - 'total_steps': 5, - 'steps': [ - {'name': 'query-alarms'}, - {'name': 'lock-hosts', - 'entity_names': ['compute-0']}, - {'name': 'upgrade-hosts', - 'entity_names': ['compute-0']}, - _unlock_hosts_stage_as_dict(['compute-0']), - {'name': 'system-stabilize'} - ] - }, - {'name': 'sw-upgrade-worker-hosts', - 'total_steps': 5, - 'steps': [ - {'name': 'query-alarms'}, - {'name': 'lock-hosts', - 'entity_names': ['compute-1']}, - {'name': 'upgrade-hosts', - 'entity_names': ['compute-1']}, - _unlock_hosts_stage_as_dict(['compute-1']), - {'name': 'system-stabilize'} - ] - }, - {'name': 'sw-upgrade-worker-hosts', - 'total_steps': 5, - 'steps': [ - {'name': 'query-alarms'}, - {'name': 'lock-hosts', - 'entity_names': ['compute-2']}, - {'name': 'upgrade-hosts', - 'entity_names': ['compute-2']}, - _unlock_hosts_stage_as_dict(['compute-2']), - {'name': 'system-stabilize'} - ] - }, - {'name': 'sw-upgrade-worker-hosts', - 'total_steps': 5, - 'steps': [ - {'name': 'query-alarms'}, - {'name': 'lock-hosts', - 'entity_names': ['compute-3']}, - {'name': 'upgrade-hosts', - 'entity_names': ['compute-3']}, - _unlock_hosts_stage_as_dict(['compute-3']), - {'name': 'system-stabilize'} - ] + { + 'name': 'sw-upgrade-worker-hosts', + 'total_steps': 6, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'lock-hosts', + 'entity_names': [f'compute-{i}']}, + {'name': 'upgrade-hosts', + 'entity_names': [f'compute-{i}']}, + {'name': 'system-stabilize', 'timeout': 15}, + {'name': 'unlock-hosts', + 'entity_names': [f'compute-{i}']}, + {'name': 'system-stabilize', 'timeout': 60}, + ] } + for i in range(4) ] } @@ -929,45 +1018,34 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): apply_phase = strategy.apply_phase.as_dict() expected_results = { - 'total_stages': 3, + 'total_stages': 2, 'stages': [ {'name': 'sw-upgrade-storage-hosts', - 'total_steps': 5, + 'total_steps': 6, 'steps': [ {'name': 'query-alarms'}, {'name': 'lock-hosts', - 'entity_names': ['storage-0']}, + 'entity_names': ['storage-0', 'storage-2']}, {'name': 'upgrade-hosts', - 'entity_names': ['storage-0']}, - _unlock_hosts_stage_as_dict(['storage-0']), + 'entity_names': ['storage-0', 'storage-2']}, + {'name': 'system-stabilize', 'timeout': 15}, + _unlock_hosts_stage_as_dict(['storage-0', 'storage-2']), {'name': 'wait-data-sync', - 'timeout': 7200} + 'timeout': 1800} ] }, {'name': 'sw-upgrade-storage-hosts', - 'total_steps': 5, + 'total_steps': 6, 'steps': [ {'name': 'query-alarms'}, {'name': 'lock-hosts', - 'entity_names': ['storage-1', 'storage-2']}, + 'entity_names': ['storage-1', 'storage-3']}, {'name': 'upgrade-hosts', - 'entity_names': ['storage-1', 'storage-2']}, - _unlock_hosts_stage_as_dict(['storage-1', 'storage-2']), + 'entity_names': ['storage-1', 'storage-3']}, + {'name': 'system-stabilize', 'timeout': 15}, + _unlock_hosts_stage_as_dict(['storage-1', 'storage-3']), {'name': 'wait-data-sync', - 'timeout': 7200} - ] - }, - {'name': 'sw-upgrade-storage-hosts', - 'total_steps': 5, - 'steps': [ - {'name': 'query-alarms'}, - {'name': 'lock-hosts', - 'entity_names': ['storage-3']}, - {'name': 'upgrade-hosts', - 'entity_names': ['storage-3']}, - _unlock_hosts_stage_as_dict(['storage-3']), - {'name': 'wait-data-sync', - 'timeout': 7200} + 'timeout': 1800} ] } ] @@ -1014,62 +1092,22 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): expected_results = { 'total_stages': 4, 'stages': [ - {'name': 'sw-upgrade-storage-hosts', - 'total_steps': 5, - 'steps': [ - {'name': 'query-alarms'}, - {'name': 'lock-hosts', - 'entity_names': ['storage-0']}, - {'name': 'upgrade-hosts', - 'entity_names': ['storage-0']}, - _unlock_hosts_stage_as_dict(['storage-0']), - {'name': 'wait-data-sync', - 'ignore_alarms': ['900.005', '900.201', '750.006', '100.119', '900.701'], - 'timeout': 7200} - ] - }, - {'name': 'sw-upgrade-storage-hosts', - 'total_steps': 5, - 'steps': [ - {'name': 'query-alarms'}, - {'name': 'lock-hosts', - 'entity_names': ['storage-1']}, - {'name': 'upgrade-hosts', - 'entity_names': ['storage-1']}, - _unlock_hosts_stage_as_dict(['storage-1']), - {'name': 'wait-data-sync', - 'ignore_alarms': ['900.005', '900.201', '750.006', '100.119', '900.701'], - 'timeout': 7200} - ] - }, - {'name': 'sw-upgrade-storage-hosts', - 'total_steps': 5, - 'steps': [ - {'name': 'query-alarms'}, - {'name': 'lock-hosts', - 'entity_names': ['storage-2']}, - {'name': 'upgrade-hosts', - 'entity_names': ['storage-2']}, - _unlock_hosts_stage_as_dict(['storage-2']), - {'name': 'wait-data-sync', - 'ignore_alarms': ['900.005', '900.201', '750.006', '100.119', '900.701'], - 'timeout': 7200} - ] - }, - {'name': 'sw-upgrade-storage-hosts', - 'total_steps': 5, - 'steps': [ - {'name': 'query-alarms'}, - {'name': 'lock-hosts', - 'entity_names': ['storage-3']}, - {'name': 'upgrade-hosts', - 'entity_names': ['storage-3']}, - _unlock_hosts_stage_as_dict(['storage-3']), - {'name': 'wait-data-sync', - 'ignore_alarms': ['900.005', '900.201', '750.006', '100.119', '900.701'], - 'timeout': 7200} - ] - }, + { + 'name': 'sw-upgrade-storage-hosts', + 'total_steps': 6, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'lock-hosts', + 'entity_names': [f'storage-{i}']}, + {'name': 'upgrade-hosts', + 'entity_names': [f'storage-{i}']}, + {'name': 'system-stabilize', 'timeout': 15}, + {'name': 'unlock-hosts', + 'entity_names': [f'storage-{i}']}, + {'name': 'wait-data-sync', 'timeout': 1800}, + ] + } + for i in range(4) ] } @@ -1077,7 +1115,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): sw_update_testcase.validate_phase(apply_phase, expected_results) @mock.patch('nfv_vim.strategy._strategy.get_local_host_name', - sw_update_testcase.fake_host_name_controller_1) + sw_update_testcase.fake_host_name_controller_0) def test_sw_upgrade_strategy_controller_stages_serial(self): """ Test the sw_upgrade strategy add controller strategy stages: @@ -1105,17 +1143,20 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): 'total_stages': 1, 'stages': [ {'name': 'sw-upgrade-controllers', - 'total_steps': 5, + 'total_steps': 7, 'steps': [ {'name': 'query-alarms'}, + {'name': 'swact-hosts', + 'entity_names': ['controller-0']}, {'name': 'lock-hosts', 'entity_names': ['controller-0']}, {'name': 'upgrade-hosts', 'entity_names': ['controller-0']}, + {'name': 'system-stabilize', 'timeout': 15}, _unlock_hosts_stage_as_dict(['controller-0']), - {'name': 'wait-data-sync', + {'name': 'wait-alarms-clear', 'ignore_alarms': ['900.005', '900.201', '750.006', '100.119', '900.701'], - 'timeout': 14400} + 'timeout': 2400} ] } ] @@ -1152,21 +1193,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): 'total_stages': 2, 'stages': [ {'name': 'sw-upgrade-controllers', - 'total_steps': 5, - 'steps': [ - {'name': 'query-alarms'}, - {'name': 'lock-hosts', - 'entity_names': ['controller-1']}, - {'name': 'upgrade-hosts', - 'entity_names': ['controller-1']}, - _unlock_hosts_stage_as_dict(['controller-1']), - {'name': 'wait-data-sync', - 'ignore_alarms': ['900.005', '900.201', '750.006', '100.119', '900.701'], - 'timeout': 14400} - ] - }, - {'name': 'sw-upgrade-controllers', - 'total_steps': 6, + 'total_steps': 7, 'steps': [ {'name': 'query-alarms'}, {'name': 'swact-hosts', @@ -1175,10 +1202,28 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): 'entity_names': ['controller-0']}, {'name': 'upgrade-hosts', 'entity_names': ['controller-0']}, + {'name': 'system-stabilize', 'timeout': 15}, _unlock_hosts_stage_as_dict(['controller-0']), - {'name': 'wait-data-sync', + {'name': 'wait-alarms-clear', 'ignore_alarms': ['900.005', '900.201', '750.006', '100.119', '900.701'], - 'timeout': 14400} + 'timeout': 2400}, + ] + }, + {'name': 'sw-upgrade-controllers', + 'total_steps': 7, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'swact-hosts', + 'entity_names': ['controller-1']}, + {'name': 'lock-hosts', + 'entity_names': ['controller-1']}, + {'name': 'upgrade-hosts', + 'entity_names': ['controller-1']}, + {'name': 'system-stabilize', 'timeout': 15}, + _unlock_hosts_stage_as_dict(['controller-1']), + {'name': 'wait-alarms-clear', + 'ignore_alarms': ['900.005', '900.201', '750.006', '100.119', '900.701'], + 'timeout': 2400}, ] } ] @@ -1237,7 +1282,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): 'total_stages': 2, 'stages': [ {'name': 'sw-upgrade-worker-hosts', - 'total_steps': 6, + 'total_steps': 7, 'steps': [ {'name': 'query-alarms'}, {'name': 'swact-hosts', @@ -1246,14 +1291,15 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): 'entity_names': ['controller-0']}, {'name': 'upgrade-hosts', 'entity_names': ['controller-0']}, + {'name': 'system-stabilize', 'timeout': 15}, _unlock_hosts_stage_as_dict(['controller-0']), - {'name': 'wait-data-sync', + {'name': 'wait-alarms-clear', 'ignore_alarms': ['900.005', '900.201', '750.006', '100.119', '900.701'], - 'timeout': 14400} + 'timeout': 2400} ] }, {'name': 'sw-upgrade-worker-hosts', - 'total_steps': 6, + 'total_steps': 7, 'steps': [ {'name': 'query-alarms'}, {'name': 'swact-hosts', @@ -1262,10 +1308,11 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): 'entity_names': ['controller-1']}, {'name': 'upgrade-hosts', 'entity_names': ['controller-1']}, + {'name': 'system-stabilize', 'timeout': 15}, _unlock_hosts_stage_as_dict(['controller-1']), - {'name': 'wait-data-sync', + {'name': 'wait-alarms-clear', 'ignore_alarms': ['900.005', '900.201', '750.006', '100.119', '900.701'], - 'timeout': 14400} + 'timeout': 2400} ] }, ] @@ -1274,40 +1321,48 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): sw_update_testcase.validate_strategy_persists(strategy) sw_update_testcase.validate_phase(apply_phase, expected_results) - @mock.patch('nfv_vim.strategy._strategy.get_local_host_name', - sw_update_testcase.fake_host_name_controller_1) - def test_sw_upgrade_strategy_aiosx_stages_serial(self): + def test_sw_upgrade_strategy_aiosx_controllers_serial_rr(self): """ Test the sw_upgrade strategy add controller strategy stages: - - aio-sx hosts + - aio-sx host - serial apply + - reboot required + - stop_start instances + - no instances Verify: - failure """ - self.create_host('controller-0', aio=True) - controller_hosts = [] - for host in list(self._host_table.values()): - if (HOST_PERSONALITY.CONTROLLER in host.personality and - HOST_NAME.CONTROLLER_0 == host.name): - controller_hosts.append(host) + controller_hosts, strategy = self._gen_aiosx_hosts_and_strategy() - strategy = self.create_sw_upgrade_strategy(single_controller=True) - - success, reason = strategy._add_controller_strategy_stages( - controllers=controller_hosts, + success, reason = strategy._add_worker_strategy_stages( + worker_hosts=controller_hosts, reboot=True) - assert success is False - assert reason == "not enough controllers to apply software upgrades" + assert success is True, reason + apply_phase = strategy.apply_phase.as_dict() expected_results = { - 'total_stages': 0, + 'total_stages': 1, + 'stages': [ + { + 'name': 'sw-upgrade-worker-hosts', + 'total_steps': 6, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'lock-hosts', 'entity_names': ['controller-0']}, + {'name': 'upgrade-hosts', 'entity_names': ['controller-0']}, + {'name': 'system-stabilize', 'timeout': 15}, + {'name': 'unlock-hosts', 'entity_names': ['controller-0'], 'retry_count': 0, 'retry_delay': 120}, + {'name': 'wait-alarms-clear', 'timeout': 2400}, + ] + }, + ], } + sw_update_testcase.validate_strategy_persists(strategy) sw_update_testcase.validate_phase(apply_phase, expected_results) - @testtools.skip('No support for start_upgrade') def test_sw_upgrade_strategy_build_complete_serial_migrate_start_complete(self): """ Test the sw_upgrade strategy build_complete: @@ -1329,10 +1384,10 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): 'compute-0') strategy = self.create_sw_upgrade_strategy( + controller_apply_type=SW_UPDATE_APPLY_TYPE.SERIAL, storage_apply_type=SW_UPDATE_APPLY_TYPE.SERIAL, worker_apply_type=SW_UPDATE_APPLY_TYPE.SERIAL, - start_upgrade=True, - complete_upgrade=True + nfvi_upgrade=True, ) fake_upgrade_obj = SwUpgrade() @@ -1346,124 +1401,148 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): 'stages': [ {'name': 'sw-upgrade-start', 'total_steps': 4, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'sw-deploy-precheck', + 'release': strategy.nfvi_upgrade.release}, + {'name': 'start-upgrade', + 'release': strategy.nfvi_upgrade.release}, + {'name': 'system-stabilize', + 'timeout': 60}, + ] + }, + {'name': 'sw-upgrade-controllers', + 'total_steps': 7, 'steps': [ {'name': 'query-alarms'}, {'name': 'swact-hosts', 'entity_names': ['controller-1']}, - {'name': 'start-upgrade'}, - {'name': 'system-stabilize', - 'timeout': 60} - ] - }, - {'name': 'sw-upgrade-controllers', - 'total_steps': 5, - 'steps': [ - {'name': 'query-alarms'}, {'name': 'lock-hosts', 'entity_names': ['controller-1']}, {'name': 'upgrade-hosts', 'entity_names': ['controller-1']}, + {'name': 'system-stabilize', + 'timeout': 15}, _unlock_hosts_stage_as_dict(['controller-1']), - {'name': 'wait-data-sync', + {'name': 'wait-alarms-clear', 'ignore_alarms': ['900.005', '900.201', '750.006', '100.119', '900.701'], - 'timeout': 14400} + 'timeout': 2400} ] }, {'name': 'sw-upgrade-controllers', - 'total_steps': 5, + 'total_steps': 7, 'steps': [ {'name': 'query-alarms'}, + {'name': 'swact-hosts', + 'entity_names': ['controller-0']}, {'name': 'lock-hosts', 'entity_names': ['controller-0']}, {'name': 'upgrade-hosts', 'entity_names': ['controller-0']}, - _unlock_hosts_stage_as_dict(['controller-0']), - {'name': 'wait-data-sync', - 'ignore_alarms': ['900.005', '900.201', '750.006', '100.119', '900.701'], - 'timeout': 14400} - ] - }, - {'name': 'sw-upgrade-storage-hosts', - 'total_steps': 5, - 'steps': [ - {'name': 'query-alarms'}, - {'name': 'lock-hosts', - 'entity_names': ['storage-0']}, - {'name': 'upgrade-hosts', - 'entity_names': ['storage-0']}, - _unlock_hosts_stage_as_dict(['storage-0']), - {'name': 'wait-data-sync', - 'ignore_alarms': ['900.005', '900.201', '750.006', '100.119', '900.701'], - 'timeout': 7200} - ] - }, - {'name': 'sw-upgrade-storage-hosts', - 'total_steps': 5, - 'steps': [ - {'name': 'query-alarms'}, - {'name': 'lock-hosts', - 'entity_names': ['storage-1']}, - {'name': 'upgrade-hosts', - 'entity_names': ['storage-1']}, - _unlock_hosts_stage_as_dict(['storage-1']), - {'name': 'wait-data-sync', - 'ignore_alarms': ['900.005', '900.201', '750.006', '100.119', '900.701'], - 'timeout': 7200} - ] - }, - {'name': 'sw-upgrade-worker-hosts', - 'total_steps': 5, - 'steps': [ - {'name': 'query-alarms'}, - {'name': 'lock-hosts', - 'entity_names': ['compute-1']}, - {'name': 'upgrade-hosts', - 'entity_names': ['compute-1']}, - _unlock_hosts_stage_as_dict(['compute-1']), {'name': 'system-stabilize', - 'timeout': 60} + 'timeout': 15}, + _unlock_hosts_stage_as_dict(['controller-0']), + {'name': 'wait-alarms-clear', + 'ignore_alarms': ['900.005', '900.201', '750.006', '100.119', '900.701'], + 'timeout': 2400} ] }, - {'name': 'sw-upgrade-worker-hosts', + {'name': 'sw-upgrade-storage-hosts', 'total_steps': 6, 'steps': [ {'name': 'query-alarms'}, - {'name': 'migrate-instances', + {'name': 'lock-hosts', + 'entity_names': ['storage-0']}, + {'name': 'upgrade-hosts', + 'entity_names': ['storage-0']}, + {'name': 'system-stabilize', + 'timeout': 15}, + {'name': 'unlock-hosts', + 'entity_names': ['storage-0']}, + {'name': 'wait-data-sync', + 'ignore_alarms': ['900.005', '900.201', '750.006', '100.119', '900.701'], + 'timeout': 1800} + ] + }, + {'name': 'sw-upgrade-storage-hosts', + 'total_steps': 6, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'lock-hosts', + 'entity_names': ['storage-1']}, + {'name': 'upgrade-hosts', + 'entity_names': ['storage-1']}, + {'name': 'system-stabilize', + 'timeout': 15}, + {'name': 'unlock-hosts', + 'entity_names': ['storage-1']}, + {'name': 'wait-data-sync', + 'ignore_alarms': ['900.005', '900.201', '750.006', '100.119', '900.701'], + 'timeout': 1800} + ] + }, + {'name': 'sw-upgrade-worker-hosts', + 'total_steps': 7, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'migrate-instances-from-host', + 'host_names': ['compute-1']}, + {'name': 'lock-hosts', + 'entity_names': ['compute-1']}, + {'name': 'upgrade-hosts', + 'entity_names': ['compute-1']}, + {'name': 'system-stabilize', + 'timeout': 15}, + _unlock_hosts_stage_as_dict(['compute-1']), + {'name': 'wait-alarms-clear', + 'timeout': 600}, + ] + }, + {'name': 'sw-upgrade-worker-hosts', + 'total_steps': 7, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'migrate-instances-from-host', + 'host_names': ['compute-0'], 'entity_names': ['test_instance_0']}, {'name': 'lock-hosts', 'entity_names': ['compute-0']}, {'name': 'upgrade-hosts', 'entity_names': ['compute-0']}, - _unlock_hosts_stage_as_dict(['compute-0']), {'name': 'system-stabilize', - 'timeout': 60} + 'timeout': 15}, + _unlock_hosts_stage_as_dict(['compute-0']), + {'name': 'wait-alarms-clear', + 'timeout': 600}, ] }, {'name': 'sw-upgrade-complete', - 'total_steps': 5, + 'total_steps': 4, 'steps': [ {'name': 'query-alarms'}, - {'name': 'swact-hosts', - 'entity_names': ['controller-1']}, - {'name': 'activate-upgrade'}, - {'name': 'complete-upgrade'}, + {'name': 'activate-upgrade', + 'release': strategy.nfvi_upgrade.release}, + {'name': 'complete-upgrade', + 'release': strategy.nfvi_upgrade.release}, {'name': 'system-stabilize', - 'timeout': 60} - ] - } + 'timeout': 60}, + ] + } ] } sw_update_testcase.validate_strategy_persists(strategy) sw_update_testcase.validate_phase(apply_phase, expected_results) + # pylint: disable=no-member @mock.patch('nfv_vim.strategy._strategy.get_local_host_name', - sw_update_testcase.fake_host_name_controller_1) + sw_update_testcase.fake_host_name_flipper('controller-1', 'controller-0', n=2)) def test_sw_upgrade_strategy_build_complete_serial_migrate(self): """ Test the sw_upgrade strategy build_complete: - serial apply - migrate instance action + - start on controller-1 Verify: - hosts with no instances upgraded first """ @@ -1478,10 +1557,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): strategy = self.create_sw_upgrade_strategy( worker_apply_type=SW_UPDATE_APPLY_TYPE.SERIAL, - nfvi_upgrade=nfvi.objects.v1.Upgrade( - UPGRADE_STATE.UPGRADING_CONTROLLERS, - '12.01', - '13.01') + nfvi_upgrade=True, ) fake_upgrade_obj = SwUpgrade() @@ -1491,24 +1567,44 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): apply_phase = strategy.apply_phase.as_dict() expected_results = { - 'total_stages': 4, + 'total_stages': 6, 'stages': [ - {'name': 'sw-upgrade-controllers', - 'total_steps': 5, + {'name': 'sw-upgrade-start', + 'total_steps': 6, 'steps': [ {'name': 'query-alarms'}, + {'name': 'swact-hosts', + 'entity_names': ['controller-1']}, + {'name': 'sw-deploy-precheck', + 'release': strategy.nfvi_upgrade.release}, + {'name': 'start-upgrade', + 'release': strategy.nfvi_upgrade.release}, + {'name': 'system-stabilize', + 'timeout': 60}, + {'name': 'swact-hosts', + 'entity_names': ['controller-0']}, + ] + }, + {'name': 'sw-upgrade-controllers', + 'total_steps': 7, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'swact-hosts', + 'entity_names': ['controller-1']}, {'name': 'lock-hosts', 'entity_names': ['controller-1']}, {'name': 'upgrade-hosts', 'entity_names': ['controller-1']}, + {'name': 'system-stabilize', + 'timeout': 15}, _unlock_hosts_stage_as_dict(['controller-1']), - {'name': 'wait-data-sync', + {'name': 'wait-alarms-clear', 'ignore_alarms': ['900.005', '900.201', '750.006', '100.119', '900.701'], - 'timeout': 14400} + 'timeout': 2400} ] }, {'name': 'sw-upgrade-controllers', - 'total_steps': 6, + 'total_steps': 7, 'steps': [ {'name': 'query-alarms'}, {'name': 'swact-hosts', @@ -1517,40 +1613,62 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): 'entity_names': ['controller-0']}, {'name': 'upgrade-hosts', 'entity_names': ['controller-0']}, + {'name': 'system-stabilize', + 'timeout': 15}, _unlock_hosts_stage_as_dict(['controller-0']), - {'name': 'wait-data-sync', + {'name': 'wait-alarms-clear', 'ignore_alarms': ['900.005', '900.201', '750.006', '100.119', '900.701'], - 'timeout': 14400} + 'timeout': 2400} ] }, {'name': 'sw-upgrade-worker-hosts', - 'total_steps': 5, + 'total_steps': 7, 'steps': [ {'name': 'query-alarms'}, + {'name': 'migrate-instances-from-host', + 'entity_names': []}, {'name': 'lock-hosts', 'entity_names': ['compute-1']}, {'name': 'upgrade-hosts', 'entity_names': ['compute-1']}, + {'name': 'system-stabilize', + 'timeout': 15}, _unlock_hosts_stage_as_dict(['compute-1']), {'name': 'wait-alarms-clear', 'timeout': 600} ] }, {'name': 'sw-upgrade-worker-hosts', - 'total_steps': 6, + 'total_steps': 7, 'steps': [ {'name': 'query-alarms'}, - {'name': 'migrate-instances', + {'name': 'migrate-instances-from-host', 'entity_names': ['test_instance_0']}, {'name': 'lock-hosts', 'entity_names': ['compute-0']}, {'name': 'upgrade-hosts', 'entity_names': ['compute-0']}, + {'name': 'system-stabilize', + 'timeout': 15}, _unlock_hosts_stage_as_dict(['compute-0']), {'name': 'wait-alarms-clear', 'timeout': 600} ] - } + }, + {'name': 'sw-upgrade-complete', + 'total_steps': 5, + 'steps': [ + {'name': 'query-alarms'}, + {'name': 'swact-hosts', + 'entity_names': ['controller-1']}, + {'name': 'activate-upgrade', + 'release': strategy.nfvi_upgrade.release}, + {'name': 'complete-upgrade', + 'release': strategy.nfvi_upgrade.release}, + {'name': 'system-stabilize', + 'timeout': 60}, + ] + } ] } @@ -1582,10 +1700,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): strategy = self.create_sw_upgrade_strategy( worker_apply_type=SW_UPDATE_APPLY_TYPE.SERIAL, - nfvi_upgrade=nfvi.objects.v1.Upgrade( - UPGRADE_STATE.DATA_MIGRATION_COMPLETE, - '12.01', - '13.01') + nfvi_upgrade=True, ) fake_upgrade_obj = SwUpgrade() @@ -1596,49 +1711,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): expected_results = { 'total_stages': 0, - 'result': 'failed', - 'result_reason': 'invalid upgrade state for orchestration: data-migration-complete' - } - - sw_update_testcase.validate_phase(build_phase, expected_results) - - @mock.patch('nfv_vim.strategy._strategy.get_local_host_name', - sw_update_testcase.fake_host_name_controller_1) - def test_sw_upgrade_strategy_build_complete_no_upgrade_required(self): - """ - Test the sw_upgrade strategy build_complete: - - no upgrade required - Verify: - - build fails - """ - self.create_host('controller-0') - self.create_host('controller-1') - self.create_host('compute-0') - self.create_host('compute-1') - self.create_host('compute-2') - self.create_host('compute-3') - - self.create_instance('small', - "test_instance_0", - 'compute-0') - self.create_instance('small', - "test_instance_1", - 'compute-1') - - strategy = self.create_sw_upgrade_strategy( - worker_apply_type=SW_UPDATE_APPLY_TYPE.SERIAL - ) - - fake_upgrade_obj = SwUpgrade() - strategy.sw_update_obj = fake_upgrade_obj - strategy.build_complete(common_strategy.STRATEGY_RESULT.SUCCESS, "") - - build_phase = strategy.build_phase.as_dict() - - expected_results = { - 'total_stages': 0, - 'result': 'failed', - 'result_reason': 'no upgrade in progress' + 'result': 'initial', } sw_update_testcase.validate_phase(build_phase, expected_results) @@ -1668,10 +1741,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): strategy = self.create_sw_upgrade_strategy( worker_apply_type=SW_UPDATE_APPLY_TYPE.SERIAL, - nfvi_upgrade=nfvi.objects.v1.Upgrade( - UPGRADE_STATE.DATA_MIGRATION_COMPLETE, - '12.01', - '13.01') + nfvi_upgrade=True, ) fake_upgrade_obj = SwUpgrade() @@ -1682,54 +1752,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): expected_results = { 'total_stages': 0, - 'result': 'failed', - 'result_reason': 'invalid upgrade state for orchestration: data-migration-complete' - } - - sw_update_testcase.validate_phase(build_phase, expected_results) - - @mock.patch('nfv_vim.strategy._strategy.get_local_host_name', - sw_update_testcase.fake_host_name_controller_0) - def test_sw_upgrade_strategy_build_complete_from_controller_0(self): - """ - Test the sw_upgrade strategy build_complete: - - attempting build from controller-0 - Verify: - - build fails - """ - self.create_host('controller-0') - self.create_host('controller-1') - self.create_host('compute-0') - self.create_host('compute-1') - self.create_host('compute-2') - self.create_host('compute-3') - - self.create_instance('small', - "test_instance_0", - 'compute-0') - self.create_instance('small', - "test_instance_1", - 'compute-1') - - strategy = self.create_sw_upgrade_strategy( - worker_apply_type=SW_UPDATE_APPLY_TYPE.SERIAL, - nfvi_upgrade=nfvi.objects.v1.Upgrade( - UPGRADE_STATE.UPGRADING_CONTROLLERS, - '12.01', - '13.01') - ) - - fake_upgrade_obj = SwUpgrade() - strategy.sw_update_obj = fake_upgrade_obj - strategy.build_complete(common_strategy.STRATEGY_RESULT.SUCCESS, "") - - build_phase = strategy.build_phase.as_dict() - - expected_results = { - 'total_stages': 0, - 'result': 'failed', - 'result_reason': 'controller-1 must be active for orchestration ' - 'to upgrade controller-0' + 'result': 'initial', } sw_update_testcase.validate_phase(build_phase, expected_results) @@ -1760,10 +1783,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): strategy = self.create_sw_upgrade_strategy( worker_apply_type=SW_UPDATE_APPLY_TYPE.SERIAL, - nfvi_upgrade=nfvi.objects.v1.Upgrade( - UPGRADE_STATE.UPGRADING_CONTROLLERS, - '12.01', - '13.01') + nfvi_upgrade=True, ) fake_upgrade_obj = SwUpgrade() @@ -1807,10 +1827,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): strategy = self.create_sw_upgrade_strategy( worker_apply_type=SW_UPDATE_APPLY_TYPE.SERIAL, - nfvi_upgrade=nfvi.objects.v1.Upgrade( - UPGRADE_STATE.UPGRADING_CONTROLLERS, - '12.01', - '13.01') + nfvi_upgrade=True, ) fake_upgrade_obj = SwUpgrade() @@ -1828,6 +1845,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase): sw_update_testcase.validate_phase(build_phase, expected_results) + @testtools.skip('Retries not implemented') @mock.patch('nfv_vim.strategy._strategy.get_local_host_name', sw_update_testcase.fake_host_name_controller_1) def test_sw_upgrade_strategy_controller_missing_strategy_fields(self): diff --git a/nfv/nfv-vim/nfv_vim/api/controllers/v1/orchestration/sw_update/_sw_update_strategy.py b/nfv/nfv-vim/nfv_vim/api/controllers/v1/orchestration/sw_update/_sw_update_strategy.py index 75ed4bdc..f5a95395 100755 --- a/nfv/nfv-vim/nfv_vim/api/controllers/v1/orchestration/sw_update/_sw_update_strategy.py +++ b/nfv/nfv-vim/nfv_vim/api/controllers/v1/orchestration/sw_update/_sw_update_strategy.py @@ -186,11 +186,6 @@ class SwUpgradeStrategyCreateData(wsme_types.Base): default_instance_action = wsme_types.wsattr(SwUpdateInstanceActionTypes, mandatory=True, name='default-instance-action') - # Disable support for start-upgrade as it was not completed - # start_upgrade = wsme_types.wsattr( - # bool, mandatory=False, default=False, name='start-upgrade') - complete_upgrade = wsme_types.wsattr( - bool, mandatory=False, default=False, name='complete-upgrade') alarm_restrictions = wsme_types.wsattr( SwUpdateAlarmRestrictionTypes, mandatory=False, default=SW_UPDATE_ALARM_RESTRICTION_TYPES.STRICT, @@ -695,9 +690,6 @@ class SwUpgradeStrategyAPI(SwUpdateStrategyAPI): request_data.max_parallel_worker_hosts rpc_request.default_instance_action = request_data.default_instance_action rpc_request.alarm_restrictions = request_data.alarm_restrictions - # rpc_request.start_upgrade = request_data.start_upgrade - rpc_request.start_upgrade = False - rpc_request.complete_upgrade = request_data.complete_upgrade vim_connection = pecan.request.vim.open_connection() vim_connection.send(rpc_request.serialize()) msg = vim_connection.receive(timeout_in_secs=30) diff --git a/nfv/nfv-vim/nfv_vim/api/openstack/_objects.py b/nfv/nfv-vim/nfv_vim/api/openstack/_objects.py index 7488a6c2..ffe1078d 100755 --- a/nfv/nfv-vim/nfv_vim/api/openstack/_objects.py +++ b/nfv/nfv-vim/nfv_vim/api/openstack/_objects.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2015-2019, 2021-2022 Wind River Systems, Inc. +# Copyright (c) 2015-2019, 2021-2024 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -40,6 +40,7 @@ class PlatformServices(Constants): SYSINV = Constant('sysinv') PATCHING = Constant('patching') FM = Constant('fm') + USM = Constant('usm') # Platform Services Constant diff --git a/nfv/nfv-vim/nfv_vim/debug.ini b/nfv/nfv-vim/nfv_vim/debug.ini index 279a91e0..53436ea4 100644 --- a/nfv/nfv-vim/nfv_vim/debug.ini +++ b/nfv/nfv-vim/nfv_vim/debug.ini @@ -1,5 +1,5 @@ # -# Copyright (c) 2015-2023 Wind River Systems, Inc. +# Copyright (c) 2015-2024 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -193,6 +193,7 @@ nfv_plugins.nfvi_plugins.openstack.fm: debug.level.info nfv_plugins.nfvi_plugins.openstack.patching: debug.level.info nfv_plugins.nfvi_plugins.openstack.keystone: debug.level.info nfv_plugins.nfvi_plugins.openstack.sysinv: debug.level.info +nfv_plugins.nfvi_plugins.openstack.usm: debug.level.info nfv_plugins.nfvi_plugins.openstack.mtc: debug.level.info nfv_plugins.nfvi_plugins.openstack.glance: debug.level.info nfv_plugins.nfvi_plugins.openstack.cinder: debug.level.info diff --git a/nfv/nfv-vim/nfv_vim/directors/_host_director.py b/nfv/nfv-vim/nfv_vim/directors/_host_director.py index f976784a..799df374 100755 --- a/nfv/nfv-vim/nfv_vim/directors/_host_director.py +++ b/nfv/nfv-vim/nfv_vim/directors/_host_director.py @@ -693,15 +693,8 @@ class HostDirector(object): host_operation.set_failed(reason) return host_operation - if host.is_locked(): - host_operation.add_host(host.name, OPERATION_STATE.INPROGRESS) - self._nfvi_upgrade_host(host.uuid, host.name) - - else: - reason = "Cannot upgrade unlocked host %s." % host_name - DLOG.info(reason) - host_operation.set_failed(reason) - return host_operation + host_operation.add_host(host.name, OPERATION_STATE.INPROGRESS) + self._nfvi_upgrade_host(host.uuid, host.name) if host_operation.is_inprogress(): self._host_operation = host_operation diff --git a/nfv/nfv-vim/nfv_vim/directors/_sw_mgmt_director.py b/nfv/nfv-vim/nfv_vim/directors/_sw_mgmt_director.py index ae0f8444..3ed0ce40 100755 --- a/nfv/nfv-vim/nfv_vim/directors/_sw_mgmt_director.py +++ b/nfv/nfv-vim/nfv_vim/directors/_sw_mgmt_director.py @@ -74,10 +74,10 @@ class SwMgmtDirector(object): self._sw_update.strategy) return strategy_uuid, '' - def create_sw_upgrade_strategy(self, storage_apply_type, worker_apply_type, - max_parallel_worker_hosts, - alarm_restrictions, release, start_upgrade, - complete_upgrade, callback): + def create_sw_upgrade_strategy(self, + controller_apply_type, storage_apply_type, worker_apply_type, + max_parallel_worker_hosts, default_instance_action, + alarm_restrictions, release, callback): """ Create Software Upgrade Strategy """ @@ -92,11 +92,10 @@ class SwMgmtDirector(object): self._sw_update = objects.SwUpgrade() success, reason = self._sw_update.strategy_build( - strategy_uuid, storage_apply_type, - worker_apply_type, max_parallel_worker_hosts, - alarm_restrictions, release, start_upgrade, - complete_upgrade, self._ignore_alarms, - self._single_controller) + strategy_uuid, controller_apply_type, storage_apply_type, + worker_apply_type, max_parallel_worker_hosts, default_instance_action, + alarm_restrictions, release, + self._ignore_alarms, self._single_controller) schedule.schedule_function_call(callback, success, reason, self._sw_update.strategy) diff --git a/nfv/nfv-vim/nfv_vim/events/_vim_sw_update_api_events.py b/nfv/nfv-vim/nfv_vim/events/_vim_sw_update_api_events.py index de699daf..4032e003 100755 --- a/nfv/nfv-vim/nfv_vim/events/_vim_sw_update_api_events.py +++ b/nfv/nfv-vim/nfv_vim/events/_vim_sw_update_api_events.py @@ -100,13 +100,10 @@ def vim_sw_update_api_create_strategy(connection, msg): alarm_restrictions, _vim_sw_update_api_create_strategy_callback) elif 'sw-upgrade' == msg.sw_update_type: release = msg.release - start_upgrade = msg.start_upgrade - complete_upgrade = msg.complete_upgrade uuid, reason = sw_mgmt_director.create_sw_upgrade_strategy( - storage_apply_type, worker_apply_type, max_parallel_worker_hosts, - alarm_restrictions, - release, - start_upgrade, complete_upgrade, + controller_apply_type, storage_apply_type, + worker_apply_type, max_parallel_worker_hosts, + default_instance_action, alarm_restrictions, release, _vim_sw_update_api_create_strategy_callback) elif 'fw-update' == msg.sw_update_type: uuid, reason = sw_mgmt_director.create_fw_update_strategy( diff --git a/nfv/nfv-vim/nfv_vim/nfvi/__init__.py b/nfv/nfv-vim/nfv_vim/nfvi/__init__.py index 859edea7..baaf1a52 100755 --- a/nfv/nfv-vim/nfv_vim/nfvi/__init__.py +++ b/nfv/nfv-vim/nfv_vim/nfvi/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2015-2023 Wind River Systems, Inc. +# Copyright (c) 2015-2024 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -149,6 +149,7 @@ from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_register_host_state_ch from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_register_host_update_callback # noqa: F401 from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_register_host_upgrade_callback # noqa: F401 from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_register_sw_update_get_callback # noqa: F401 +from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_sw_deploy_precheck # noqa: F401 from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_swact_from_host # noqa: F401 from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_unlock_host # noqa: F401 from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_upgrade_activate # noqa: F401 diff --git a/nfv/nfv-vim/nfv_vim/nfvi/_nfvi_infrastructure_module.py b/nfv/nfv-vim/nfv_vim/nfvi/_nfvi_infrastructure_module.py index 466b7b1c..1447d2a3 100755 --- a/nfv/nfv-vim/nfv_vim/nfvi/_nfvi_infrastructure_module.py +++ b/nfv/nfv-vim/nfv_vim/nfvi/_nfvi_infrastructure_module.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2015-2023 Wind River Systems, Inc. +# Copyright (c) 2015-2024 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -367,38 +367,52 @@ def nfvi_get_kube_version_list(callback): return cmd_id -def nfvi_get_upgrade(callback): +def nfvi_get_upgrade(release, callback): """ - Get upgrade + Get Software deploy """ cmd_id = _infrastructure_plugin.invoke_plugin('get_upgrade', + release, callback=callback) return cmd_id -def nfvi_upgrade_start(callback): +def nfvi_sw_deploy_precheck(release, callback): """ - Upgrade start + Software deploy precheck """ - cmd_id = _infrastructure_plugin.invoke_plugin('upgrade_start', + cmd_id = _infrastructure_plugin.invoke_plugin('sw_deploy_precheck', + release, callback=callback) return cmd_id -def nfvi_upgrade_activate(callback): +def nfvi_upgrade_start(release, callback): """ - Upgrade activate + Software deploy start """ - cmd_id = _infrastructure_plugin.invoke_plugin('upgrade_activate', + cmd_id = _infrastructure_plugin.invoke_plugin('sw_deploy_start', + release, callback=callback) return cmd_id -def nfvi_upgrade_complete(callback): +def nfvi_upgrade_activate(release, callback): """ - Upgrade complete + Software deploy activate """ - cmd_id = _infrastructure_plugin.invoke_plugin('upgrade_complete', + cmd_id = _infrastructure_plugin.invoke_plugin('sw_deploy_activate', + release, + callback=callback) + return cmd_id + + +def nfvi_upgrade_complete(release, callback): + """ + Software deploy complete + """ + cmd_id = _infrastructure_plugin.invoke_plugin('sw_deploy_complete', + release, callback=callback) return cmd_id diff --git a/nfv/nfv-vim/nfv_vim/nfvi/objects/v1/__init__.py b/nfv/nfv-vim/nfv_vim/nfvi/objects/v1/__init__.py index 2a0e1139..bcbd4b28 100755 --- a/nfv/nfv-vim/nfv_vim/nfvi/objects/v1/__init__.py +++ b/nfv/nfv-vim/nfv_vim/nfvi/objects/v1/__init__.py @@ -69,7 +69,6 @@ from nfv_vim.nfvi.objects.v1._sw_patch import SwPatch # noqa: F401 from nfv_vim.nfvi.objects.v1._system import System # noqa: F401 from nfv_vim.nfvi.objects.v1._tenant import Tenant # noqa: F401 from nfv_vim.nfvi.objects.v1._upgrade import Upgrade # noqa: F401 -from nfv_vim.nfvi.objects.v1._upgrade import UPGRADE_STATE # noqa: F401 from nfv_vim.nfvi.objects.v1._volume import Volume # noqa: F401 from nfv_vim.nfvi.objects.v1._volume import VOLUME_ACTION # noqa: F401 from nfv_vim.nfvi.objects.v1._volume import VOLUME_AVAIL_STATUS # noqa: F401 diff --git a/nfv/nfv-vim/nfv_vim/nfvi/objects/v1/_upgrade.py b/nfv/nfv-vim/nfv_vim/nfvi/objects/v1/_upgrade.py index e01bd484..584c5873 100755 --- a/nfv/nfv-vim/nfv_vim/nfvi/objects/v1/_upgrade.py +++ b/nfv/nfv-vim/nfv_vim/nfvi/objects/v1/_upgrade.py @@ -1,50 +1,80 @@ # -# Copyright (c) 2016 Wind River Systems, Inc. +# Copyright (c) 2016-2024 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # -import six - -from nfv_common.helpers import Constant -from nfv_common.helpers import Constants -from nfv_common.helpers import Singleton from nfv_vim.nfvi.objects.v1._object import ObjectData - -@six.add_metaclass(Singleton) -class UpgradeState(Constants): - """ - Upgrade State Constants - """ - UNKNOWN = Constant('unknown') - STARTING = Constant('starting') - STARTED = Constant('started') - DATA_MIGRATION = Constant('data-migration') - DATA_MIGRATION_COMPLETE = Constant('data-migration-complete') - DATA_MIGRATION_FAILED = Constant('data-migration-failed') - UPGRADING_CONTROLLERS = Constant('upgrading-controllers') - UPGRADING_HOSTS = Constant('upgrading-hosts') - ACTIVATION_REQUESTED = Constant('activation-requested') - ACTIVATING = Constant('activating') - ACTIVATION_COMPLETE = Constant('activation-complete') - COMPLETING = Constant('completing') - COMPLETED = Constant('completed') - ABORTING = Constant('aborting') - ABORT_COMPLETING = Constant('abort-completing') - ABORTING_ROLLBACK = Constant('aborting-reinstall') - - -# Upgrade Constant Instantiation -UPGRADE_STATE = UpgradeState() +# USM states +USM_ABORTING = 'aborting' +USM_AVAILABLE = 'available' +USM_COMMITTED = 'committed' +USM_DEPLOYED = 'deployed' +USM_DEPLOYING_ACTIVATE = 'deploying-activate' +USM_DEPLOYING_COMPLETE = 'deploying-complete' +USM_DEPLOYING_HOST = 'deploying-host' +USM_DEPLOYING_START = 'deploying-start' +USM_REMOVING = 'removing' +USM_UNAVAILABLE = 'unavailable' +USM_UNKNOWN = 'n/a' +USM_REBOOT_REQUIRED = "Y" class Upgrade(ObjectData): """ NFVI Upgrade Object """ - def __init__(self, state, from_release, to_release): + def __init__(self, release, release_info, hosts_info): super(Upgrade, self).__init__('1.0.0') - self.update(dict(state=state, - from_release=from_release, - to_release=to_release)) + self.update(dict(release=release, + release_info=release_info, + hosts_info=hosts_info)) + + @property + def state(self): + if self.release_info is None: + return None + + return self.release_info["state"] + + @property + def reboot_required(self): + if self.release_info is None: + return None + + return self.release_info["reboot_required"] == USM_REBOOT_REQUIRED + + @property + def is_available(self): + return self.state == USM_AVAILABLE + + @property + def is_deployed(self): + return self.state == USM_DEPLOYED + + @property + def is_commited(self): + return self.state == USM_COMMITTED + + @property + def is_started(self): + return self.state == USM_DEPLOYING_START + + @property + def is_activated(self): + return self.state == USM_DEPLOYING_ACTIVATE + + def is_host_deployed(self, hostname): + """Return if hostname is deployed + + If is missing it is assumed not deployed. + """ + + if self.hosts_info is None: + return False + + if hostname not in self.hosts_info: + raise EnvironmentError(f"{hostname} does not exist on system") + + return self.hosts_info[hostname]["state"] == USM_DEPLOYED diff --git a/nfv/nfv-vim/nfv_vim/objects/_sw_upgrade.py b/nfv/nfv-vim/nfv_vim/objects/_sw_upgrade.py index 1faf802a..ab5ccdc1 100755 --- a/nfv/nfv-vim/nfv_vim/objects/_sw_upgrade.py +++ b/nfv/nfv-vim/nfv_vim/objects/_sw_upgrade.py @@ -29,10 +29,10 @@ class SwUpgrade(SwUpdate): sw_update_uuid=sw_update_uuid, strategy_data=strategy_data) - def strategy_build(self, strategy_uuid, storage_apply_type, - worker_apply_type, max_parallel_worker_hosts, - alarm_restrictions, release, start_upgrade, - complete_upgrade, ignore_alarms, single_controller): + def strategy_build(self, strategy_uuid, controller_apply_type, storage_apply_type, + worker_apply_type, default_instance_action, max_parallel_worker_hosts, + alarm_restrictions, release, + ignore_alarms, single_controller): """ Create a software upgrade strategy """ @@ -43,10 +43,17 @@ class SwUpgrade(SwUpdate): return False, reason self._strategy = strategy.SwUpgradeStrategy( - strategy_uuid, storage_apply_type, worker_apply_type, + strategy_uuid, + controller_apply_type, + storage_apply_type, + worker_apply_type, + default_instance_action, max_parallel_worker_hosts, - alarm_restrictions, release, start_upgrade, complete_upgrade, - ignore_alarms, single_controller) + alarm_restrictions, + release, + ignore_alarms, + single_controller, + ) self._strategy.sw_update_obj = self self._strategy.build() diff --git a/nfv/nfv-vim/nfv_vim/rpc/_rpc_message_sw_update.py b/nfv/nfv-vim/nfv_vim/rpc/_rpc_message_sw_update.py index 29955ac9..73964895 100755 --- a/nfv/nfv-vim/nfv_vim/rpc/_rpc_message_sw_update.py +++ b/nfv/nfv-vim/nfv_vim/rpc/_rpc_message_sw_update.py @@ -59,10 +59,8 @@ class APIRequestCreateSwUpdateStrategy(RPCMessage): class APIRequestCreateSwUpgradeStrategy(APIRequestCreateSwUpdateStrategy): """ - RPC API Request Message - Create Software Upgrade Strategy + RPC API Request Message - Create Software Deploy Strategy """ - start_upgrade = None - complete_upgrade = None release = None def __init__(self, msg_version=RPC_MSG_VERSION.VERSION_1_0, @@ -73,18 +71,14 @@ class APIRequestCreateSwUpgradeStrategy(APIRequestCreateSwUpdateStrategy): def serialize_payload(self, msg): super(APIRequestCreateSwUpgradeStrategy, self).serialize_payload(msg) - msg['start_upgrade'] = self.start_upgrade - msg['complete_upgrade'] = self.complete_upgrade msg['release'] = self.release def deserialize_payload(self, msg): super(APIRequestCreateSwUpgradeStrategy, self).deserialize_payload(msg) - self.start_upgrade = msg.get('start_upgrade', None) - self.complete_upgrade = msg.get('complete_upgrade', None) self.release = msg.get('release', None) def __str__(self): - return "create-sw-upgrade-strategy request: %s" % self.deserialize_payload + return "create-sw-deploy-strategy request: %s" % self.deserialize_payload class APIRequestCreateKubeRootcaUpdateStrategy(APIRequestCreateSwUpdateStrategy): diff --git a/nfv/nfv-vim/nfv_vim/strategy/__init__.py b/nfv/nfv-vim/nfv_vim/strategy/__init__.py index 50fa177f..4ddcf392 100755 --- a/nfv/nfv-vim/nfv_vim/strategy/__init__.py +++ b/nfv/nfv-vim/nfv_vim/strategy/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2015-2023 Wind River Systems, Inc. +# Copyright (c) 2015-2024 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -56,6 +56,7 @@ from nfv_vim.strategy._strategy_steps import StartInstancesStep # noqa: F401 from nfv_vim.strategy._strategy_steps import StopInstancesStep # noqa: F401 from nfv_vim.strategy._strategy_steps import STRATEGY_STEP_NAME # noqa: F401 from nfv_vim.strategy._strategy_steps import SwactHostsStep # noqa: F401 +from nfv_vim.strategy._strategy_steps import SwDeployPrecheckStep # noqa: F401 from nfv_vim.strategy._strategy_steps import SwPatchHostsStep # noqa: F401 from nfv_vim.strategy._strategy_steps import SystemConfigUpdateHostsStep # noqa: F401 from nfv_vim.strategy._strategy_steps import SystemStabilizeStep # noqa: F401 diff --git a/nfv/nfv-vim/nfv_vim/strategy/_strategy.py b/nfv/nfv-vim/nfv_vim/strategy/_strategy.py index 93aff2f8..ffc56fcc 100755 --- a/nfv/nfv-vim/nfv_vim/strategy/_strategy.py +++ b/nfv/nfv-vim/nfv_vim/strategy/_strategy.py @@ -14,7 +14,6 @@ from nfv_common.helpers import get_local_host_name from nfv_common.helpers import Singleton from nfv_common import strategy from nfv_vim.nfvi.objects.v1 import KUBE_ROOTCA_UPDATE_STATE -from nfv_vim.nfvi.objects.v1 import UPGRADE_STATE from nfv_vim.objects import HOST_GROUP_POLICY from nfv_vim.objects import HOST_NAME from nfv_vim.objects import HOST_PERSONALITY @@ -1059,6 +1058,16 @@ class PatchControllerHostsMixin(UpdateControllerHostsMixin): strategy.SwPatchHostsStep) +class SwDeployControllerHostsMixin(UpdateControllerHostsMixin): + def _add_controller_strategy_stages(self, controllers, reboot): + from nfv_vim import strategy + return self._add_update_controller_strategy_stages( + controllers, + reboot, + strategy.STRATEGY_STAGE_NAME.SW_UPGRADE_CONTROLLERS, + strategy.UpgradeHostsStep) + + class UpdateSystemConfigControllerHostsMixin(UpdateControllerHostsMixin): def _add_system_config_controller_strategy_stages(self, controllers): """ @@ -1147,6 +1156,19 @@ class PatchStorageHostsMixin(UpdateStorageHostsMixin): strategy.SwPatchHostsStep) +class SwDeployStorageHostsMixin(UpdateStorageHostsMixin): + def _add_storage_strategy_stages(self, storage_hosts, reboot): + """ + Add storage software patch stages to a strategy + """ + from nfv_vim import strategy + return self._add_update_storage_strategy_stages( + storage_hosts, + reboot, + strategy.STRATEGY_STAGE_NAME.SW_UPGRADE_STORAGE_HOSTS, + strategy.UpgradeHostsStep) + + class UpdateSystemConfigStorageHostsMixin(UpdateStorageHostsMixin): def _add_system_config_storage_strategy_stages(self, storage_hosts): """ @@ -1245,6 +1267,8 @@ class UpdateWorkerHostsMixin(object): # Migrate or stop instances as necessary if SW_UPDATE_INSTANCE_ACTION.MIGRATE == self._default_instance_action: + # TODO(jkraitbe): Should probably be: + # if len(openstack_hosts) and len(instance_list) if len(openstack_hosts): if SW_UPDATE_APPLY_TYPE.PARALLEL == self._worker_apply_type: # Disable host services before migrating to ensure @@ -1335,6 +1359,16 @@ class PatchWorkerHostsMixin(UpdateWorkerHostsMixin): strategy.SwPatchHostsStep) +class SwDeployWorkerHostsMixin(UpdateWorkerHostsMixin): + def _add_worker_strategy_stages(self, worker_hosts, reboot): + from nfv_vim import strategy + return self._add_update_worker_strategy_stages( + worker_hosts, + reboot, + strategy.STRATEGY_STAGE_NAME.SW_UPGRADE_WORKER_HOSTS, + strategy.UpgradeHostsStep) + + class UpgradeKubeletWorkerHostsMixin(UpdateWorkerHostsMixin): def _add_kubelet_worker_strategy_stages(self, worker_hosts, to_version, reboot, stage_name): from nfv_vim import strategy @@ -1716,41 +1750,34 @@ class SwPatchStrategy(SwUpdateStrategy, # The Software Upgrade Strategy # ################################################################### -class SwUpgradeStrategy(SwUpdateStrategy): +class SwUpgradeStrategy( + SwUpdateStrategy, + SwDeployControllerHostsMixin, + SwDeployStorageHostsMixin, + SwDeployWorkerHostsMixin, +): """ Software Upgrade - Strategy """ - def __init__(self, uuid, storage_apply_type, worker_apply_type, - max_parallel_worker_hosts, - alarm_restrictions, release, start_upgrade, complete_upgrade, + def __init__(self, uuid, + controller_apply_type, storage_apply_type, worker_apply_type, + max_parallel_worker_hosts, default_instance_action, + alarm_restrictions, release, ignore_alarms, single_controller): super(SwUpgradeStrategy, self).__init__( uuid, STRATEGY_NAME.SW_UPGRADE, - SW_UPDATE_APPLY_TYPE.SERIAL, + controller_apply_type, storage_apply_type, SW_UPDATE_APPLY_TYPE.IGNORE, worker_apply_type, max_parallel_worker_hosts, - SW_UPDATE_INSTANCE_ACTION.STOP_START, + default_instance_action, alarm_restrictions, ignore_alarms) - # Note: The support for start_upgrade was implemented and (mostly) - # tested, but there is a problem. When the sw-upgrade-start stage - # runs, it will start the upgrade, upgrade controller-1 and swact to - # it. However, when controller-1 becomes active, it will be using the - # snapshot of the VIM database that was created when the upgrade was - # started, so the strategy object created from the database will be - # long out of date (it thinks the upgrade start step is still in - # progress) and the strategy apply will fail. Fixing this would be - # complex, so we will not support the start_upgrade option for now, - # which would only have been for lab use. - if start_upgrade: - raise Exception("No support for start_upgrade") self._release = release - self._start_upgrade = start_upgrade - self._complete_upgrade = complete_upgrade + # The following alarms will not prevent a software upgrade operation IGNORE_ALARMS = ['900.005', # Upgrade in progress '900.201', # Software upgrade auto apply in progress @@ -1761,6 +1788,7 @@ class SwUpgradeStrategy(SwUpdateStrategy): self._ignore_alarms += IGNORE_ALARMS self._single_controller = single_controller self._nfvi_upgrade = None + self._ignore_alarms_conditional = None @property def nfvi_upgrade(self): @@ -1782,37 +1810,53 @@ class SwUpgradeStrategy(SwUpdateStrategy): """ from nfv_vim import strategy - stage = strategy.StrategyStage( - strategy.STRATEGY_STAGE_NAME.SW_UPGRADE_QUERY) - stage.add_step(strategy.QueryAlarmsStep( - ignore_alarms=self._ignore_alarms)) - stage.add_step(strategy.QueryUpgradeStep()) + stage = strategy.StrategyStage(strategy.STRATEGY_STAGE_NAME.SW_UPGRADE_QUERY) + stage.add_step(strategy.QueryAlarmsStep(ignore_alarms=self._ignore_alarms)) + stage.add_step(strategy.QueryUpgradeStep(release=self._release)) self.build_phase.add_stage(stage) + super(SwUpgradeStrategy, self).build() + def _swact_fix(self, stage, controller_name): + """Add a SWACT to a stage on DX systems + + Currently, certain steps during sw-deploy must be done on a specific controller. + Here we insert arbitrary SWACTs to meet those requirements. + + If controller_name does not match the current active host we return because + we are already on ideal host. + """ + + if self._single_controller or controller_name != get_local_host_name(): + return + + from nfv_vim import strategy + from nfv_vim import tables + + host_table = tables.tables_get_host_table() + for host in host_table.get_by_personality(HOST_PERSONALITY.CONTROLLER): + if controller_name == host.name: + stage.add_step(strategy.SwactHostsStep([host])) + break + def _add_upgrade_start_stage(self): """ Add upgrade start strategy stage """ from nfv_vim import strategy - from nfv_vim import tables - host_table = tables.tables_get_host_table() - controller_1_host = None - for host in host_table.get_by_personality(HOST_PERSONALITY.CONTROLLER): - if HOST_NAME.CONTROLLER_1 == host.name: - controller_1_host = host - break - host_list = [controller_1_host] + stage = strategy.StrategyStage(strategy.STRATEGY_STAGE_NAME.SW_UPGRADE_START) - stage = strategy.StrategyStage( - strategy.STRATEGY_STAGE_NAME.SW_UPGRADE_START) # Do not ignore any alarms when starting an upgrade - stage.add_step(strategy.QueryAlarmsStep(True)) - # Upgrade start can only be done from controller-0 - stage.add_step(strategy.SwactHostsStep(host_list)) - stage.add_step(strategy.UpgradeStartStep()) - stage.add_step(strategy.SystemStabilizeStep()) + stage.add_step(strategy.QueryAlarmsStep(True, ignore_alarms=self._ignore_alarms)) + if self.nfvi_upgrade.is_available: + # sw-deploy start must be done on controller-0 + self._swact_fix(stage, HOST_NAME.CONTROLLER_1) + stage.add_step(strategy.SwDeployPrecheckStep(release=self._release)) + stage.add_step(strategy.UpgradeStartStep(release=self._release)) + stage.add_step(strategy.SystemStabilizeStep()) + # sw-deploy host must first be on controller-1 + self._swact_fix(stage, HOST_NAME.CONTROLLER_0) self.apply_phase.add_stage(stage) def _add_upgrade_complete_stage(self): @@ -1820,287 +1864,23 @@ class SwUpgradeStrategy(SwUpdateStrategy): Add upgrade complete strategy stage """ from nfv_vim import strategy - from nfv_vim import tables - host_table = tables.tables_get_host_table() - controller_1_host = None - for host in host_table.get_by_personality(HOST_PERSONALITY.CONTROLLER): - if HOST_NAME.CONTROLLER_1 == host.name: - controller_1_host = host - break - host_list = [controller_1_host] - - stage = strategy.StrategyStage( - strategy.STRATEGY_STAGE_NAME.SW_UPGRADE_COMPLETE) - stage.add_step(strategy.QueryAlarmsStep( - True, ignore_alarms=self._ignore_alarms)) - # Upgrade complete can only be done from controller-0 - stage.add_step(strategy.SwactHostsStep(host_list)) - stage.add_step(strategy.UpgradeActivateStep()) - stage.add_step(strategy.UpgradeCompleteStep()) + stage = strategy.StrategyStage(strategy.STRATEGY_STAGE_NAME.SW_UPGRADE_COMPLETE) + stage.add_step(strategy.QueryAlarmsStep(ignore_alarms=self._ignore_alarms)) + # sw-deploy complete should be on controller-0 + self._swact_fix(stage, HOST_NAME.CONTROLLER_1) + if not self.nfvi_upgrade.is_activated: + stage.add_step(strategy.UpgradeActivateStep(release=self._release)) + stage.add_step(strategy.UpgradeCompleteStep(release=self._release)) stage.add_step(strategy.SystemStabilizeStep()) self.apply_phase.add_stage(stage) - def _add_controller_strategy_stages(self, controllers, reboot): - """ - Add controller software upgrade strategy stages - """ - from nfv_vim import strategy - from nfv_vim import tables - - host_table = tables.tables_get_host_table() - - if 2 > host_table.total_by_personality(HOST_PERSONALITY.CONTROLLER): - DLOG.warn("Not enough controllers to apply software upgrades.") - reason = 'not enough controllers to apply software upgrades' - return False, reason - - controller_0_host = None - controller_1_host = None - - for host in controllers: - if HOST_PERSONALITY.WORKER in host.personality: - # Do nothing for AIO hosts. We let the worker code handle everything. - # This is done to handle the case where stx-openstack is - # installed and there could be instances running on the - # AIO-DX controllers which need to be migrated. - if self._single_controller: - DLOG.warn("Cannot apply software upgrades to AIO-SX deployment.") - reason = 'cannot apply software upgrades to AIO-SX deployment' - return False, reason - else: - return True, '' - elif HOST_NAME.CONTROLLER_1 == host.name: - controller_1_host = host - elif HOST_NAME.CONTROLLER_0 == host.name: - controller_0_host = host - - if controller_1_host is not None: - host_list = [controller_1_host] - stage = strategy.StrategyStage( - strategy.STRATEGY_STAGE_NAME.SW_UPGRADE_CONTROLLERS) - stage.add_step(strategy.QueryAlarmsStep( - True, ignore_alarms=self._ignore_alarms)) - stage.add_step(strategy.LockHostsStep(host_list)) - stage.add_step(strategy.UpgradeHostsStep(host_list)) - # During an upgrade, unlock may need to retry. Bug details: - # https://bugs.launchpad.net/starlingx/+bug/1946255 - stage.add_step(strategy.UnlockHostsStep( - host_list, - retry_count=strategy.UnlockHostsStep.MAX_RETRIES)) - # Allow up to four hours for controller disks to synchronize - stage.add_step(strategy.WaitDataSyncStep( - timeout_in_secs=4 * 60 * 60, - ignore_alarms=self._ignore_alarms)) - self.apply_phase.add_stage(stage) - - if controller_0_host is not None: - host_list = [controller_0_host] - stage = strategy.StrategyStage( - strategy.STRATEGY_STAGE_NAME.SW_UPGRADE_CONTROLLERS) - stage.add_step(strategy.QueryAlarmsStep( - True, ignore_alarms=self._ignore_alarms)) - if controller_1_host is not None: - # Only swact to controller-1 if it was upgraded. If we are only - # upgrading controller-0, then controller-1 needs to be - # active already. - stage.add_step(strategy.SwactHostsStep(host_list)) - stage.add_step(strategy.LockHostsStep(host_list)) - stage.add_step(strategy.UpgradeHostsStep(host_list)) - # During an upgrade, unlock may need to retry. Bug details: - # https://bugs.launchpad.net/starlingx/+bug/1946255 - stage.add_step(strategy.UnlockHostsStep( - host_list, - retry_count=strategy.UnlockHostsStep.MAX_RETRIES)) - # Allow up to four hours for controller disks to synchronize - stage.add_step(strategy.WaitDataSyncStep( - timeout_in_secs=4 * 60 * 60, - ignore_alarms=self._ignore_alarms)) - self.apply_phase.add_stage(stage) - - return True, '' - - def _add_storage_strategy_stages(self, storage_hosts, reboot): - """ - Add storage software upgrade strategy stages - """ - from nfv_vim import strategy - - storage_0_host_list = list() - storage_0_host_lists = list() - other_storage_host_list = list() - - for host in storage_hosts: - if HOST_NAME.STORAGE_0 == host.name: - storage_0_host_list.append(host) - else: - other_storage_host_list.append(host) - - if len(storage_0_host_list) == 1: - storage_0_host_lists, reason = self._create_storage_host_lists( - storage_0_host_list) - if storage_0_host_lists is None: - return False, reason - - other_storage_host_lists, reason = self._create_storage_host_lists( - other_storage_host_list) - if other_storage_host_lists is None: - return False, reason - - # Upgrade storage-0 first and on its own since it has a ceph monitor - if len(storage_0_host_lists) == 1: - combined_host_lists = storage_0_host_lists + other_storage_host_lists - else: - combined_host_lists = other_storage_host_lists - - for host_list in combined_host_lists: - stage = strategy.StrategyStage( - strategy.STRATEGY_STAGE_NAME.SW_UPGRADE_STORAGE_HOSTS) - stage.add_step(strategy.QueryAlarmsStep( - True, ignore_alarms=self._ignore_alarms)) - stage.add_step(strategy.LockHostsStep(host_list)) - stage.add_step(strategy.UpgradeHostsStep(host_list)) - # During an upgrade, unlock may need to retry. Bug details: - # https://bugs.launchpad.net/starlingx/+bug/1946255 - stage.add_step(strategy.UnlockHostsStep( - host_list, - retry_count=strategy.UnlockHostsStep.MAX_RETRIES)) - - # After storage node(s) are unlocked, we need extra time to - # allow the OSDs to go back in sync and the storage related - # alarms to clear. We no longer wipe the OSD disks when upgrading - # a storage node, so they should only be syncing data that changed - # while they were being upgraded. - stage.add_step(strategy.WaitDataSyncStep( - timeout_in_secs=2 * 60 * 60, - ignore_alarms=self._ignore_alarms)) - self.apply_phase.add_stage(stage) - - return True, '' - - def _add_worker_strategy_stages(self, worker_hosts, reboot): - """ - Add worker software upgrade strategy stages - """ - from nfv_vim import strategy - from nfv_vim import tables - - host_lists, reason = self._create_worker_host_lists(worker_hosts, reboot) - if host_lists is None: - return False, reason - - instance_table = tables.tables_get_instance_table() - - for host_list in host_lists: - instance_list = list() - - for host in host_list: - for instance in instance_table.on_host(host.name): - if not instance.is_locked(): - instance_list.append(instance) - else: - DLOG.warn("Instance %s must not be shut down" % - instance.name) - reason = ('instance %s must not be shut down' % - instance.name) - return False, reason - - # Computes with no instances - if 0 == len(instance_list): - stage = strategy.StrategyStage( - strategy.STRATEGY_STAGE_NAME.SW_UPGRADE_WORKER_HOSTS) - stage.add_step(strategy.QueryAlarmsStep( - True, ignore_alarms=self._ignore_alarms)) - if HOST_PERSONALITY.CONTROLLER in host_list[0].personality: - stage.add_step(strategy.SwactHostsStep(host_list)) - stage.add_step(strategy.LockHostsStep(host_list)) - stage.add_step(strategy.UpgradeHostsStep(host_list)) - # During an upgrade, unlock may need to retry. Bug details: - # https://bugs.launchpad.net/starlingx/+bug/1914836 - stage.add_step(strategy.UnlockHostsStep( - host_list, - retry_count=strategy.UnlockHostsStep.MAX_RETRIES)) - if HOST_PERSONALITY.CONTROLLER in host_list[0].personality: - # AIO Controller hosts will undergo WaitDataSyncStep step - # Allow up to four hours for controller disks to synchronize - stage.add_step(strategy.WaitDataSyncStep( - timeout_in_secs=4 * 60 * 60, - ignore_alarms=self._ignore_alarms)) - else: - # Worker hosts will undergo: - # 1) WaitAlarmsClear step if openstack is installed. - # 2) SystemStabilizeStep step if openstack is not installed. - if any([host.openstack_control or host.openstack_compute - for host in host_list]): - # Hosts with openstack that just need to wait for services to start up: - stage.add_step(strategy.WaitAlarmsClearStep( - timeout_in_secs=10 * 60, - ignore_alarms=self._ignore_alarms)) - else: - stage.add_step(strategy.SystemStabilizeStep()) - self.apply_phase.add_stage(stage) - continue - - # Computes with instances - stage = strategy.StrategyStage( - strategy.STRATEGY_STAGE_NAME.SW_UPGRADE_WORKER_HOSTS) - - stage.add_step(strategy.QueryAlarmsStep( - True, ignore_alarms=self._ignore_alarms)) - - if SW_UPDATE_APPLY_TYPE.PARALLEL == self._worker_apply_type: - # Disable host services before migrating to ensure - # instances do not migrate to worker hosts in the - # same set of hosts. - if host_list[0].host_service_configured( - HOST_SERVICES.COMPUTE): - stage.add_step(strategy.DisableHostServicesStep( - host_list, HOST_SERVICES.COMPUTE)) - # TODO(ksmith) - # When support is added for orchestration on - # non-OpenStack worker nodes, support for disabling - # kubernetes services will have to be added. - - stage.add_step(strategy.MigrateInstancesStep(instance_list)) - if HOST_PERSONALITY.CONTROLLER in host_list[0].personality: - stage.add_step(strategy.SwactHostsStep(host_list)) - stage.add_step(strategy.LockHostsStep(host_list)) - stage.add_step(strategy.UpgradeHostsStep(host_list)) - # During an upgrade, unlock may need to retry. Bug details: - # https://bugs.launchpad.net/starlingx/+bug/1914836 - stage.add_step(strategy.UnlockHostsStep( - host_list, - retry_count=strategy.UnlockHostsStep.MAX_RETRIES)) - if HOST_PERSONALITY.CONTROLLER in host_list[0].personality: - # AIO Controller hosts will undergo WaitDataSyncStep step - # Allow up to four hours for controller disks to synchronize - stage.add_step(strategy.WaitDataSyncStep( - timeout_in_secs=4 * 60 * 60, - ignore_alarms=self._ignore_alarms)) - else: - # Worker hosts will undergo: - # 1) WaitAlarmsClear step if openstack is installed. - # 2) SystemStabilizeStep step if openstack is not installed. - if any([host.openstack_control or host.openstack_compute - for host in host_list]): - # Hosts with openstack that just need to wait for - # services to start up: - stage.add_step(strategy.WaitAlarmsClearStep( - timeout_in_secs=10 * 60, - ignore_alarms=self._ignore_alarms)) - else: - stage.add_step(strategy.SystemStabilizeStep()) - self.apply_phase.add_stage(stage) - - return True, '' - def build_complete(self, result, result_reason): """ Strategy Build Complete """ from nfv_vim import strategy from nfv_vim import tables - import re result, result_reason = \ super(SwUpgradeStrategy, self).build_complete(result, result_reason) @@ -2108,102 +1888,50 @@ class SwUpgradeStrategy(SwUpdateStrategy): DLOG.info("Build Complete Callback, result=%s, reason=%s." % (result, result_reason)) - # todo (vselvara): release validation needs to be replaced with validation - # based on results from 'software release' query api in the upcoming story. - release_type = re.compile(r"^[a-z]+[\-]+[\d*]+[\.]+[\d*]+[\.]+[\d*]+$") - if not release_type.match(self._release): - DLOG.warn("Invalid Software Release.") - self._state = strategy.STRATEGY_STATE.BUILD_FAILED - self.build_phase.result = strategy.STRATEGY_PHASE_RESULT.FAILED - self.build_phase.result_reason = 'invalid software release' - self.sw_update_obj.strategy_build_complete( - False, self.build_phase.result_reason) - self.save() - return - if result in [strategy.STRATEGY_RESULT.SUCCESS, strategy.STRATEGY_RESULT.DEGRADED]: - # Check whether the upgrade is in a valid state for orchestration - if self.nfvi_upgrade is None: - if not self._start_upgrade: - DLOG.warn("No upgrade in progress.") - self._state = strategy.STRATEGY_STATE.BUILD_FAILED - self.build_phase.result = strategy.STRATEGY_PHASE_RESULT.FAILED - self.build_phase.result_reason = 'no upgrade in progress' - self.sw_update_obj.strategy_build_complete( - False, self.build_phase.result_reason) - self.save() - return - else: - if self._start_upgrade: - valid_states = [UPGRADE_STATE.STARTED, - UPGRADE_STATE.DATA_MIGRATION_COMPLETE, - UPGRADE_STATE.UPGRADING_CONTROLLERS, - UPGRADE_STATE.UPGRADING_HOSTS] - else: - valid_states = [UPGRADE_STATE.UPGRADING_CONTROLLERS, - UPGRADE_STATE.UPGRADING_HOSTS] - - if self.nfvi_upgrade.state not in valid_states: - DLOG.warn("Invalid upgrade state for orchestration: %s." % - self.nfvi_upgrade.state) - self._state = strategy.STRATEGY_STATE.BUILD_FAILED - self.build_phase.result = strategy.STRATEGY_PHASE_RESULT.FAILED - self.build_phase.result_reason = ( - 'invalid upgrade state for orchestration: %s' % - self.nfvi_upgrade.state) - self.sw_update_obj.strategy_build_complete( - False, self.build_phase.result_reason) - self.save() - return - - # If controller-1 has been upgraded and we have yet to upgrade - # controller-0, then controller-1 must be active. - if UPGRADE_STATE.UPGRADING_CONTROLLERS == self.nfvi_upgrade.state: - if HOST_NAME.CONTROLLER_1 != get_local_host_name(): - DLOG.warn( - "Controller-1 must be active for orchestration to " - "upgrade controller-0.") - self._state = strategy.STRATEGY_STATE.BUILD_FAILED - self.build_phase.result = \ - strategy.STRATEGY_PHASE_RESULT.FAILED - self.build_phase.result_reason = ( - 'controller-1 must be active for orchestration to ' - 'upgrade controller-0') - self.sw_update_obj.strategy_build_complete( - False, self.build_phase.result_reason) - self.save() - return - - if self._nfvi_alarms: - DLOG.warn( - "Active alarms found, can't apply software upgrade.") - alarm_id_list = "" - for alarm_data in self._nfvi_alarms: - if alarm_id_list: - alarm_id_list += ', ' - alarm_id_list += alarm_data['alarm_id'] - DLOG.warn("... active alarms: %s" % alarm_id_list) + if self.nfvi_upgrade.release_info is None: + reason = "Software release does not exist." + DLOG.warn(reason) self._state = strategy.STRATEGY_STATE.BUILD_FAILED self.build_phase.result = strategy.STRATEGY_PHASE_RESULT.FAILED - self.build_phase.result_reason = 'active alarms present ; ' - self.build_phase.result_reason += alarm_id_list + self.build_phase.result_reason = reason + self.sw_update_obj.strategy_build_complete( + False, self.build_phase.result_reason) + self.save() + return + + if self.nfvi_upgrade.is_deployed or self.nfvi_upgrade.is_commited: + reason = "Software release is already deployed or committed." + DLOG.warn(reason) + self._state = strategy.STRATEGY_STATE.BUILD_FAILED + self.build_phase.result = \ + strategy.STRATEGY_PHASE_RESULT.FAILED + self.build_phase.result_reason = reason + self.sw_update_obj.strategy_build_complete( + False, self.build_phase.result_reason) + self.save() + return + + if self._nfvi_alarms: + DLOG.warn("Active alarms found, can't apply sw-deployment.") + self._state = strategy.STRATEGY_STATE.BUILD_FAILED + self.build_phase.result = strategy.STRATEGY_PHASE_RESULT.FAILED + self.build_phase.result_reason = 'active alarms present' self.sw_update_obj.strategy_build_complete( False, self.build_phase.result_reason) self.save() return host_table = tables.tables_get_host_table() + for host in list(host_table.values()): - # Only allow upgrade orchestration when all hosts are - # available. It is not safe to automate upgrade application - # when we do not have full redundancy. - if not (host.is_unlocked() and host.is_enabled() and - host.is_available()): + # All hosts must be unlock/enabled/available + if not (host.is_unlocked() and host.is_enabled() and host.is_available()): DLOG.warn( - "All %s hosts must be unlocked-enabled-available, " - "can't apply software upgrades." % host.personality) + "All hosts must be unlocked-enabled-available, " + "can't apply sw-deployment: %s" % host.name) self._state = strategy.STRATEGY_STATE.BUILD_FAILED self.build_phase.result = \ strategy.STRATEGY_PHASE_RESULT.FAILED @@ -2215,73 +1943,76 @@ class SwUpgradeStrategy(SwUpdateStrategy): self.save() return - controller_hosts = list() + reboot_required = self.nfvi_upgrade.reboot_required + controller_strategy = self._add_controller_strategy_stages + controllers_hosts = list() storage_hosts = list() worker_hosts = list() - if self.nfvi_upgrade is None: - # Start upgrade - self._add_upgrade_start_stage() + self._add_upgrade_start_stage() - # All hosts will be upgraded - for host in list(host_table.values()): + if not self.nfvi_upgrade.is_activated: + # TODO(jkraitbe): Exclude hosts that are already deployed. + # The hosts states are found in self.nfvi_upgrade.hosts_info. + # None means deployment hasn't started. + + # TODO(jkraitbe): SWACT to controller-1 should be + # here instead of in _add_upgrade_start_stage. + + for host in host_table.values(): if HOST_PERSONALITY.CONTROLLER in host.personality: - controller_hosts.append(host) + controllers_hosts.append(host) + if HOST_PERSONALITY.WORKER in host.personality: + # We need to use this strategy on AIO type + controller_strategy = self._add_worker_strategy_stages elif HOST_PERSONALITY.STORAGE in host.personality: storage_hosts.append(host) - if HOST_PERSONALITY.WORKER in host.personality: - worker_hosts.append(host) - else: - # Only hosts not yet upgraded will be upgraded - to_load = self.nfvi_upgrade.to_release - for host in list(host_table.values()): - if host.software_load == to_load: - # No need to upgrade this host - continue - - if HOST_PERSONALITY.CONTROLLER in host.personality: - controller_hosts.append(host) - - elif HOST_PERSONALITY.STORAGE in host.personality: - storage_hosts.append(host) - - if HOST_PERSONALITY.WORKER in host.personality: + elif HOST_PERSONALITY.WORKER in host.personality: worker_hosts.append(host) - STRATEGY_CREATION_COMMANDS = [ - (self._add_controller_strategy_stages, - controller_hosts, True), - (self._add_storage_strategy_stages, - storage_hosts, True), - (self._add_worker_strategy_stages, - worker_hosts, True) - ] - - for add_strategy_stages_function, host_list, reboot in \ - STRATEGY_CREATION_COMMANDS: - if host_list: - success, reason = add_strategy_stages_function( - host_list, reboot) - if not success: + else: + DLOG.error(f"Unsupported personality for host {host.name}.") self._state = strategy.STRATEGY_STATE.BUILD_FAILED self.build_phase.result = \ strategy.STRATEGY_PHASE_RESULT.FAILED - self.build_phase.result_reason = reason + self.build_phase.result_reason = \ + 'Unsupported personality for host' self.sw_update_obj.strategy_build_complete( False, self.build_phase.result_reason) self.save() return - if self._complete_upgrade: - self._add_upgrade_complete_stage() + # Reverse controller hosts so controller-1 is first + controllers_hosts = controllers_hosts[::-1] + + strategy_pairs = [ + (controller_strategy, controllers_hosts), + (self._add_storage_strategy_stages, storage_hosts), + (self._add_worker_strategy_stages, worker_hosts) + ] + + for stage_func, host_list in strategy_pairs: + if host_list: + success, reason = stage_func(host_list, reboot_required) + if not success: + self._state = strategy.STRATEGY_STATE.BUILD_FAILED + self.build_phase.result = \ + strategy.STRATEGY_PHASE_RESULT.FAILED + self.build_phase.result_reason = reason + self.sw_update_obj.strategy_build_complete( + False, self.build_phase.result_reason) + self.save() + return + + self._add_upgrade_complete_stage() if 0 == len(self.apply_phase.stages): - DLOG.warn("No software upgrades need to be applied.") + DLOG.warn("No sw-deployments need to be applied.") self._state = strategy.STRATEGY_STATE.BUILD_FAILED self.build_phase.result = strategy.STRATEGY_PHASE_RESULT.FAILED - self.build_phase.result_reason = ('no software upgrades need to be ' + self.build_phase.result_reason = ('no sw-deployments patches need to be ' 'applied') self.sw_update_obj.strategy_build_complete( False, self.build_phase.result_reason) @@ -2303,15 +2034,13 @@ class SwUpgradeStrategy(SwUpdateStrategy): super(SwUpgradeStrategy, self).from_dict(data, build_phase, apply_phase, abort_phase) self._single_controller = data['single_controller'] - self._start_upgrade = data['start_upgrade'] - self._complete_upgrade = data['complete_upgrade'] self._release = data['release'] nfvi_upgrade_data = data['nfvi_upgrade_data'] if nfvi_upgrade_data: self._nfvi_upgrade = nfvi.objects.v1.Upgrade( - nfvi_upgrade_data['state'], - nfvi_upgrade_data['from_release'], - nfvi_upgrade_data['to_release']) + nfvi_upgrade_data['release'], + nfvi_upgrade_data['release_info'], + nfvi_upgrade_data['hosts_info']) else: self._nfvi_upgrade = None @@ -2323,8 +2052,6 @@ class SwUpgradeStrategy(SwUpdateStrategy): """ data = super(SwUpgradeStrategy, self).as_dict() data['single_controller'] = self._single_controller - data['start_upgrade'] = self._start_upgrade - data['complete_upgrade'] = self._complete_upgrade data['release'] = self._release if self._nfvi_upgrade: nfvi_upgrade_data = self._nfvi_upgrade.as_dict() diff --git a/nfv/nfv-vim/nfv_vim/strategy/_strategy_steps.py b/nfv/nfv-vim/nfv_vim/strategy/_strategy_steps.py index 9a54a90e..a2446bd6 100755 --- a/nfv/nfv-vim/nfv_vim/strategy/_strategy_steps.py +++ b/nfv/nfv-vim/nfv_vim/strategy/_strategy_steps.py @@ -36,6 +36,7 @@ class StrategyStepNames(Constants): UNLOCK_HOSTS = Constant('unlock-hosts') REBOOT_HOSTS = Constant('reboot-hosts') UPGRADE_HOSTS = Constant('upgrade-hosts') + SW_DEPLOY_PRECHECK = Constant('sw-deploy-precheck') START_UPGRADE = Constant('start-upgrade') ACTIVATE_UPGRADE = Constant('activate-upgrade') COMPLETE_UPGRADE = Constant('complete-upgrade') @@ -54,7 +55,6 @@ class StrategyStepNames(Constants): QUERY_SW_PATCH_HOSTS = Constant('query-sw-patch-hosts') QUERY_FW_UPDATE_HOST = Constant('query-fw-update-host') QUERY_UPGRADE = Constant('query-upgrade') - QUERY_SOFTWARE_RELEASE = Constant('query-software-release') DISABLE_HOST_SERVICES = Constant('disable-host-services') ENABLE_HOST_SERVICES = Constant('enable-host-services') # kube rootca update steps @@ -935,6 +935,8 @@ class UpgradeHostsStep(strategy.StrategyStep): """ Returns the number of hosts that are upgraded """ + + # TODO(jkraitbe): Use deploy/host_list instead total_hosts_upgraded = 0 host_table = tables.tables_get_host_table() for host_name in self._host_names: @@ -943,8 +945,8 @@ class UpgradeHostsStep(strategy.StrategyStep): return -1 if (host.is_online() and - host.target_load == self.strategy.nfvi_upgrade.to_release and - host.software_load == self.strategy.nfvi_upgrade.to_release): + host.target_load == self.strategy.nfvi_upgrade.release and + host.software_load == self.strategy.nfvi_upgrade.release): total_hosts_upgraded += 1 return total_hosts_upgraded @@ -1031,16 +1033,72 @@ class UpgradeHostsStep(strategy.StrategyStep): return data +class SwDeployPrecheckStep(strategy.StrategyStep): + """ + Software Deploy Precheck - Strategy Step + """ + def __init__(self, release): + super(SwDeployPrecheckStep, self).__init__( + STRATEGY_STEP_NAME.SW_DEPLOY_PRECHECK, timeout_in_secs=600) + + self._release = release + + @coroutine + def _sw_deploy_precheck_callback(self): + """ + Software deploy precheck callback + """ + response = (yield) + DLOG.debug("sw-deploy precheck callback response=%s." % response) + + if response['completed']: + result = strategy.STRATEGY_STEP_RESULT.SUCCESS + self.stage.step_complete(result, "") + else: + # TODO(jkraitbe): Add error message + result = strategy.STRATEGY_STEP_RESULT.FAILED + self.stage.step_complete(result, "") + + def apply(self): + """ + Software deploy precheck + """ + from nfv_vim import nfvi + + DLOG.info("Step (%s) apply." % self._name) + nfvi.nfvi_sw_deploy_precheck(self._release, self._sw_deploy_precheck_callback()) + return strategy.STRATEGY_STEP_RESULT.WAIT, "" + + def from_dict(self, data): + """ + Returns the sw-deploy precheck step object initialized using the given + dictionary + """ + super(SwDeployPrecheckStep, self).from_dict(data) + self._release = data["release"] + return self + + def as_dict(self): + """ + Represent the sw-deploy precheck step as a dictionary + """ + data = super(SwDeployPrecheckStep, self).as_dict() + data['release'] = self._release + data['entity_type'] = '' + data['entity_names'] = list() + data['entity_uuids'] = list() + return data + + class UpgradeStartStep(strategy.StrategyStep): """ Upgrade Start - Strategy Step """ - def __init__(self): + def __init__(self, release): super(UpgradeStartStep, self).__init__( STRATEGY_STEP_NAME.START_UPGRADE, timeout_in_secs=600) - self._wait_time = 0 - self._query_inprogress = False + self._release = release @coroutine def _start_upgrade_callback(self): @@ -1051,37 +1109,11 @@ class UpgradeStartStep(strategy.StrategyStep): DLOG.debug("Start-Upgrade callback response=%s." % response) if response['completed']: - if self.strategy is not None: - self.strategy.nfvi_upgrade = response['result-data'] - else: - result = strategy.STRATEGY_STEP_RESULT.FAILED + # TODO(jkraitbe): Consider updating self.strategy.nfvi_upgrade.hosts_info + result = strategy.STRATEGY_STEP_RESULT.SUCCESS self.stage.step_complete(result, "") - - @coroutine - def _get_upgrade_callback(self): - """ - Get Upgrade Callback - """ - from nfv_vim import nfvi - - response = (yield) - DLOG.debug("Get-Upgrade callback response=%s." % response) - - self._query_inprogress = False - - if response['completed']: - if self.strategy is not None: - self.strategy.nfvi_upgrade = response['result-data'] - - if self.strategy.nfvi_upgrade.state != \ - nfvi.objects.v1.UPGRADE_STATE.STARTED: - # Keep waiting for upgrade to start - pass - else: - # Upgrade has started - result = strategy.STRATEGY_STEP_RESULT.SUCCESS - self.stage.step_complete(result, "") else: + # TODO(jkraitbe): Add error message result = strategy.STRATEGY_STEP_RESULT.FAILED self.stage.step_complete(result, "") @@ -1092,39 +1124,16 @@ class UpgradeStartStep(strategy.StrategyStep): from nfv_vim import nfvi DLOG.info("Step (%s) apply." % self._name) - nfvi.nfvi_upgrade_start(self._start_upgrade_callback()) + nfvi.nfvi_upgrade_start(self._release, self._start_upgrade_callback()) return strategy.STRATEGY_STEP_RESULT.WAIT, "" - def handle_event(self, event, event_data=None): - """ - Handle Host events - """ - from nfv_vim import nfvi - - DLOG.debug("Step (%s) handle event (%s)." % (self._name, event)) - - if event == STRATEGY_EVENT.HOST_AUDIT: - if 0 == self._wait_time: - self._wait_time = timers.get_monotonic_timestamp_in_ms() - - now_ms = timers.get_monotonic_timestamp_in_ms() - secs_expired = (now_ms - self._wait_time) // 1000 - # Wait at least 60 seconds before checking upgrade for first time - if 60 <= secs_expired and not self._query_inprogress: - self._query_inprogress = True - nfvi.nfvi_get_upgrade(self._get_upgrade_callback()) - return True - - return False - def from_dict(self, data): """ Returns the upgrade start step object initialized using the given dictionary """ super(UpgradeStartStep, self).from_dict(data) - self._wait_time = 0 - self._query_inprogress = False + self._release = data["release"] return self def as_dict(self): @@ -1135,6 +1144,7 @@ class UpgradeStartStep(strategy.StrategyStep): data['entity_type'] = '' data['entity_names'] = list() data['entity_uuids'] = list() + data['release'] = self._release return data @@ -1143,12 +1153,11 @@ class UpgradeActivateStep(strategy.StrategyStep): Upgrade Activate - Strategy Step """ - def __init__(self): + def __init__(self, release): super(UpgradeActivateStep, self).__init__( STRATEGY_STEP_NAME.ACTIVATE_UPGRADE, timeout_in_secs=900) - self._wait_time = 0 - self._query_inprogress = False + self._release = release @coroutine def _activate_upgrade_callback(self): @@ -1159,37 +1168,10 @@ class UpgradeActivateStep(strategy.StrategyStep): DLOG.debug("Activate-Upgrade callback response=%s." % response) if response['completed']: - if self.strategy is not None: - self.strategy.nfvi_upgrade = response['result-data'] - else: - result = strategy.STRATEGY_STEP_RESULT.FAILED + result = strategy.STRATEGY_STEP_RESULT.SUCCESS self.stage.step_complete(result, "") - - @coroutine - def _get_upgrade_callback(self): - """ - Get Upgrade Callback - """ - from nfv_vim import nfvi - - response = (yield) - DLOG.debug("Get-Upgrade callback response=%s." % response) - - self._query_inprogress = False - - if response['completed']: - if self.strategy is not None: - self.strategy.nfvi_upgrade = response['result-data'] - - if self.strategy.nfvi_upgrade.state != \ - nfvi.objects.v1.UPGRADE_STATE.ACTIVATION_COMPLETE: - # Keep waiting for upgrade to activate - pass - else: - # Upgrade has activated - result = strategy.STRATEGY_STEP_RESULT.SUCCESS - self.stage.step_complete(result, "") else: + # TODO(jkraitbe): Add error message result = strategy.STRATEGY_STEP_RESULT.FAILED self.stage.step_complete(result, "") @@ -1200,39 +1182,16 @@ class UpgradeActivateStep(strategy.StrategyStep): from nfv_vim import nfvi DLOG.info("Step (%s) apply." % self._name) - nfvi.nfvi_upgrade_activate(self._activate_upgrade_callback()) + nfvi.nfvi_upgrade_activate(self._release, self._activate_upgrade_callback()) return strategy.STRATEGY_STEP_RESULT.WAIT, "" - def handle_event(self, event, event_data=None): - """ - Handle Host events - """ - from nfv_vim import nfvi - - DLOG.debug("Step (%s) handle event (%s)." % (self._name, event)) - - if event == STRATEGY_EVENT.HOST_AUDIT: - if 0 == self._wait_time: - self._wait_time = timers.get_monotonic_timestamp_in_ms() - - now_ms = timers.get_monotonic_timestamp_in_ms() - secs_expired = (now_ms - self._wait_time) // 1000 - # Wait at least 60 seconds before checking upgrade for first time - if 60 <= secs_expired and not self._query_inprogress: - self._query_inprogress = True - nfvi.nfvi_get_upgrade(self._get_upgrade_callback()) - return True - - return False - def from_dict(self, data): """ Returns the upgrade activate step object initialized using the given dictionary """ super(UpgradeActivateStep, self).from_dict(data) - self._wait_time = 0 - self._query_inprogress = False + self._release = data["release"] return self def as_dict(self): @@ -1243,6 +1202,7 @@ class UpgradeActivateStep(strategy.StrategyStep): data['entity_type'] = '' data['entity_names'] = list() data['entity_uuids'] = list() + data['release'] = self._release return data @@ -1251,12 +1211,11 @@ class UpgradeCompleteStep(strategy.StrategyStep): Upgrade Complete - Strategy Step """ - def __init__(self): + def __init__(self, release): super(UpgradeCompleteStep, self).__init__( STRATEGY_STEP_NAME.COMPLETE_UPGRADE, timeout_in_secs=300) - self._wait_time = 0 - self._query_inprogress = False + self._release = release @coroutine def _complete_upgrade_callback(self): @@ -1267,34 +1226,10 @@ class UpgradeCompleteStep(strategy.StrategyStep): DLOG.debug("Complete-Upgrade callback response=%s." % response) if response['completed']: - if self.strategy is not None: - self.strategy.nfvi_upgrade = response['result-data'] - else: - result = strategy.STRATEGY_STEP_RESULT.FAILED + result = strategy.STRATEGY_STEP_RESULT.SUCCESS self.stage.step_complete(result, "") - - @coroutine - def _get_upgrade_callback(self): - """ - Get Upgrade Callback - """ - response = (yield) - DLOG.debug("Get-Upgrade callback response=%s." % response) - - self._query_inprogress = False - - if response['completed']: - if self.strategy is not None: - self.strategy.nfvi_upgrade = response['result-data'] - - if self.strategy.nfvi_upgrade is not None: - # Keep waiting for upgrade to complete - pass - else: - # Upgrade has been completed (upgrade record deleted) - result = strategy.STRATEGY_STEP_RESULT.SUCCESS - self.stage.step_complete(result, "") else: + # TODO(jkraitbe): Add error message result = strategy.STRATEGY_STEP_RESULT.FAILED self.stage.step_complete(result, "") @@ -1305,39 +1240,16 @@ class UpgradeCompleteStep(strategy.StrategyStep): from nfv_vim import nfvi DLOG.info("Step (%s) apply." % self._name) - nfvi.nfvi_upgrade_complete(self._complete_upgrade_callback()) + nfvi.nfvi_upgrade_complete(self._release, self._complete_upgrade_callback()) return strategy.STRATEGY_STEP_RESULT.WAIT, "" - def handle_event(self, event, event_data=None): - """ - Handle Host events - """ - from nfv_vim import nfvi - - DLOG.debug("Step (%s) handle event (%s)." % (self._name, event)) - - if event == STRATEGY_EVENT.HOST_AUDIT: - if 0 == self._wait_time: - self._wait_time = timers.get_monotonic_timestamp_in_ms() - - now_ms = timers.get_monotonic_timestamp_in_ms() - secs_expired = (now_ms - self._wait_time) // 1000 - # Wait at least 60 seconds before checking upgrade for first time - if 60 <= secs_expired and not self._query_inprogress: - self._query_inprogress = True - nfvi.nfvi_get_upgrade(self._get_upgrade_callback()) - return True - - return False - def from_dict(self, data): """ Returns the upgrade complete step object initialized using the given dictionary """ super(UpgradeCompleteStep, self).from_dict(data) - self._wait_time = 0 - self._query_inprogress = False + self._release = data["release"] return self def as_dict(self): @@ -1348,6 +1260,7 @@ class UpgradeCompleteStep(strategy.StrategyStep): data['entity_type'] = '' data['entity_names'] = list() data['entity_uuids'] = list() + data['release'] = self._release return data @@ -2800,15 +2713,18 @@ class QueryUpgradeStep(strategy.StrategyStep): """ Query Upgrade - Strategy Step """ - def __init__(self): + def __init__(self, release): super(QueryUpgradeStep, self).__init__( STRATEGY_STEP_NAME.QUERY_UPGRADE, timeout_in_secs=60) + self._release = release + @coroutine def _get_upgrade_callback(self): """ Get Upgrade Callback """ + response = (yield) DLOG.debug("Query-Upgrade callback response=%s." % response) @@ -2829,7 +2745,7 @@ class QueryUpgradeStep(strategy.StrategyStep): from nfv_vim import nfvi DLOG.info("Step (%s) apply." % self._name) - nfvi.nfvi_get_upgrade(self._get_upgrade_callback()) + nfvi.nfvi_get_upgrade(self._release, self._get_upgrade_callback()) return strategy.STRATEGY_STEP_RESULT.WAIT, "" def as_dict(self): @@ -2840,6 +2756,7 @@ class QueryUpgradeStep(strategy.StrategyStep): data['entity_type'] = '' data['entity_names'] = list() data['entity_uuids'] = list() + data['release'] = None return data @@ -5010,6 +4927,9 @@ def strategy_step_rebuild_from_dict(data): elif STRATEGY_STEP_NAME.UPGRADE_HOSTS == data['name']: step_obj = object.__new__(UpgradeHostsStep) + elif STRATEGY_STEP_NAME.SW_DEPLOY_PRECHECK == data['name']: + step_obj = object.__new__(SwDeployPrecheckStep) + elif STRATEGY_STEP_NAME.START_UPGRADE == data['name']: step_obj = object.__new__(UpgradeStartStep) diff --git a/nfv/tox.ini b/nfv/tox.ini index cf0b6123..c21dcb7d 100644 --- a/nfv/tox.ini +++ b/nfv/tox.ini @@ -1,5 +1,5 @@ # -# Copyright (c) 2018-2023 Wind River Systems, Inc. +# Copyright (c) 2018-2024 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -38,11 +38,11 @@ nfv_plugins_src_dir = {[nfv]nfv_plugins_dir}/nfv_plugins nfv_vim_src_dir = {[nfv]nfv_vim_dir}/nfv_vim nfv_test_src_dir = {[nfv]nfv_test_dir} -deps = {[nfv]nfv_client_dir} - {[nfv]nfv_common_dir} - {[nfv]nfv_plugins_dir} - {[nfv]nfv_vim_dir} - {[nfv]stx_fault_dir}/fm-api/source +deps = -e{[nfv]nfv_client_dir} + -e{[nfv]nfv_common_dir} + -e{[nfv]nfv_plugins_dir} + -e{[nfv]nfv_vim_dir} + -e{[nfv]stx_fault_dir}/fm-api/source iso8601 keyring kombu