From b90eeb143637f5f7ddc2ae99cd4a7487bfde0b70 Mon Sep 17 00:00:00 2001 From: Luiz Felipe Kina Date: Tue, 25 Jul 2023 07:40:14 -0400 Subject: [PATCH] Nfv upgrade orchestration for kube-upgrade-storage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This adds the kube-upgrade-storage step on the orchestration for k8s upgrade. VIM build stages stay the same, but the VIM apply stage changes. After the networking upgrade, there is an addition of storage upgrade and after that everything stays the same. Test Plan: PASS: Run a kubernetes upgrade with the kube-upgrade-storage step and observe that the image for volume-snapshot-controller is changed PASS: Run the kube-upgrade-storage with an unexpected state and expect failure. Story: 2010877 Task: 48588 Change-Id: Ib5ff848ed67c3e57c9cfcf6d1a41abbc192a7935 Signed-off-by: Luiz Felipe Kina Signed-off-by: Gabriel de Araújo Cabral --- .../nfvi_plugins/nfvi_infrastructure_api.py | 58 +++++++++++++ .../nfvi_plugins/openstack/sysinv.py | 7 ++ .../tests/test_kube_upgrade_strategy.py | 81 ++++++++++++++++++- nfv/nfv-vim/nfv_vim/nfvi/__init__.py | 1 + .../nfvi/_nfvi_infrastructure_module.py | 9 +++ .../nfv_vim/nfvi/objects/v1/_kube_upgrade.py | 3 + nfv/nfv-vim/nfv_vim/strategy/__init__.py | 1 + nfv/nfv-vim/nfv_vim/strategy/_strategy.py | 26 +++++- .../nfv_vim/strategy/_strategy_stages.py | 1 + .../nfv_vim/strategy/_strategy_steps.py | 36 +++++++++ 10 files changed, 221 insertions(+), 2 deletions(-) diff --git a/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_infrastructure_api.py b/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_infrastructure_api.py index 2ba51e8c..0c7ac08b 100755 --- a/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_infrastructure_api.py +++ b/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_infrastructure_api.py @@ -2074,6 +2074,64 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI): callback.send(response) callback.close() + def kube_upgrade_storage(self, future, callback): + """ + Start kube upgrade storage + """ + response = dict() + response['completed'] = False + response['reason'] = '' + action_type = 'kube-upgrade-storage' + + try: + future.set_timeouts(config.CONF.get('nfvi-timeouts', None)) + + if self._platform_token is None or \ + self._platform_token.is_expired(): + future.work(openstack.get_token, self._platform_directory) + future.result = (yield) + + if not future.result.is_complete() or \ + future.result.data is None: + DLOG.error("OpenStack get-token did not complete.") + return + + self._platform_token = future.result.data + + future.work(sysinv.kube_upgrade_storage, self._platform_token) + future.result = (yield) + + if not future.result.is_complete(): + DLOG.error("%s did not complete." % action_type) + return + + kube_upgrade_data = future.result.data + kube_upgrade_obj = nfvi.objects.v1.KubeUpgrade( + kube_upgrade_data['state'], + kube_upgrade_data['from_version'], + kube_upgrade_data['to_version']) + + response['result-data'] = kube_upgrade_obj + response['completed'] = True + + except exceptions.OpenStackRestAPIException as e: + if httplib.UNAUTHORIZED == e.http_status_code: + response['error-code'] = nfvi.NFVI_ERROR_CODE.TOKEN_EXPIRED + if self._platform_token is not None: + self._platform_token.set_expired() + + else: + DLOG.exception("Caught API exception while trying %s. error=%s" + % (action_type, e)) + + except Exception as e: + DLOG.exception("Caught exception while trying %s. error=%s" + % (action_type, e)) + + finally: + callback.send(response) + callback.close() + def kube_upgrade_start(self, future, to_version, force, alarm_ignore_list, callback): """ diff --git a/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/sysinv.py b/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/sysinv.py index 369f4a7d..5fb4baff 100755 --- a/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/sysinv.py +++ b/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/sysinv.py @@ -443,6 +443,13 @@ def kube_upgrade_networking(token): return _patch_kube_upgrade_state(token, "upgrading-networking") +def kube_upgrade_storage(token): + """ + Ask System Inventory to kube upgrade storage + """ + return _patch_kube_upgrade_state(token, "upgrading-storage") + + def kube_host_cordon(token, hostname, force): """ system kube-host-cordon diff --git a/nfv/nfv-tests/nfv_unit_tests/tests/test_kube_upgrade_strategy.py b/nfv/nfv-tests/nfv_unit_tests/tests/test_kube_upgrade_strategy.py index c1b7c7e2..2ae59587 100755 --- a/nfv/nfv-tests/nfv_unit_tests/tests/test_kube_upgrade_strategy.py +++ b/nfv/nfv-tests/nfv_unit_tests/tests/test_kube_upgrade_strategy.py @@ -340,6 +340,17 @@ class ApplyStageMixin(object): ], } + def _kube_upgrade_storage_stage(self): + return { + 'name': 'kube-upgrade-storage', + 'total_steps': 1, + 'steps': [ + {'name': 'kube-upgrade-storage', + 'success_state': 'upgraded-storage', + 'fail_state': 'upgrading-storage-failed'}, + ], + } + def _kube_upgrade_complete_stage(self): return { 'name': 'kube-upgrade-complete', @@ -485,6 +496,7 @@ class ApplyStageMixin(object): add_start=True, add_download=True, add_networking=True, + add_storage=True, add_cordon=True, add_first_control_plane=True, add_second_control_plane=True, @@ -507,6 +519,8 @@ class ApplyStageMixin(object): stages.append(self._kube_upgrade_download_images_stage()) if add_networking: stages.append(self._kube_upgrade_networking_stage()) + if add_storage: + stages.append(self._kube_upgrade_storage_stage()) if add_cordon: stages.append(self._kube_host_cordon_stage()) for ver in self.kube_versions: @@ -622,6 +636,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase, stages = [ self._kube_upgrade_download_images_stage(), self._kube_upgrade_networking_stage(), + self._kube_upgrade_storage_stage(), ] if self.is_simplex(): stages.append(self._kube_host_cordon_stage()) @@ -652,6 +667,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase, self.default_to_version) stages = [ self._kube_upgrade_networking_stage(), + self._kube_upgrade_storage_stage(), ] if self.is_simplex(): stages.append(self._kube_host_cordon_stage()) @@ -735,6 +751,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase, self.default_to_version) stages = [ self._kube_upgrade_networking_stage(), + self._kube_upgrade_storage_stage() ] if self.is_simplex(): stages.append(self._kube_host_cordon_stage()) @@ -758,12 +775,74 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase, """ Test the kube_upgrade strategy creation when there is only a simplex and the upgrade had previously stopped after successful networking. - It is expected to resume at the first control plane + It is expected to resume at the storage stage """ kube_upgrade = self._create_kube_upgrade_obj( KUBE_UPGRADE_STATE.KUBE_UPGRADED_NETWORKING, self.default_from_version, self.default_to_version) + stages = [ + self._kube_upgrade_storage_stage() + ] + if self.is_simplex(): + stages.append(self._kube_host_cordon_stage()) + for ver in self.kube_versions: + stages.append(self._kube_upgrade_first_control_plane_stage( + ver)) + stages.extend(self._kube_upgrade_kubelet_stages( + ver, + self.std_controller_list, + self.aio_controller_list, + self.worker_list)) + if self.is_simplex(): + stages.append(self._kube_host_uncordon_stage()) + stages.extend([ + self._kube_upgrade_complete_stage(), + self._kube_upgrade_cleanup_stage(), + ]) + self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages) + + def test_resume_after_storage_failed(self): + """ + Test the kube_upgrade strategy creation when there is only a simplex + and the upgrade had previously failed during storage. + It is expected to retry and resume at the storage stage + """ + kube_upgrade = self._create_kube_upgrade_obj( + KUBE_UPGRADE_STATE.KUBE_UPGRADING_STORAGE_FAILED, + self.default_from_version, + self.default_to_version) + stages = [ + self._kube_upgrade_storage_stage(), + ] + if self.is_simplex(): + stages.append(self._kube_host_cordon_stage()) + for ver in self.kube_versions: + stages.append(self._kube_upgrade_first_control_plane_stage( + ver)) + stages.extend(self._kube_upgrade_kubelet_stages( + ver, + self.std_controller_list, + self.aio_controller_list, + self.worker_list)) + if self.is_simplex(): + stages.append(self._kube_host_uncordon_stage()) + stages.extend([ + self._kube_upgrade_complete_stage(), + self._kube_upgrade_cleanup_stage(), + ]) + self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages) + + def test_resume_after_storage_succeeded(self): + """ + Test the kube_upgrade strategy creation when there is only a simplex + and the upgrade had previously stopped after successful storage. + It is expected to resume at the first control plane + """ + kube_upgrade = self._create_kube_upgrade_obj( + KUBE_UPGRADE_STATE.KUBE_UPGRADED_STORAGE, + self.default_from_version, + self.default_to_version) stages = [] if self.is_simplex(): stages.append(self._kube_host_cordon_stage()) diff --git a/nfv/nfv-vim/nfv_vim/nfvi/__init__.py b/nfv/nfv-vim/nfv_vim/nfvi/__init__.py index 1171a520..859edea7 100755 --- a/nfv/nfv-vim/nfv_vim/nfvi/__init__.py +++ b/nfv/nfv-vim/nfv_vim/nfvi/__init__.py @@ -130,6 +130,7 @@ from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_upgrade_complete from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_upgrade_download_images # noqa: F401 from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_upgrade_networking # noqa: F401 from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_upgrade_start # noqa: F401 +from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_upgrade_storage # noqa: F401 from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_list_deployment_hosts # noqa: F401 from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_lock_host # noqa: F401 from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_notify_host_failed # noqa: F401 diff --git a/nfv/nfv-vim/nfv_vim/nfvi/_nfvi_infrastructure_module.py b/nfv/nfv-vim/nfv_vim/nfvi/_nfvi_infrastructure_module.py index ed5bd6d4..466b7b1c 100755 --- a/nfv/nfv-vim/nfv_vim/nfvi/_nfvi_infrastructure_module.py +++ b/nfv/nfv-vim/nfv_vim/nfvi/_nfvi_infrastructure_module.py @@ -299,6 +299,15 @@ def nfvi_kube_upgrade_networking(callback): return cmd_id +def nfvi_kube_upgrade_storage(callback): + """ + Kube Upgrade Storage + """ + cmd_id = _infrastructure_plugin.invoke_plugin('kube_upgrade_storage', + callback=callback) + return cmd_id + + def nfvi_kube_upgrade_start(to_version, force, alarm_ignore_list, callback): """ Kube Upgrade Start diff --git a/nfv/nfv-vim/nfv_vim/nfvi/objects/v1/_kube_upgrade.py b/nfv/nfv-vim/nfv_vim/nfvi/objects/v1/_kube_upgrade.py index 6d736010..328a08b2 100755 --- a/nfv/nfv-vim/nfv_vim/nfvi/objects/v1/_kube_upgrade.py +++ b/nfv/nfv-vim/nfv_vim/nfvi/objects/v1/_kube_upgrade.py @@ -44,6 +44,9 @@ class KubeUpgradeState(Constants): KUBE_UPGRADING_NETWORKING = Constant('upgrading-networking') KUBE_UPGRADING_NETWORKING_FAILED = Constant('upgrading-networking-failed') KUBE_UPGRADED_NETWORKING = Constant('upgraded-networking') + KUBE_UPGRADING_STORAGE = Constant('upgrading-storage') + KUBE_UPGRADING_STORAGE_FAILED = Constant('upgrading-storage-failed') + KUBE_UPGRADED_STORAGE = Constant('upgraded-storage') KUBE_UPGRADING_FIRST_MASTER = Constant('upgrading-first-master') KUBE_UPGRADING_FIRST_MASTER_FAILED = Constant('upgrading-first-master-failed') KUBE_UPGRADED_FIRST_MASTER = Constant('upgraded-first-master') diff --git a/nfv/nfv-vim/nfv_vim/strategy/__init__.py b/nfv/nfv-vim/nfv_vim/strategy/__init__.py index 21e5af77..50fa177f 100755 --- a/nfv/nfv-vim/nfv_vim/strategy/__init__.py +++ b/nfv/nfv-vim/nfv_vim/strategy/__init__.py @@ -36,6 +36,7 @@ from nfv_vim.strategy._strategy_steps import KubeUpgradeCompleteStep # noqa: F4 from nfv_vim.strategy._strategy_steps import KubeUpgradeDownloadImagesStep # noqa: F401 from nfv_vim.strategy._strategy_steps import KubeUpgradeNetworkingStep # noqa: F401 from nfv_vim.strategy._strategy_steps import KubeUpgradeStartStep # noqa: F401 +from nfv_vim.strategy._strategy_steps import KubeUpgradeStorageStep # noqa: F401 from nfv_vim.strategy._strategy_steps import LockHostsStep # noqa: F401 from nfv_vim.strategy._strategy_steps import MigrateInstancesFromHostStep # noqa: F401 from nfv_vim.strategy._strategy_steps import MigrateInstancesStep # noqa: F401 diff --git a/nfv/nfv-vim/nfv_vim/strategy/_strategy.py b/nfv/nfv-vim/nfv_vim/strategy/_strategy.py index b10e8f2d..ec81a475 100755 --- a/nfv/nfv-vim/nfv_vim/strategy/_strategy.py +++ b/nfv/nfv-vim/nfv_vim/strategy/_strategy.py @@ -3458,6 +3458,22 @@ class KubeUpgradeStrategy(SwUpdateStrategy, stage.add_step(strategy.KubeUpgradeNetworkingStep()) self.apply_phase.add_stage(stage) + # Next stage after networking is upgrade storage + self._add_kube_upgrade_storage_stage() + + def _add_kube_upgrade_storage_stage(self): + """ + Add kube upgrade storage stage. + This stage only occurs after upgrade networking + It then proceeds to the next stage + """ + from nfv_vim import strategy + + stage = strategy.StrategyStage( + strategy.STRATEGY_STAGE_NAME.KUBE_UPGRADE_STORAGE) + stage.add_step(strategy.KubeUpgradeStorageStep()) + self.apply_phase.add_stage(stage) + # Next stage after networking is cordon self._add_kube_host_cordon_stage() @@ -3794,8 +3810,16 @@ class KubeUpgradeStrategy(SwUpdateStrategy, nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADING_NETWORKING_FAILED: self._add_kube_upgrade_networking_stage, - # After networking -> cordon + # After networking -> upgrade storage nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADED_NETWORKING: + self._add_kube_upgrade_storage_stage, + + # if storage state failed, resync at storage state + nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADING_STORAGE_FAILED: + self._add_kube_upgrade_storage_stage, + + # After storage -> cordon + nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADED_STORAGE: self._add_kube_host_cordon_stage, # If the state is cordon-failed, resume at cordon stage diff --git a/nfv/nfv-vim/nfv_vim/strategy/_strategy_stages.py b/nfv/nfv-vim/nfv_vim/strategy/_strategy_stages.py index a061ae90..dd02a0d9 100755 --- a/nfv/nfv-vim/nfv_vim/strategy/_strategy_stages.py +++ b/nfv/nfv-vim/nfv_vim/strategy/_strategy_stages.py @@ -66,6 +66,7 @@ class StrategyStageNames(Constants): KUBE_UPGRADE_FIRST_CONTROL_PLANE = \ Constant('kube-upgrade-first-control-plane') KUBE_UPGRADE_NETWORKING = Constant('kube-upgrade-networking') + KUBE_UPGRADE_STORAGE = Constant('kube-upgrade-storage') KUBE_UPGRADE_SECOND_CONTROL_PLANE = \ Constant('kube-upgrade-second-control-plane') KUBE_UPGRADE_PATCH = Constant('kube-upgrade-patch') diff --git a/nfv/nfv-vim/nfv_vim/strategy/_strategy_steps.py b/nfv/nfv-vim/nfv_vim/strategy/_strategy_steps.py index 3d6a6534..15cec490 100755 --- a/nfv/nfv-vim/nfv_vim/strategy/_strategy_steps.py +++ b/nfv/nfv-vim/nfv_vim/strategy/_strategy_steps.py @@ -83,6 +83,7 @@ class StrategyStepNames(Constants): KUBE_UPGRADE_COMPLETE = Constant('kube-upgrade-complete') KUBE_UPGRADE_DOWNLOAD_IMAGES = Constant('kube-upgrade-download-images') KUBE_UPGRADE_NETWORKING = Constant('kube-upgrade-networking') + KUBE_UPGRADE_STORAGE = Constant('kube-upgrade-storage') KUBE_HOST_UPGRADE_CONTROL_PLANE = \ Constant('kube-host-upgrade-control-plane') KUBE_HOST_UPGRADE_KUBELET = Constant('kube-host-upgrade-kubelet') @@ -4403,6 +4404,40 @@ class KubeUpgradeNetworkingStep(AbstractKubeUpgradeStep): return strategy.STRATEGY_STEP_RESULT.WAIT, "" +class KubeUpgradeStorageStep(AbstractKubeUpgradeStep): + """Kube Upgrade Storage - Strategy Step""" + + def __init__(self): + from nfv_vim import nfvi + super(KubeUpgradeStorageStep, self).__init__( + STRATEGY_STEP_NAME.KUBE_UPGRADE_STORAGE, + nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADED_STORAGE, + nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADING_STORAGE_FAILED, + timeout_in_secs=900) + + @coroutine + def _response_callback(self): + """Kube Upgrade Storage - Callback""" + + response = (yield) + DLOG.debug("%s callback response=%s." % (self._name, response)) + + if response['completed']: + if self.strategy is not None: + self.strategy.nfvi_kube_upgrade = response['result-data'] + else: + result = strategy.STRATEGY_STEP_RESULT.FAILED + self.stage.step_complete(result, response['reason']) + + def apply(self): + """Kube Upgrade Storage""" + + from nfv_vim import nfvi + + nfvi.nfvi_kube_upgrade_storage(self._response_callback()) + return strategy.STRATEGY_STEP_RESULT.WAIT, "" + + class AbstractKubeHostUpgradeStep(AbstractKubeUpgradeStep): """Kube Upgrade Host - Abstract Strategy Step @@ -4851,6 +4886,7 @@ def strategy_step_rebuild_from_dict(data): STRATEGY_STEP_NAME.KUBE_UPGRADE_DOWNLOAD_IMAGES: KubeUpgradeDownloadImagesStep, STRATEGY_STEP_NAME.KUBE_UPGRADE_NETWORKING: KubeUpgradeNetworkingStep, + STRATEGY_STEP_NAME.KUBE_UPGRADE_STORAGE: KubeUpgradeStorageStep, STRATEGY_STEP_NAME.KUBE_UPGRADE_START: KubeUpgradeStartStep, STRATEGY_STEP_NAME.QUERY_KUBE_HOST_UPGRADE: QueryKubeHostUpgradeStep, STRATEGY_STEP_NAME.QUERY_KUBE_UPGRADE: QueryKubeUpgradeStep,