diff --git a/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_infrastructure_api.py b/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_infrastructure_api.py index f155ebfd..083fa315 100755 --- a/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_infrastructure_api.py +++ b/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_infrastructure_api.py @@ -2017,6 +2017,124 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI): callback.send(response) callback.close() + def kube_pre_application_update(self, future, callback): + """ + Start kube pre application update + """ + response = dict() + response['completed'] = False + response['reason'] = '' + action_type = 'kube-pre-application-update' + + try: + future.set_timeouts(config.CONF.get('nfvi-timeouts', None)) + + if self._platform_token is None or \ + self._platform_token.is_expired(): + future.work(openstack.get_token, self._platform_directory) + future.result = (yield) + + if not future.result.is_complete() or \ + future.result.data is None: + DLOG.error("OpenStack get-token did not complete.") + return + + self._platform_token = future.result.data + + future.work(sysinv.kube_pre_application_update, + self._platform_token) + future.result = (yield) + + if not future.result.is_complete(): + DLOG.error("%s did not complete." % action_type) + return + + kube_upgrade_data = future.result.data + kube_upgrade_obj = nfvi.objects.v1.KubeUpgrade( + kube_upgrade_data['state'], + kube_upgrade_data['from_version'], + kube_upgrade_data['to_version']) + + response['result-data'] = kube_upgrade_obj + response['completed'] = True + + except exceptions.OpenStackRestAPIException as e: + if httplib.UNAUTHORIZED == e.http_status_code: + response['error-code'] = nfvi.NFVI_ERROR_CODE.TOKEN_EXPIRED + if self._platform_token is not None: + self._platform_token.set_expired() + + else: + DLOG.exception("Caught API exception while trying %s. error=%s" + % (action_type, e)) + + except Exception as e: + DLOG.exception("Caught exception while trying %s. error=%s" + % (action_type, e)) + + finally: + callback.send(response) + callback.close() + + def kube_post_application_update(self, future, callback): + """ + Start kube post application update + """ + response = dict() + response['completed'] = False + response['reason'] = '' + action_type = 'kube-post-application-update' + + try: + future.set_timeouts(config.CONF.get('nfvi-timeouts', None)) + + if self._platform_token is None or \ + self._platform_token.is_expired(): + future.work(openstack.get_token, self._platform_directory) + future.result = (yield) + + if not future.result.is_complete() or \ + future.result.data is None: + DLOG.error("OpenStack get-token did not complete.") + return + + self._platform_token = future.result.data + + future.work(sysinv.kube_post_application_update, + self._platform_token) + future.result = (yield) + + if not future.result.is_complete(): + DLOG.error("%s did not complete." % action_type) + return + + kube_upgrade_data = future.result.data + kube_upgrade_obj = nfvi.objects.v1.KubeUpgrade( + kube_upgrade_data['state'], + kube_upgrade_data['from_version'], + kube_upgrade_data['to_version']) + + response['result-data'] = kube_upgrade_obj + response['completed'] = True + + except exceptions.OpenStackRestAPIException as e: + if httplib.UNAUTHORIZED == e.http_status_code: + response['error-code'] = nfvi.NFVI_ERROR_CODE.TOKEN_EXPIRED + if self._platform_token is not None: + self._platform_token.set_expired() + + else: + DLOG.exception("Caught API exception while trying %s. error=%s" + % (action_type, e)) + + except Exception as e: + DLOG.exception("Caught exception while trying %s. error=%s" + % (action_type, e)) + + finally: + callback.send(response) + callback.close() + def kube_upgrade_networking(self, future, callback): """ Start kube upgrade networking diff --git a/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/sysinv.py b/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/sysinv.py index 5fb4baff..129c5169 100755 --- a/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/sysinv.py +++ b/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/sysinv.py @@ -436,6 +436,13 @@ def kube_upgrade_download_images(token): return _patch_kube_upgrade_state(token, "downloading-images") +def kube_pre_application_update(token): + """ + Ask System Inventory to kube pre application update + """ + return _patch_kube_upgrade_state(token, "pre-updating-apps") + + def kube_upgrade_networking(token): """ Ask System Inventory to kube upgrade networking @@ -583,6 +590,13 @@ def upgrade_activate(token): return response +def kube_post_application_update(token): + """ + Ask System Inventory to kube post application update + """ + return _patch_kube_upgrade_state(token, "post-updating-apps") + + def upgrade_complete(token): """ Ask System Inventory to complete an upgrade diff --git a/nfv/nfv-tests/nfv_unit_tests/tests/test_kube_upgrade_strategy.py b/nfv/nfv-tests/nfv_unit_tests/tests/test_kube_upgrade_strategy.py index 5849f08f..e982da6e 100755 --- a/nfv/nfv-tests/nfv_unit_tests/tests/test_kube_upgrade_strategy.py +++ b/nfv/nfv-tests/nfv_unit_tests/tests/test_kube_upgrade_strategy.py @@ -269,8 +269,7 @@ class ApplyStageMixin(object): 'total_steps': 1, 'steps': [ {'name': 'kube-upgrade-start', - 'success_state': 'upgrade-started', - 'fail_state': 'upgrade-starting-failed'}, + 'success_state': 'upgrade-started'}, ], } @@ -285,6 +284,17 @@ class ApplyStageMixin(object): ], } + def _kube_pre_application_update_stage(self): + return { + 'name': 'kube-pre-application-update', + 'total_steps': 1, + 'steps': [ + {'name': 'kube-pre-application-update', + 'success_state': 'pre-updated-apps', + 'fail_state': 'pre-updating-apps-failed'}, + ], + } + def _kube_host_cordon_stage(self, ver="N/A"): return { 'name': 'kube-host-cordon', @@ -362,6 +372,17 @@ class ApplyStageMixin(object): ], } + def _kube_post_application_update_stage(self): + return { + 'name': 'kube-post-application-update', + 'total_steps': 1, + 'steps': [ + {'name': 'kube-post-application-update', + 'success_state': 'post-updated-apps', + 'fail_state': 'post-updating-apps-failed'}, + ], + } + def _kube_upgrade_cleanup_stage(self): return { 'name': 'kube-upgrade-cleanup', @@ -496,6 +517,7 @@ class ApplyStageMixin(object): storage_list=None, add_start=True, add_download=True, + add_pre_app_update=True, add_networking=True, add_storage=True, add_cordon=True, @@ -503,6 +525,7 @@ class ApplyStageMixin(object): add_second_control_plane=True, add_kubelets=True, add_uncordon=True, + add_post_app_update=True, add_complete=True, add_cleanup=True): """The order of the host_list determines the kubelets""" @@ -518,6 +541,8 @@ class ApplyStageMixin(object): stages.append(self._kube_upgrade_start_stage()) if add_download: stages.append(self._kube_upgrade_download_images_stage()) + if add_pre_app_update: + stages.append(self._kube_pre_application_update_stage()) if add_networking: stages.append(self._kube_upgrade_networking_stage()) if add_storage: @@ -539,6 +564,8 @@ class ApplyStageMixin(object): stages.append(self._kube_host_uncordon_stage()) if add_complete: stages.append(self._kube_upgrade_complete_stage()) + if add_post_app_update: + stages.append(self._kube_post_application_update_stage()) if add_cleanup: stages.append(self._kube_upgrade_cleanup_stage()) return stages @@ -582,7 +609,7 @@ class ApplyStageMixin(object): """ Test the kube_upgrade strategy creation when the upgrade had previously stopped after upgrade-completed. - It is expected to resume at the cleanup stage + It is expected to resume at the post application update stage """ kube_upgrade = self._create_kube_upgrade_obj( KUBE_UPGRADE_STATE.KUBE_UPGRADE_COMPLETE, @@ -590,6 +617,7 @@ class ApplyStageMixin(object): self.default_to_version) # not using build_stage_list utility since the list of stages is small stages = [ + self._kube_post_application_update_stage(), self._kube_upgrade_cleanup_stage(), ] self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages) @@ -624,39 +652,6 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase, self.worker_list = [] self.storage_list = [] - def test_resume_after_starting_failed(self): - """ - Test the kube_upgrade strategy creation when the upgrade had previously - stopped with 'upgrade-starting-failed' - It is expected to resume at the 'upgrade-starting' stage - """ - kube_upgrade = self._create_kube_upgrade_obj( - KUBE_UPGRADE_STATE.KUBE_UPGRADE_STARTING_FAILED, - self.default_from_version, - self.default_to_version) - stages = [ - self._kube_upgrade_start_stage(), - self._kube_upgrade_download_images_stage(), - self._kube_upgrade_networking_stage(), - self._kube_upgrade_storage_stage(), - ] - if self.is_simplex(): - stages.append(self._kube_host_cordon_stage()) - for ver in self.kube_versions: - stages.append(self._kube_upgrade_first_control_plane_stage(ver)) - stages.extend(self._kube_upgrade_kubelet_stages( - ver, - self.std_controller_list, - self.aio_controller_list, - self.worker_list)) - if self.is_simplex(): - stages.append(self._kube_host_uncordon_stage()) - stages.extend([ - self._kube_upgrade_complete_stage(), - self._kube_upgrade_cleanup_stage(), - ]) - self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages) - def test_resume_after_download_images_failed(self): """ Test the kube_upgrade strategy creation when the upgrade had previously @@ -669,6 +664,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase, self.default_to_version) stages = [ self._kube_upgrade_download_images_stage(), + self._kube_pre_application_update_stage(), self._kube_upgrade_networking_stage(), self._kube_upgrade_storage_stage(), ] @@ -685,6 +681,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase, stages.append(self._kube_host_uncordon_stage()) stages.extend([ self._kube_upgrade_complete_stage(), + self._kube_post_application_update_stage(), self._kube_upgrade_cleanup_stage(), ]) self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages) @@ -693,12 +690,79 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase, """ Test the kube_upgrade strategy creation when the upgrade had previously stopped with 'downloaded-images' - It is expected to resume at the 'first control plane' stage. + It is expected to resume at the 'pre application update' stage. """ kube_upgrade = self._create_kube_upgrade_obj( KUBE_UPGRADE_STATE.KUBE_UPGRADE_DOWNLOADED_IMAGES, self.default_from_version, self.default_to_version) + stages = [ + self._kube_pre_application_update_stage(), + self._kube_upgrade_networking_stage(), + self._kube_upgrade_storage_stage(), + ] + if self.is_simplex(): + stages.append(self._kube_host_cordon_stage()) + for ver in self.kube_versions: + stages.append(self._kube_upgrade_first_control_plane_stage( + ver)) + stages.extend(self._kube_upgrade_kubelet_stages( + ver, + self.std_controller_list, + self.aio_controller_list, + self.worker_list)) + if self.is_simplex(): + stages.append(self._kube_host_uncordon_stage()) + stages.extend([ + self._kube_upgrade_complete_stage(), + self._kube_post_application_update_stage(), + self._kube_upgrade_cleanup_stage(), + ]) + self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages) + + def test_resume_after_pre_app_update_failed(self): + """ + Test the kube_upgrade strategy creation when the upgrade had previously + stopped with 'pre-updating-apps-failed'. + It is expected to resume at the pre application update stage. + """ + kube_upgrade = self._create_kube_upgrade_obj( + KUBE_UPGRADE_STATE.KUBE_PRE_UPDATING_APPS_FAILED, + self.default_from_version, + self.default_to_version) + stages = [ + self._kube_pre_application_update_stage(), + self._kube_upgrade_networking_stage(), + self._kube_upgrade_storage_stage(), + ] + if self.is_simplex(): + stages.append(self._kube_host_cordon_stage()) + for ver in self.kube_versions: + stages.append(self._kube_upgrade_first_control_plane_stage(ver)) + stages.extend(self._kube_upgrade_kubelet_stages( + ver, + self.std_controller_list, + self.aio_controller_list, + self.worker_list)) + if self.is_simplex(): + stages.append(self._kube_host_uncordon_stage()) + stages.extend([ + self._kube_upgrade_complete_stage(), + self._kube_post_application_update_stage(), + self._kube_upgrade_cleanup_stage(), + ]) + self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages) + + def test_resume_after_pre_app_update_succeeded(self): + """ + Test the kube_upgrade strategy creation when the upgrade had previously + stopped with 'pre-updated-apps'. + It is expected to resume at the upgrade networking stage. + """ + kube_upgrade = self._create_kube_upgrade_obj( + KUBE_UPGRADE_STATE.KUBE_PRE_UPDATED_APPS, + self.default_from_version, + self.default_to_version) stages = [ self._kube_upgrade_networking_stage(), self._kube_upgrade_storage_stage(), @@ -717,6 +781,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase, stages.append(self._kube_host_uncordon_stage()) stages.extend([ self._kube_upgrade_complete_stage(), + self._kube_post_application_update_stage(), self._kube_upgrade_cleanup_stage(), ]) self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages) @@ -744,6 +809,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase, stages.append(self._kube_host_uncordon_stage()) stages.extend([ self._kube_upgrade_complete_stage(), + self._kube_post_application_update_stage(), self._kube_upgrade_cleanup_stage(), ]) self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages) @@ -769,6 +835,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase, stages.append(self._kube_host_uncordon_stage()) stages.extend([ self._kube_upgrade_complete_stage(), + self._kube_post_application_update_stage(), self._kube_upgrade_cleanup_stage(), ]) self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages) @@ -801,6 +868,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase, stages.append(self._kube_host_uncordon_stage()) stages.extend([ self._kube_upgrade_complete_stage(), + self._kube_post_application_update_stage(), self._kube_upgrade_cleanup_stage(), ]) self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages) @@ -832,6 +900,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase, stages.append(self._kube_host_uncordon_stage()) stages.extend([ self._kube_upgrade_complete_stage(), + self._kube_post_application_update_stage(), self._kube_upgrade_cleanup_stage(), ]) self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages) @@ -863,6 +932,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase, stages.append(self._kube_host_uncordon_stage()) stages.extend([ self._kube_upgrade_complete_stage(), + self._kube_post_application_update_stage(), self._kube_upgrade_cleanup_stage(), ]) self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages) @@ -892,6 +962,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase, stages.append(self._kube_host_uncordon_stage()) stages.extend([ self._kube_upgrade_complete_stage(), + self._kube_post_application_update_stage(), self._kube_upgrade_cleanup_stage(), ]) self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages) @@ -919,6 +990,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase, stages.append(self._kube_host_uncordon_stage()) stages.extend([ self._kube_upgrade_complete_stage(), + self._kube_post_application_update_stage(), self._kube_upgrade_cleanup_stage(), ]) self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages) @@ -946,10 +1018,42 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase, stages.append(self._kube_host_uncordon_stage()) stages.extend([ self._kube_upgrade_complete_stage(), + self._kube_post_application_update_stage(), self._kube_upgrade_cleanup_stage(), ]) self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages) + def test_resume_after_post_app_update_failed(self): + """ + Test the kube_upgrade strategy creation when the upgrade had previously + stopped with 'post-updating-apps-failed' + It is expected to resume at the post application update stage. + """ + kube_upgrade = self._create_kube_upgrade_obj( + KUBE_UPGRADE_STATE.KUBE_POST_UPDATING_APPS_FAILED, + self.default_from_version, + self.default_to_version) + stages = [ + self._kube_post_application_update_stage(), + self._kube_upgrade_cleanup_stage(), + ] + self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages) + + def test_resume_after_post_app_update_succeeded(self): + """ + Test the kube_upgrade strategy creation when the upgrade had previously + stopped with 'post-updated-apps'. + It is expected to resume at the cleanup stage. + """ + kube_upgrade = self._create_kube_upgrade_obj( + KUBE_UPGRADE_STATE.KUBE_POST_UPDATED_APPS, + self.default_from_version, + self.default_to_version) + stages = [ + self._kube_upgrade_cleanup_stage(), + ] + self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages) + @mock.patch('nfv_vim.event_log._instance._event_issue', sw_update_testcase.fake_event_issue) diff --git a/nfv/nfv-vim/nfv_vim/nfvi/__init__.py b/nfv/nfv-vim/nfv_vim/nfvi/__init__.py index baaf1a52..422d9a99 100755 --- a/nfv/nfv-vim/nfv_vim/nfvi/__init__.py +++ b/nfv/nfv-vim/nfv_vim/nfvi/__init__.py @@ -117,6 +117,8 @@ from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_host_cordon # no from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_host_uncordon # noqa: F401 from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_host_upgrade_control_plane # noqa: F401 from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_host_upgrade_kubelet # noqa: F401 +from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_post_application_update # noqa: F401 +from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_pre_application_update # noqa: F401 from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_rootca_update_abort # noqa: F401 from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_rootca_update_complete # noqa: F401 from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_rootca_update_generate_cert # noqa: F401 diff --git a/nfv/nfv-vim/nfv_vim/nfvi/_nfvi_infrastructure_module.py b/nfv/nfv-vim/nfv_vim/nfvi/_nfvi_infrastructure_module.py index 1447d2a3..80e2636f 100755 --- a/nfv/nfv-vim/nfv_vim/nfvi/_nfvi_infrastructure_module.py +++ b/nfv/nfv-vim/nfv_vim/nfvi/_nfvi_infrastructure_module.py @@ -321,6 +321,26 @@ def nfvi_kube_upgrade_start(to_version, force, alarm_ignore_list, callback): return cmd_id +def nfvi_kube_pre_application_update(callback): + """ + Kube Upgrade Pre Application Update + """ + cmd_id = _infrastructure_plugin.invoke_plugin( + 'kube_pre_application_update', + callback=callback) + return cmd_id + + +def nfvi_kube_post_application_update(callback): + """ + Kube Upgrade Post Application Update + """ + cmd_id = _infrastructure_plugin.invoke_plugin( + 'kube_post_application_update', + callback=callback) + return cmd_id + + def nfvi_get_kube_host_upgrade_list(callback): """ Get kube host upgrade list diff --git a/nfv/nfv-vim/nfv_vim/nfvi/objects/v1/_kube_upgrade.py b/nfv/nfv-vim/nfv_vim/nfvi/objects/v1/_kube_upgrade.py index 04d50364..fe956e2b 100755 --- a/nfv/nfv-vim/nfv_vim/nfvi/objects/v1/_kube_upgrade.py +++ b/nfv/nfv-vim/nfv_vim/nfvi/objects/v1/_kube_upgrade.py @@ -37,12 +37,16 @@ class KubeUpgradeState(Constants): Maintaining the same order as defined in kubernetes.py """ - KUBE_UPGRADE_STARTING = Constant('upgrade-starting') - KUBE_UPGRADE_STARTING_FAILED = Constant('upgrade-starting-failed') KUBE_UPGRADE_STARTED = Constant('upgrade-started') KUBE_UPGRADE_DOWNLOADING_IMAGES = Constant('downloading-images') KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED = Constant('downloading-images-failed') KUBE_UPGRADE_DOWNLOADED_IMAGES = Constant('downloaded-images') + KUBE_PRE_UPDATING_APPS = Constant('pre-updating-apps') + KUBE_PRE_UPDATING_APPS_FAILED = Constant('pre-updating-apps-failed') + KUBE_PRE_UPDATED_APPS = Constant('pre-updated-apps') + KUBE_POST_UPDATING_APPS = Constant('post-updating-apps') + KUBE_POST_UPDATING_APPS_FAILED = Constant('post-updating-apps-failed') + KUBE_POST_UPDATED_APPS = Constant('post-updated-apps') KUBE_UPGRADING_NETWORKING = Constant('upgrading-networking') KUBE_UPGRADING_NETWORKING_FAILED = Constant('upgrading-networking-failed') KUBE_UPGRADED_NETWORKING = Constant('upgraded-networking') diff --git a/nfv/nfv-vim/nfv_vim/strategy/__init__.py b/nfv/nfv-vim/nfv_vim/strategy/__init__.py index 4ddcf392..008ae29a 100755 --- a/nfv/nfv-vim/nfv_vim/strategy/__init__.py +++ b/nfv/nfv-vim/nfv_vim/strategy/__init__.py @@ -21,6 +21,8 @@ from nfv_vim.strategy._strategy_steps import KubeHostCordonStep # noqa: F401 from nfv_vim.strategy._strategy_steps import KubeHostUncordonStep # noqa: F401 from nfv_vim.strategy._strategy_steps import KubeHostUpgradeControlPlaneStep # noqa: F401 from nfv_vim.strategy._strategy_steps import KubeHostUpgradeKubeletStep # noqa: F401 +from nfv_vim.strategy._strategy_steps import KubePostApplicationUpdateStep # noqa: F401 +from nfv_vim.strategy._strategy_steps import KubePreApplicationUpdateStep # noqa: F401 from nfv_vim.strategy._strategy_steps import KubeRootcaUpdateCompleteStep # noqa: F401 from nfv_vim.strategy._strategy_steps import KubeRootcaUpdateGenerateCertStep # noqa: F401 from nfv_vim.strategy._strategy_steps import KubeRootcaUpdateHostTrustBothcasStep # noqa: F401 diff --git a/nfv/nfv-vim/nfv_vim/strategy/_strategy.py b/nfv/nfv-vim/nfv_vim/strategy/_strategy.py index ffc56fcc..f9a8e977 100755 --- a/nfv/nfv-vim/nfv_vim/strategy/_strategy.py +++ b/nfv/nfv-vim/nfv_vim/strategy/_strategy.py @@ -3211,7 +3211,23 @@ class KubeUpgradeStrategy(SwUpdateStrategy, strategy.STRATEGY_STAGE_NAME.KUBE_UPGRADE_DOWNLOAD_IMAGES) stage.add_step(strategy.KubeUpgradeDownloadImagesStep()) self.apply_phase.add_stage(stage) - # Next stage after download images is upgrade networking + # Next stage after download images is pre application update + self._add_kube_pre_application_update_stage() + + def _add_kube_pre_application_update_stage(self): + """ + Add kube pre application update stage. + This stage only occurs after download images + It then proceeds to the next stage + """ + from nfv_vim import strategy + + stage = strategy.StrategyStage( + strategy.STRATEGY_STAGE_NAME.KUBE_PRE_APPLICATION_UPDATE) + stage.add_step(strategy.KubePreApplicationUpdateStep()) + self.apply_phase.add_stage(stage) + + # Next stage after pre application update is upgrade networking self._add_kube_upgrade_networking_stage() def _add_kube_upgrade_networking_stage(self): @@ -3496,13 +3512,29 @@ class KubeUpgradeStrategy(SwUpdateStrategy, strategy.STRATEGY_STAGE_NAME.KUBE_UPGRADE_COMPLETE) stage.add_step(strategy.KubeUpgradeCompleteStep()) self.apply_phase.add_stage(stage) - # stage after kube upgrade complete stage, cleans up the kube upgrade + # Next stage after upgrade complete is post application update + self._add_kube_post_application_update_stage() + + def _add_kube_post_application_update_stage(self): + """ + Add kube post application update stage. + This stage occurs after the upgrade is completed + It then proceeds to the next stage + """ + from nfv_vim import strategy + + stage = strategy.StrategyStage( + strategy.STRATEGY_STAGE_NAME.KUBE_POST_APPLICATION_UPDATE) + stage.add_step(strategy.KubePostApplicationUpdateStep()) + self.apply_phase.add_stage(stage) + + # Next stage after post application update is upgrade cleanup self._add_kube_upgrade_cleanup_stage() def _add_kube_upgrade_cleanup_stage(self): """ kube upgrade cleanup stage deletes the kube upgrade. - This stage occurs after all kube upgrade is completed + This stage occurs after the post application update stage """ from nfv_vim import strategy stage = strategy.StrategyStage( @@ -3563,10 +3595,6 @@ class KubeUpgradeStrategy(SwUpdateStrategy, # Note: there are no resume states for actions that are still running # ie: KUBE_UPGRADE_DOWNLOADING_IMAGES RESUME_STATE = { - # If upgrade start failed, allow to restart - nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_STARTING_FAILED: - self._add_kube_upgrade_start_stage, - # after upgrade-started -> download images nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_STARTED: self._add_kube_upgrade_download_images_stage, @@ -3575,8 +3603,16 @@ class KubeUpgradeStrategy(SwUpdateStrategy, nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED: self._add_kube_upgrade_download_images_stage, - # After downloading images -> upgrade networking + # After downloading images -> pre update applications nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_DOWNLOADED_IMAGES: + self._add_kube_pre_application_update_stage, + + # if pre updating apps failed, resume at pre updating apps + nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_PRE_UPDATING_APPS_FAILED: + self._add_kube_pre_application_update_stage, + + # After pre updating apps -> upgrade networking + nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_PRE_UPDATED_APPS: self._add_kube_upgrade_networking_stage, # if networking state failed, resync at networking state @@ -3631,8 +3667,16 @@ class KubeUpgradeStrategy(SwUpdateStrategy, nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_HOST_UNCORDON_COMPLETE: self._add_kube_upgrade_complete_stage, - # upgrade is completed, delete the upgrade + # upgrade is completed, post update apps nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_COMPLETE: + self._add_kube_post_application_update_stage, + + # If post updating apps failed, resume at post application update stage + nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_POST_UPDATING_APPS_FAILED: + self._add_kube_post_application_update_stage, + + # After post updating apps, delete the upgrade + nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_POST_UPDATED_APPS: self._add_kube_upgrade_cleanup_stage, } diff --git a/nfv/nfv-vim/nfv_vim/strategy/_strategy_stages.py b/nfv/nfv-vim/nfv_vim/strategy/_strategy_stages.py index dd02a0d9..07c9c0f2 100755 --- a/nfv/nfv-vim/nfv_vim/strategy/_strategy_stages.py +++ b/nfv/nfv-vim/nfv_vim/strategy/_strategy_stages.py @@ -63,6 +63,7 @@ class StrategyStageNames(Constants): KUBE_UPGRADE_QUERY = Constant('kube-upgrade-query') KUBE_UPGRADE_START = Constant('kube-upgrade-start') KUBE_UPGRADE_DOWNLOAD_IMAGES = Constant('kube-upgrade-download-images') + KUBE_PRE_APPLICATION_UPDATE = Constant('kube-pre-application-update') KUBE_UPGRADE_FIRST_CONTROL_PLANE = \ Constant('kube-upgrade-first-control-plane') KUBE_UPGRADE_NETWORKING = Constant('kube-upgrade-networking') @@ -73,6 +74,7 @@ class StrategyStageNames(Constants): KUBE_UPGRADE_KUBELETS_CONTROLLERS = \ Constant('kube-upgrade-kubelets-controllers') KUBE_UPGRADE_KUBELETS_WORKERS = Constant('kube-upgrade-kubelets-workers') + KUBE_POST_APPLICATION_UPDATE = Constant('kube-post-application-update') KUBE_UPGRADE_COMPLETE = Constant('kube-upgrade-complete') KUBE_UPGRADE_CLEANUP = Constant('kube-upgrade-cleanup') diff --git a/nfv/nfv-vim/nfv_vim/strategy/_strategy_steps.py b/nfv/nfv-vim/nfv_vim/strategy/_strategy_steps.py index a2446bd6..5b684fc1 100755 --- a/nfv/nfv-vim/nfv_vim/strategy/_strategy_steps.py +++ b/nfv/nfv-vim/nfv_vim/strategy/_strategy_steps.py @@ -88,6 +88,8 @@ class StrategyStepNames(Constants): KUBE_HOST_UPGRADE_CONTROL_PLANE = \ Constant('kube-host-upgrade-control-plane') KUBE_HOST_UPGRADE_KUBELET = Constant('kube-host-upgrade-kubelet') + KUBE_PRE_APPLICATION_UPDATE = Constant('kube-pre-application-update') + KUBE_POST_APPLICATION_UPDATE = Constant('kube-post-application-update') # system config update specific steps QUERY_SYSTEM_CONFIG_UPDATE_HOSTS = Constant('query-system-config-update-hosts') SYSTEM_CONFIG_UPDATE_HOSTS = Constant('system-config-update-hosts') @@ -4134,8 +4136,7 @@ class KubeUpgradeStartStep(AbstractKubeUpgradeStep): super(KubeUpgradeStartStep, self).__init__( STRATEGY_STEP_NAME.KUBE_UPGRADE_START, nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_STARTED, - nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_STARTING_FAILED, - timeout_in_secs=1800) + None) # there is no failure state if upgrade-start fails # next 2 attributes must be persisted through from_dict/as_dict self._to_version = to_version self._force = force @@ -4171,9 +4172,13 @@ class KubeUpgradeStartStep(AbstractKubeUpgradeStep): response = (yield) DLOG.debug("%s callback response=%s." % (self._name, response)) + # kube-upgrade-start will return a result when it completes, + # so we do not want to use handle_event if response['completed']: if self.strategy is not None: self.strategy.nfvi_kube_upgrade = response['result-data'] + result = strategy.STRATEGY_STEP_RESULT.SUCCESS + self.stage.step_complete(result, "") else: result = strategy.STRATEGY_STEP_RESULT.FAILED self.stage.step_complete(result, response['reason']) @@ -4363,6 +4368,82 @@ class KubeUpgradeDownloadImagesStep(AbstractKubeUpgradeStep): return strategy.STRATEGY_STEP_RESULT.WAIT, "" +class KubePreApplicationUpdateStep(AbstractKubeUpgradeStep): + """Kube Pre Application Update - Strategy Step""" + + def __init__(self): + from nfv_vim import nfvi + super(KubePreApplicationUpdateStep, self).__init__( + STRATEGY_STEP_NAME.KUBE_PRE_APPLICATION_UPDATE, + nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_PRE_UPDATED_APPS, + nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_PRE_UPDATING_APPS_FAILED, + timeout_in_secs=1800) + + def abort(self): + """ + Returns the abort step related to this step + """ + return [KubeUpgradeAbortStep()] + + @coroutine + def _response_callback(self): + """Kube Pre Application Update - Callback""" + + response = (yield) + DLOG.debug("%s callback response=%s." % (self._name, response)) + + if response['completed']: + if self.strategy is not None: + self.strategy.nfvi_kube_upgrade = response['result-data'] + else: + result = strategy.STRATEGY_STEP_RESULT.FAILED + self.stage.step_complete(result, response['reason']) + + def apply(self): + """Kube Pre Application Update """ + + from nfv_vim import nfvi + DLOG.info("Step (%s) apply." % self._name) + + nfvi.nfvi_kube_pre_application_update(self._response_callback()) + return strategy.STRATEGY_STEP_RESULT.WAIT, "" + + +class KubePostApplicationUpdateStep(AbstractKubeUpgradeStep): + """Kube Post Application Update - Strategy Step""" + + def __init__(self): + from nfv_vim import nfvi + super(KubePostApplicationUpdateStep, self).__init__( + STRATEGY_STEP_NAME.KUBE_POST_APPLICATION_UPDATE, + nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_POST_UPDATED_APPS, + nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_POST_UPDATING_APPS_FAILED, + timeout_in_secs=1800) + + @coroutine + def _response_callback(self): + """Kube Post Application Update - Callback""" + + response = (yield) + DLOG.debug("%s callback response=%s." % (self._name, response)) + + if response['completed']: + if self.strategy is not None: + self.strategy.nfvi_kube_upgrade = response['result-data'] + else: + result = strategy.STRATEGY_STEP_RESULT.FAILED + self.stage.step_complete(result, response['reason']) + + def apply(self): + """Kube Post Application Update """ + + from nfv_vim import nfvi + DLOG.info("Step (%s) apply." % self._name) + + nfvi.nfvi_kube_post_application_update(self._response_callback()) + return strategy.STRATEGY_STEP_RESULT.WAIT, "" + + class KubeUpgradeNetworkingStep(AbstractKubeUpgradeStep): """Kube Upgrade Networking - Strategy Step""" @@ -4894,6 +4975,8 @@ def strategy_step_rebuild_from_dict(data): STRATEGY_STEP_NAME.KUBE_UPGRADE_NETWORKING: KubeUpgradeNetworkingStep, STRATEGY_STEP_NAME.KUBE_UPGRADE_STORAGE: KubeUpgradeStorageStep, STRATEGY_STEP_NAME.KUBE_UPGRADE_START: KubeUpgradeStartStep, + STRATEGY_STEP_NAME.KUBE_PRE_APPLICATION_UPDATE: KubePreApplicationUpdateStep, + STRATEGY_STEP_NAME.KUBE_POST_APPLICATION_UPDATE: KubePostApplicationUpdateStep, STRATEGY_STEP_NAME.QUERY_KUBE_HOST_UPGRADE: QueryKubeHostUpgradeStep, STRATEGY_STEP_NAME.QUERY_KUBE_UPGRADE: QueryKubeUpgradeStep, STRATEGY_STEP_NAME.QUERY_KUBE_VERSIONS: QueryKubeVersionsStep,