starlingx
/
nfv
Code Issues Proposed changes

Add new application update steps for k8s upgrades 

Add two new steps to the orchestrated Kubernetes upgrade:
  * 'pre-application-update': update StarlingX applications before the
    new Kubernetes version is deployed.
  * 'post-application-update': update Starlingx applications after the
    new Kubernetes version is deployed.

This was implemented according to the changes introduced to sysinv by
the following commit:
https://review.opendev.org/c/starlingx/config/+/909172

Unit tests have also been updated to account for the new steps.

Test Plan:
PASS: build-pkgs -a && build-image
AIO-SX Test Cases:
PASS: Run an orchestrated Kubernetes upgrade with no apps to be updated.
PASS: Run a single-version orchestrated Kubernetes upgrade with apps to
      be updated during the pre-application-update and
      post-application-update steps.
PASS: Run a multi-version orchestrated Kubernetes upgrade with apps to
      be updated during the pre-application-update and
      post-application-update steps.
PASS: Run an orchestrated Kubernetes upgrade that fails during the
      pre-application-update step due to a missing metadata.yaml file.
      Confirm that the upgrade was successfully aborted.
PASS: Run an orchestrated Kubernetes upgrade that fails during the
      post-application-update step due to a missing metadata.yaml file.
      Confirm that the upgrade cannot be aborted at this point.
      Fix the application tarball and resume the upgrade.
      Confirm that the upgrade resumes from the same step.
      Confirm that the upgrade finishes successfully.
AIO-DX Test Cases:
PASS: Run an orchestrated Kubernetes upgrade with no apps to be updated.
PASS: Run a single-version orchestrated Kubernetes upgrade with apps to
      be updated during the pre-application-update and
      post-application-update steps.
PASS: Run an orchestrated Kubernetes upgrade that fails during the
      pre-application-update step due to a missing metadata.yaml file.
      Fix the application tarball and resume the upgrade.
      Confirm that the upgrade resumes from the same step.
      Confirm that the upgrade finishes successfully.
PASS: Run an orchestrated Kubernetes upgrade that fails during the
      post-application-update step due to a missing metadata.yaml file.
      Fix the application tarball and resume the upgrade.
      Confirm that the upgrade resumes from the same step.
      Confirm that the upgrade finishes successfully.

Story: 2010929
Task: 49791
Depends-On: https://review.opendev.org/c/starlingx/config/+/909172

Change-Id: I5217db1cb744320979afcd161cb436bf089ccc9d
Signed-off-by: Igor Soares <Igor.PiresSoares@windriver.com>
This commit is contained in:
Igor Soares 2024-03-27 19:16:15 -03:00
parent 6142d9f116
commit 13eb48918a
10 changed files with 449 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

118
nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_infrastructure_api.py
View File

@ -2017,6 +2017,124 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
callback.send(response)
callback.close()
def kube_pre_application_update(self, future, callback):
"""
Start kube pre application update
"""
response = dict()
response['completed'] = False
response['reason'] = ''
action_type = 'kube-pre-application-update'
try:
future.set_timeouts(config.CONF.get('nfvi-timeouts', None))
if self._platform_token is None or \
self._platform_token.is_expired():
future.work(openstack.get_token, self._platform_directory)
future.result = (yield)
if not future.result.is_complete() or \
future.result.data is None:
DLOG.error("OpenStack get-token did not complete.")
return
self._platform_token = future.result.data
future.work(sysinv.kube_pre_application_update,
self._platform_token)
future.result = (yield)
if not future.result.is_complete():
DLOG.error("%s did not complete." % action_type)
return
kube_upgrade_data = future.result.data
kube_upgrade_obj = nfvi.objects.v1.KubeUpgrade(
kube_upgrade_data['state'],
kube_upgrade_data['from_version'],
kube_upgrade_data['to_version'])
response['result-data'] = kube_upgrade_obj
response['completed'] = True
except exceptions.OpenStackRestAPIException as e:
if httplib.UNAUTHORIZED == e.http_status_code:
response['error-code'] = nfvi.NFVI_ERROR_CODE.TOKEN_EXPIRED
if self._platform_token is not None:
self._platform_token.set_expired()
else:
DLOG.exception("Caught API exception while trying %s. error=%s"
% (action_type, e))
except Exception as e:
DLOG.exception("Caught exception while trying %s. error=%s"
% (action_type, e))
finally:
callback.send(response)
callback.close()
def kube_post_application_update(self, future, callback):
"""
Start kube post application update
"""
response = dict()
response['completed'] = False
response['reason'] = ''
action_type = 'kube-post-application-update'
try:
future.set_timeouts(config.CONF.get('nfvi-timeouts', None))
if self._platform_token is None or \
self._platform_token.is_expired():
future.work(openstack.get_token, self._platform_directory)
future.result = (yield)
if not future.result.is_complete() or \
future.result.data is None:
DLOG.error("OpenStack get-token did not complete.")
return
self._platform_token = future.result.data
future.work(sysinv.kube_post_application_update,
self._platform_token)
future.result = (yield)
if not future.result.is_complete():
DLOG.error("%s did not complete." % action_type)
return
kube_upgrade_data = future.result.data
kube_upgrade_obj = nfvi.objects.v1.KubeUpgrade(
kube_upgrade_data['state'],
kube_upgrade_data['from_version'],
kube_upgrade_data['to_version'])
response['result-data'] = kube_upgrade_obj
response['completed'] = True
except exceptions.OpenStackRestAPIException as e:
if httplib.UNAUTHORIZED == e.http_status_code:
response['error-code'] = nfvi.NFVI_ERROR_CODE.TOKEN_EXPIRED
if self._platform_token is not None:
self._platform_token.set_expired()
else:
DLOG.exception("Caught API exception while trying %s. error=%s"
% (action_type, e))
except Exception as e:
DLOG.exception("Caught exception while trying %s. error=%s"
% (action_type, e))
finally:
callback.send(response)
callback.close()
def kube_upgrade_networking(self, future, callback):
"""
Start kube upgrade networking

14
nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/sysinv.py
View File

@ -436,6 +436,13 @@ def kube_upgrade_download_images(token):
return _patch_kube_upgrade_state(token, "downloading-images")
def kube_pre_application_update(token):
"""
Ask System Inventory to kube pre application update
"""
return _patch_kube_upgrade_state(token, "pre-updating-apps")
def kube_upgrade_networking(token):
"""
Ask System Inventory to kube upgrade networking
@ -583,6 +590,13 @@ def upgrade_activate(token):
return response
def kube_post_application_update(token):
"""
Ask System Inventory to kube post application update
"""
return _patch_kube_upgrade_state(token, "post-updating-apps")
def upgrade_complete(token):
"""
Ask System Inventory to complete an upgrade

178
nfv/nfv-tests/nfv_unit_tests/tests/test_kube_upgrade_strategy.py
View File

@ -269,8 +269,7 @@ class ApplyStageMixin(object):
'total_steps': 1,
'steps': [
{'name': 'kube-upgrade-start',
'success_state': 'upgrade-started',
'fail_state': 'upgrade-starting-failed'},
'success_state': 'upgrade-started'},
],
}
@ -285,6 +284,17 @@ class ApplyStageMixin(object):
],
}
def _kube_pre_application_update_stage(self):
return {
'name': 'kube-pre-application-update',
'total_steps': 1,
'steps': [
{'name': 'kube-pre-application-update',
'success_state': 'pre-updated-apps',
'fail_state': 'pre-updating-apps-failed'},
],
}
def _kube_host_cordon_stage(self, ver="N/A"):
return {
'name': 'kube-host-cordon',
@ -362,6 +372,17 @@ class ApplyStageMixin(object):
],
}
def _kube_post_application_update_stage(self):
return {
'name': 'kube-post-application-update',
'total_steps': 1,
'steps': [
{'name': 'kube-post-application-update',
'success_state': 'post-updated-apps',
'fail_state': 'post-updating-apps-failed'},
],
}
def _kube_upgrade_cleanup_stage(self):
return {
'name': 'kube-upgrade-cleanup',
@ -496,6 +517,7 @@ class ApplyStageMixin(object):
storage_list=None,
add_start=True,
add_download=True,
add_pre_app_update=True,
add_networking=True,
add_storage=True,
add_cordon=True,
@ -503,6 +525,7 @@ class ApplyStageMixin(object):
add_second_control_plane=True,
add_kubelets=True,
add_uncordon=True,
add_post_app_update=True,
add_complete=True,
add_cleanup=True):
"""The order of the host_list determines the kubelets"""
@ -518,6 +541,8 @@ class ApplyStageMixin(object):
stages.append(self._kube_upgrade_start_stage())
if add_download:
stages.append(self._kube_upgrade_download_images_stage())
if add_pre_app_update:
stages.append(self._kube_pre_application_update_stage())
if add_networking:
stages.append(self._kube_upgrade_networking_stage())
if add_storage:
@ -539,6 +564,8 @@ class ApplyStageMixin(object):
stages.append(self._kube_host_uncordon_stage())
if add_complete:
stages.append(self._kube_upgrade_complete_stage())
if add_post_app_update:
stages.append(self._kube_post_application_update_stage())
if add_cleanup:
stages.append(self._kube_upgrade_cleanup_stage())
return stages
@ -582,7 +609,7 @@ class ApplyStageMixin(object):
"""
Test the kube_upgrade strategy creation when the upgrade had previously
stopped after upgrade-completed.
It is expected to resume at the cleanup stage
It is expected to resume at the post application update stage
"""
kube_upgrade = self._create_kube_upgrade_obj(
KUBE_UPGRADE_STATE.KUBE_UPGRADE_COMPLETE,
@ -590,6 +617,7 @@ class ApplyStageMixin(object):
self.default_to_version)
# not using build_stage_list utility since the list of stages is small
stages = [
self._kube_post_application_update_stage(),
self._kube_upgrade_cleanup_stage(),
]
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
@ -624,39 +652,6 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
self.worker_list = []
self.storage_list = []
def test_resume_after_starting_failed(self):
"""
Test the kube_upgrade strategy creation when the upgrade had previously
stopped with 'upgrade-starting-failed'
It is expected to resume at the 'upgrade-starting' stage
"""
kube_upgrade = self._create_kube_upgrade_obj(
KUBE_UPGRADE_STATE.KUBE_UPGRADE_STARTING_FAILED,
self.default_from_version,
self.default_to_version)
stages = [
self._kube_upgrade_start_stage(),
self._kube_upgrade_download_images_stage(),
self._kube_upgrade_networking_stage(),
self._kube_upgrade_storage_stage(),
]
if self.is_simplex():
stages.append(self._kube_host_cordon_stage())
for ver in self.kube_versions:
stages.append(self._kube_upgrade_first_control_plane_stage(ver))
stages.extend(self._kube_upgrade_kubelet_stages(
ver,
self.std_controller_list,
self.aio_controller_list,
self.worker_list))
if self.is_simplex():
stages.append(self._kube_host_uncordon_stage())
stages.extend([
self._kube_upgrade_complete_stage(),
self._kube_upgrade_cleanup_stage(),
])
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
def test_resume_after_download_images_failed(self):
"""
Test the kube_upgrade strategy creation when the upgrade had previously
@ -669,6 +664,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
self.default_to_version)
stages = [
self._kube_upgrade_download_images_stage(),
self._kube_pre_application_update_stage(),
self._kube_upgrade_networking_stage(),
self._kube_upgrade_storage_stage(),
]
@ -685,6 +681,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
stages.append(self._kube_host_uncordon_stage())
stages.extend([
self._kube_upgrade_complete_stage(),
self._kube_post_application_update_stage(),
self._kube_upgrade_cleanup_stage(),
])
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
@ -693,12 +690,79 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
"""
Test the kube_upgrade strategy creation when the upgrade had previously
stopped with 'downloaded-images'
It is expected to resume at the 'first control plane' stage.
It is expected to resume at the 'pre application update' stage.
"""
kube_upgrade = self._create_kube_upgrade_obj(
KUBE_UPGRADE_STATE.KUBE_UPGRADE_DOWNLOADED_IMAGES,
self.default_from_version,
self.default_to_version)
stages = [
self._kube_pre_application_update_stage(),
self._kube_upgrade_networking_stage(),
self._kube_upgrade_storage_stage(),
]
if self.is_simplex():
stages.append(self._kube_host_cordon_stage())
for ver in self.kube_versions:
stages.append(self._kube_upgrade_first_control_plane_stage(
ver))
stages.extend(self._kube_upgrade_kubelet_stages(
ver,
self.std_controller_list,
self.aio_controller_list,
self.worker_list))
if self.is_simplex():
stages.append(self._kube_host_uncordon_stage())
stages.extend([
self._kube_upgrade_complete_stage(),
self._kube_post_application_update_stage(),
self._kube_upgrade_cleanup_stage(),
])
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
def test_resume_after_pre_app_update_failed(self):
"""
Test the kube_upgrade strategy creation when the upgrade had previously
stopped with 'pre-updating-apps-failed'.
It is expected to resume at the pre application update stage.
"""
kube_upgrade = self._create_kube_upgrade_obj(
KUBE_UPGRADE_STATE.KUBE_PRE_UPDATING_APPS_FAILED,
self.default_from_version,
self.default_to_version)
stages = [
self._kube_pre_application_update_stage(),
self._kube_upgrade_networking_stage(),
self._kube_upgrade_storage_stage(),
]
if self.is_simplex():
stages.append(self._kube_host_cordon_stage())
for ver in self.kube_versions:
stages.append(self._kube_upgrade_first_control_plane_stage(ver))
stages.extend(self._kube_upgrade_kubelet_stages(
ver,
self.std_controller_list,
self.aio_controller_list,
self.worker_list))
if self.is_simplex():
stages.append(self._kube_host_uncordon_stage())
stages.extend([
self._kube_upgrade_complete_stage(),
self._kube_post_application_update_stage(),
self._kube_upgrade_cleanup_stage(),
])
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
def test_resume_after_pre_app_update_succeeded(self):
"""
Test the kube_upgrade strategy creation when the upgrade had previously
stopped with 'pre-updated-apps'.
It is expected to resume at the upgrade networking stage.
"""
kube_upgrade = self._create_kube_upgrade_obj(
KUBE_UPGRADE_STATE.KUBE_PRE_UPDATED_APPS,
self.default_from_version,
self.default_to_version)
stages = [
self._kube_upgrade_networking_stage(),
self._kube_upgrade_storage_stage(),
@ -717,6 +781,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
stages.append(self._kube_host_uncordon_stage())
stages.extend([
self._kube_upgrade_complete_stage(),
self._kube_post_application_update_stage(),
self._kube_upgrade_cleanup_stage(),
])
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
@ -744,6 +809,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
stages.append(self._kube_host_uncordon_stage())
stages.extend([
self._kube_upgrade_complete_stage(),
self._kube_post_application_update_stage(),
self._kube_upgrade_cleanup_stage(),
])
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
@ -769,6 +835,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
stages.append(self._kube_host_uncordon_stage())
stages.extend([
self._kube_upgrade_complete_stage(),
self._kube_post_application_update_stage(),
self._kube_upgrade_cleanup_stage(),
])
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
@ -801,6 +868,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
stages.append(self._kube_host_uncordon_stage())
stages.extend([
self._kube_upgrade_complete_stage(),
self._kube_post_application_update_stage(),
self._kube_upgrade_cleanup_stage(),
])
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
@ -832,6 +900,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
stages.append(self._kube_host_uncordon_stage())
stages.extend([
self._kube_upgrade_complete_stage(),
self._kube_post_application_update_stage(),
self._kube_upgrade_cleanup_stage(),
])
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
@ -863,6 +932,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
stages.append(self._kube_host_uncordon_stage())
stages.extend([
self._kube_upgrade_complete_stage(),
self._kube_post_application_update_stage(),
self._kube_upgrade_cleanup_stage(),
])
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
@ -892,6 +962,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
stages.append(self._kube_host_uncordon_stage())
stages.extend([
self._kube_upgrade_complete_stage(),
self._kube_post_application_update_stage(),
self._kube_upgrade_cleanup_stage(),
])
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
@ -919,6 +990,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
stages.append(self._kube_host_uncordon_stage())
stages.extend([
self._kube_upgrade_complete_stage(),
self._kube_post_application_update_stage(),
self._kube_upgrade_cleanup_stage(),
])
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
@ -946,10 +1018,42 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
stages.append(self._kube_host_uncordon_stage())
stages.extend([
self._kube_upgrade_complete_stage(),
self._kube_post_application_update_stage(),
self._kube_upgrade_cleanup_stage(),
])
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
def test_resume_after_post_app_update_failed(self):
"""
Test the kube_upgrade strategy creation when the upgrade had previously
stopped with 'post-updating-apps-failed'
It is expected to resume at the post application update stage.
"""
kube_upgrade = self._create_kube_upgrade_obj(
KUBE_UPGRADE_STATE.KUBE_POST_UPDATING_APPS_FAILED,
self.default_from_version,
self.default_to_version)
stages = [
self._kube_post_application_update_stage(),
self._kube_upgrade_cleanup_stage(),
]
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
def test_resume_after_post_app_update_succeeded(self):
"""
Test the kube_upgrade strategy creation when the upgrade had previously
stopped with 'post-updated-apps'.
It is expected to resume at the cleanup stage.
"""
kube_upgrade = self._create_kube_upgrade_obj(
KUBE_UPGRADE_STATE.KUBE_POST_UPDATED_APPS,
self.default_from_version,
self.default_to_version)
stages = [
self._kube_upgrade_cleanup_stage(),
]
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
@mock.patch('nfv_vim.event_log._instance._event_issue',
sw_update_testcase.fake_event_issue)

2
nfv/nfv-vim/nfv_vim/nfvi/__init__.py
View File

@ -117,6 +117,8 @@ from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_host_cordon # no
from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_host_uncordon # noqa: F401
from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_host_upgrade_control_plane # noqa: F401
from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_host_upgrade_kubelet # noqa: F401
from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_post_application_update # noqa: F401
from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_pre_application_update # noqa: F401
from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_rootca_update_abort # noqa: F401
from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_rootca_update_complete # noqa: F401
from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_rootca_update_generate_cert # noqa: F401

20
nfv/nfv-vim/nfv_vim/nfvi/_nfvi_infrastructure_module.py
View File

@ -321,6 +321,26 @@ def nfvi_kube_upgrade_start(to_version, force, alarm_ignore_list, callback):
return cmd_id
def nfvi_kube_pre_application_update(callback):
"""
Kube Upgrade Pre Application Update
"""
cmd_id = _infrastructure_plugin.invoke_plugin(
'kube_pre_application_update',
callback=callback)
return cmd_id
def nfvi_kube_post_application_update(callback):
"""
Kube Upgrade Post Application Update
"""
cmd_id = _infrastructure_plugin.invoke_plugin(
'kube_post_application_update',
callback=callback)
return cmd_id
def nfvi_get_kube_host_upgrade_list(callback):
"""
Get kube host upgrade list

8
nfv/nfv-vim/nfv_vim/nfvi/objects/v1/_kube_upgrade.py
View File

@ -37,12 +37,16 @@ class KubeUpgradeState(Constants):
Maintaining the same order as defined in kubernetes.py
"""
KUBE_UPGRADE_STARTING = Constant('upgrade-starting')
KUBE_UPGRADE_STARTING_FAILED = Constant('upgrade-starting-failed')
KUBE_UPGRADE_STARTED = Constant('upgrade-started')
KUBE_UPGRADE_DOWNLOADING_IMAGES = Constant('downloading-images')
KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED = Constant('downloading-images-failed')
KUBE_UPGRADE_DOWNLOADED_IMAGES = Constant('downloaded-images')
KUBE_PRE_UPDATING_APPS = Constant('pre-updating-apps')
KUBE_PRE_UPDATING_APPS_FAILED = Constant('pre-updating-apps-failed')
KUBE_PRE_UPDATED_APPS = Constant('pre-updated-apps')
KUBE_POST_UPDATING_APPS = Constant('post-updating-apps')
KUBE_POST_UPDATING_APPS_FAILED = Constant('post-updating-apps-failed')
KUBE_POST_UPDATED_APPS = Constant('post-updated-apps')
KUBE_UPGRADING_NETWORKING = Constant('upgrading-networking')
KUBE_UPGRADING_NETWORKING_FAILED = Constant('upgrading-networking-failed')
KUBE_UPGRADED_NETWORKING = Constant('upgraded-networking')

2
nfv/nfv-vim/nfv_vim/strategy/__init__.py
View File

@ -21,6 +21,8 @@ from nfv_vim.strategy._strategy_steps import KubeHostCordonStep # noqa: F401
from nfv_vim.strategy._strategy_steps import KubeHostUncordonStep # noqa: F401
from nfv_vim.strategy._strategy_steps import KubeHostUpgradeControlPlaneStep # noqa: F401
from nfv_vim.strategy._strategy_steps import KubeHostUpgradeKubeletStep # noqa: F401
from nfv_vim.strategy._strategy_steps import KubePostApplicationUpdateStep # noqa: F401
from nfv_vim.strategy._strategy_steps import KubePreApplicationUpdateStep # noqa: F401
from nfv_vim.strategy._strategy_steps import KubeRootcaUpdateCompleteStep # noqa: F401
from nfv_vim.strategy._strategy_steps import KubeRootcaUpdateGenerateCertStep # noqa: F401
from nfv_vim.strategy._strategy_steps import KubeRootcaUpdateHostTrustBothcasStep # noqa: F401

62
nfv/nfv-vim/nfv_vim/strategy/_strategy.py
View File

@ -3211,7 +3211,23 @@ class KubeUpgradeStrategy(SwUpdateStrategy,
strategy.STRATEGY_STAGE_NAME.KUBE_UPGRADE_DOWNLOAD_IMAGES)
stage.add_step(strategy.KubeUpgradeDownloadImagesStep())
self.apply_phase.add_stage(stage)
# Next stage after download images is upgrade networking
# Next stage after download images is pre application update
self._add_kube_pre_application_update_stage()
def _add_kube_pre_application_update_stage(self):
"""
Add kube pre application update stage.
This stage only occurs after download images
It then proceeds to the next stage
"""
from nfv_vim import strategy
stage = strategy.StrategyStage(
strategy.STRATEGY_STAGE_NAME.KUBE_PRE_APPLICATION_UPDATE)
stage.add_step(strategy.KubePreApplicationUpdateStep())
self.apply_phase.add_stage(stage)
# Next stage after pre application update is upgrade networking
self._add_kube_upgrade_networking_stage()
def _add_kube_upgrade_networking_stage(self):
@ -3496,13 +3512,29 @@ class KubeUpgradeStrategy(SwUpdateStrategy,
strategy.STRATEGY_STAGE_NAME.KUBE_UPGRADE_COMPLETE)
stage.add_step(strategy.KubeUpgradeCompleteStep())
self.apply_phase.add_stage(stage)
# stage after kube upgrade complete stage, cleans up the kube upgrade
# Next stage after upgrade complete is post application update
self._add_kube_post_application_update_stage()
def _add_kube_post_application_update_stage(self):
"""
Add kube post application update stage.
This stage occurs after the upgrade is completed
It then proceeds to the next stage
"""
from nfv_vim import strategy
stage = strategy.StrategyStage(
strategy.STRATEGY_STAGE_NAME.KUBE_POST_APPLICATION_UPDATE)
stage.add_step(strategy.KubePostApplicationUpdateStep())
self.apply_phase.add_stage(stage)
# Next stage after post application update is upgrade cleanup
self._add_kube_upgrade_cleanup_stage()
def _add_kube_upgrade_cleanup_stage(self):
"""
kube upgrade cleanup stage deletes the kube upgrade.
This stage occurs after all kube upgrade is completed
This stage occurs after the post application update stage
"""
from nfv_vim import strategy
stage = strategy.StrategyStage(
@ -3563,10 +3595,6 @@ class KubeUpgradeStrategy(SwUpdateStrategy,
# Note: there are no resume states for actions that are still running
# ie: KUBE_UPGRADE_DOWNLOADING_IMAGES
RESUME_STATE = {
# If upgrade start failed, allow to restart
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_STARTING_FAILED:
self._add_kube_upgrade_start_stage,
# after upgrade-started -> download images
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_STARTED:
self._add_kube_upgrade_download_images_stage,
@ -3575,8 +3603,16 @@ class KubeUpgradeStrategy(SwUpdateStrategy,
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED:
self._add_kube_upgrade_download_images_stage,
# After downloading images -> upgrade networking
# After downloading images -> pre update applications
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_DOWNLOADED_IMAGES:
self._add_kube_pre_application_update_stage,
# if pre updating apps failed, resume at pre updating apps
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_PRE_UPDATING_APPS_FAILED:
self._add_kube_pre_application_update_stage,
# After pre updating apps -> upgrade networking
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_PRE_UPDATED_APPS:
self._add_kube_upgrade_networking_stage,
# if networking state failed, resync at networking state
@ -3631,8 +3667,16 @@ class KubeUpgradeStrategy(SwUpdateStrategy,
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_HOST_UNCORDON_COMPLETE:
self._add_kube_upgrade_complete_stage,
# upgrade is completed, delete the upgrade
# upgrade is completed, post update apps
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_COMPLETE:
self._add_kube_post_application_update_stage,
# If post updating apps failed, resume at post application update stage
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_POST_UPDATING_APPS_FAILED:
self._add_kube_post_application_update_stage,
# After post updating apps, delete the upgrade
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_POST_UPDATED_APPS:
self._add_kube_upgrade_cleanup_stage,
}

2
nfv/nfv-vim/nfv_vim/strategy/_strategy_stages.py
View File

@ -63,6 +63,7 @@ class StrategyStageNames(Constants):
KUBE_UPGRADE_QUERY = Constant('kube-upgrade-query')
KUBE_UPGRADE_START = Constant('kube-upgrade-start')
KUBE_UPGRADE_DOWNLOAD_IMAGES = Constant('kube-upgrade-download-images')
KUBE_PRE_APPLICATION_UPDATE = Constant('kube-pre-application-update')
KUBE_UPGRADE_FIRST_CONTROL_PLANE = \
Constant('kube-upgrade-first-control-plane')
KUBE_UPGRADE_NETWORKING = Constant('kube-upgrade-networking')
@ -73,6 +74,7 @@ class StrategyStageNames(Constants):
KUBE_UPGRADE_KUBELETS_CONTROLLERS = \
Constant('kube-upgrade-kubelets-controllers')
KUBE_UPGRADE_KUBELETS_WORKERS = Constant('kube-upgrade-kubelets-workers')
KUBE_POST_APPLICATION_UPDATE = Constant('kube-post-application-update')
KUBE_UPGRADE_COMPLETE = Constant('kube-upgrade-complete')
KUBE_UPGRADE_CLEANUP = Constant('kube-upgrade-cleanup')

93
nfv/nfv-vim/nfv_vim/strategy/_strategy_steps.py
View File

@ -88,6 +88,8 @@ class StrategyStepNames(Constants):
KUBE_HOST_UPGRADE_CONTROL_PLANE = \
Constant('kube-host-upgrade-control-plane')
KUBE_HOST_UPGRADE_KUBELET = Constant('kube-host-upgrade-kubelet')
KUBE_PRE_APPLICATION_UPDATE = Constant('kube-pre-application-update')
KUBE_POST_APPLICATION_UPDATE = Constant('kube-post-application-update')
# system config update specific steps
QUERY_SYSTEM_CONFIG_UPDATE_HOSTS = Constant('query-system-config-update-hosts')
SYSTEM_CONFIG_UPDATE_HOSTS = Constant('system-config-update-hosts')
@ -4134,8 +4136,7 @@ class KubeUpgradeStartStep(AbstractKubeUpgradeStep):
super(KubeUpgradeStartStep, self).__init__(
STRATEGY_STEP_NAME.KUBE_UPGRADE_START,
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_STARTED,
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_STARTING_FAILED,
timeout_in_secs=1800)
None) # there is no failure state if upgrade-start fails
# next 2 attributes must be persisted through from_dict/as_dict
self._to_version = to_version
self._force = force
@ -4171,9 +4172,13 @@ class KubeUpgradeStartStep(AbstractKubeUpgradeStep):
response = (yield)
DLOG.debug("%s callback response=%s." % (self._name, response))
# kube-upgrade-start will return a result when it completes,
# so we do not want to use handle_event
if response['completed']:
if self.strategy is not None:
self.strategy.nfvi_kube_upgrade = response['result-data']
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
self.stage.step_complete(result, "")
else:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, response['reason'])
@ -4363,6 +4368,88 @@ class KubeUpgradeDownloadImagesStep(AbstractKubeUpgradeStep):
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
class KubePreApplicationUpdateStep(AbstractKubeUpgradeStep):
"""Kube Pre Application Update - Strategy Step"""
def __init__(self):
from nfv_vim import nfvi
super(KubePreApplicationUpdateStep, self).__init__(
STRATEGY_STEP_NAME.KUBE_PRE_APPLICATION_UPDATE,
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_PRE_UPDATED_APPS,
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_PRE_UPDATING_APPS_FAILED,
timeout_in_secs=1800)
def abort(self):
"""
Returns the abort step related to this step
"""
return [KubeUpgradeAbortStep()]
@coroutine
def _response_callback(self):
"""Kube Pre Application Update - Callback"""
response = (yield)
DLOG.debug("%s callback response=%s." % (self._name, response))
if response['completed']:
if self.strategy is not None:
self.strategy.nfvi_kube_upgrade = response['result-data']
else:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, response['reason'])
def apply(self):
"""Kube Pre Application Update """
from nfv_vim import nfvi
DLOG.info("Step (%s) apply." % self._name)
nfvi.nfvi_kube_pre_application_update(self._response_callback())
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
class KubePostApplicationUpdateStep(AbstractKubeUpgradeStep):
"""Kube Post Application Update - Strategy Step"""
def __init__(self):
from nfv_vim import nfvi
super(KubePostApplicationUpdateStep, self).__init__(
STRATEGY_STEP_NAME.KUBE_POST_APPLICATION_UPDATE,
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_POST_UPDATED_APPS,
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_POST_UPDATING_APPS_FAILED,
timeout_in_secs=1800)
def abort(self):
"""
Returns the abort step related to this step
"""
return [KubeUpgradeAbortStep()]
@coroutine
def _response_callback(self):
"""Kube Post Application Update - Callback"""
response = (yield)
DLOG.debug("%s callback response=%s." % (self._name, response))
if response['completed']:
if self.strategy is not None:
self.strategy.nfvi_kube_upgrade = response['result-data']
else:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, response['reason'])
def apply(self):
"""Kube Post Application Update """
from nfv_vim import nfvi
DLOG.info("Step (%s) apply." % self._name)
nfvi.nfvi_kube_post_application_update(self._response_callback())
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
class KubeUpgradeNetworkingStep(AbstractKubeUpgradeStep):
"""Kube Upgrade Networking - Strategy Step"""
@ -4894,6 +4981,8 @@ def strategy_step_rebuild_from_dict(data):
STRATEGY_STEP_NAME.KUBE_UPGRADE_NETWORKING: KubeUpgradeNetworkingStep,
STRATEGY_STEP_NAME.KUBE_UPGRADE_STORAGE: KubeUpgradeStorageStep,
STRATEGY_STEP_NAME.KUBE_UPGRADE_START: KubeUpgradeStartStep,
STRATEGY_STEP_NAME.KUBE_PRE_APPLICATION_UPDATE: KubePreApplicationUpdateStep,
STRATEGY_STEP_NAME.KUBE_POST_APPLICATION_UPDATE: KubePostApplicationUpdateStep,
STRATEGY_STEP_NAME.QUERY_KUBE_HOST_UPGRADE: QueryKubeHostUpgradeStep,
STRATEGY_STEP_NAME.QUERY_KUBE_UPGRADE: QueryKubeUpgradeStep,
STRATEGY_STEP_NAME.QUERY_KUBE_VERSIONS: QueryKubeVersionsStep,