starlingx
/
nfv
Code Issues Proposed changes

Add new application update steps for k8s upgrades 

Add two new steps to the orchestrated Kubernetes upgrade:
  * 'pre-application-update': update StarlingX applications before the
    new Kubernetes version is deployed.
  * 'post-application-update': update Starlingx applications after the
    new Kuberneter version is deployed.

This was implemented according to the changes introduced to sysinv by
the following commit:
https://review.opendev.org/c/starlingx/config/+/909172

Unit tests have also been updated to account for the new steps.

Test Plan:
PASS: build-pkgs -a && build-image
AIO-SX Test Cases:
PASS: Run an orchestrated Kubernetes upgrade with no apps to be updated.
PASS: Run a single-version orchestrated Kubernetes upgrade with apps to
      be updated during the pre-application-update and
      post-application-update steps.
PASS: Run a multi-version orchestrated Kubernetes upgrade with apps to
      be updated during the pre-application-update and
      post-application-update steps.
PASS: Run an orchestrated Kubernetes upgrade that fails during the
      pre-application-update step due to a missing metadata.yaml file.
      Confirm that the upgrade was successfully aborted.
PASS: Run an orchestrated Kubernetes upgrade that fails during the
      post-application-update step due to a missing metadata.yaml file.
      Confirm that the upgrade was successfully aborted.
AIO-DX Test Cases:
PASS: Run an orchestrated Kubernetes upgrade with no apps to be updated.
PASS: Run a single-version orchestrated Kubernetes upgrade with apps to
      be updated during the pre-application-update and
      post-application-update steps.
PASS: Run an orchestrated Kubernetes upgrade that fails during the
      pre-application-update step due to a missing metadata.yaml file.
      Fix the application tarball and resume the upgrade.
      Confirm that the upgrade resumes from the same step.
      Confirm that the upgrade finishes successfully.
PASS: Run an orchestrated Kubernetes upgrade that fails during the
      post-application-update step due to a missing metadata.yaml file.
      Fix the application tarball and resume the upgrade.
      Confirm that the upgrade resumes from the same step.
      Confirm that the upgrade finishes successfully.

Story: 2010929
Task: 49791
Depends-On: https://review.opendev.org/c/starlingx/config/+/909172

Change-Id: I5217db1cb744320979afcd161cb436bf089ccc9d
Signed-off-by: Igor Soares <Igor.PiresSoares@windriver.com>
This commit is contained in:
Igor Soares 2024-03-27 19:16:15 -03:00
parent 6142d9f116
commit 3e06391c30
10 changed files with 443 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

118
nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_infrastructure_api.py
View File

@ -2017,6 +2017,124 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
callback.send(response)
callback.close()
def kube_pre_application_update(self, future, callback):
"""
Start kube pre application update
"""
response = dict()
response['completed'] = False
response['reason'] = ''
action_type = 'kube-pre-application-update'
try:
future.set_timeouts(config.CONF.get('nfvi-timeouts', None))
if self._platform_token is None or \
self._platform_token.is_expired():
future.work(openstack.get_token, self._platform_directory)
future.result = (yield)
if not future.result.is_complete() or \
future.result.data is None:
DLOG.error("OpenStack get-token did not complete.")
return
self._platform_token = future.result.data
future.work(sysinv.kube_pre_application_update,
self._platform_token)
future.result = (yield)
if not future.result.is_complete():
DLOG.error("%s did not complete." % action_type)
return
kube_upgrade_data = future.result.data
kube_upgrade_obj = nfvi.objects.v1.KubeUpgrade(
kube_upgrade_data['state'],
kube_upgrade_data['from_version'],
kube_upgrade_data['to_version'])
response['result-data'] = kube_upgrade_obj
response['completed'] = True
except exceptions.OpenStackRestAPIException as e:
if httplib.UNAUTHORIZED == e.http_status_code:
response['error-code'] = nfvi.NFVI_ERROR_CODE.TOKEN_EXPIRED
if self._platform_token is not None:
self._platform_token.set_expired()
else:
DLOG.exception("Caught API exception while trying %s. error=%s"
% (action_type, e))
except Exception as e:
DLOG.exception("Caught exception while trying %s. error=%s"
% (action_type, e))
finally:
callback.send(response)
callback.close()
def kube_post_application_update(self, future, callback):
"""
Start kube post application update
"""
response = dict()
response['completed'] = False
response['reason'] = ''
action_type = 'kube-post-application-update'
try:
future.set_timeouts(config.CONF.get('nfvi-timeouts', None))
if self._platform_token is None or \
self._platform_token.is_expired():
future.work(openstack.get_token, self._platform_directory)
future.result = (yield)
if not future.result.is_complete() or \
future.result.data is None:
DLOG.error("OpenStack get-token did not complete.")
return
self._platform_token = future.result.data
future.work(sysinv.kube_post_application_update,
self._platform_token)
future.result = (yield)
if not future.result.is_complete():
DLOG.error("%s did not complete." % action_type)
return
kube_upgrade_data = future.result.data
kube_upgrade_obj = nfvi.objects.v1.KubeUpgrade(
kube_upgrade_data['state'],
kube_upgrade_data['from_version'],
kube_upgrade_data['to_version'])
response['result-data'] = kube_upgrade_obj
response['completed'] = True
except exceptions.OpenStackRestAPIException as e:
if httplib.UNAUTHORIZED == e.http_status_code:
response['error-code'] = nfvi.NFVI_ERROR_CODE.TOKEN_EXPIRED
if self._platform_token is not None:
self._platform_token.set_expired()
else:
DLOG.exception("Caught API exception while trying %s. error=%s"
% (action_type, e))
except Exception as e:
DLOG.exception("Caught exception while trying %s. error=%s"
% (action_type, e))
finally:
callback.send(response)
callback.close()
def kube_upgrade_networking(self, future, callback):
"""
Start kube upgrade networking

14
nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/sysinv.py
View File

@ -436,6 +436,13 @@ def kube_upgrade_download_images(token):
return _patch_kube_upgrade_state(token, "downloading-images")
def kube_pre_application_update(token):
"""
Ask System Inventory to kube pre application update
"""
return _patch_kube_upgrade_state(token, "pre-updating-apps")
def kube_upgrade_networking(token):
"""
Ask System Inventory to kube upgrade networking
@ -583,6 +590,13 @@ def upgrade_activate(token):
return response
def kube_post_application_update(token):
"""
Ask System Inventory to kube post application update
"""
return _patch_kube_upgrade_state(token, "post-updating-apps")
def upgrade_complete(token):
"""
Ask System Inventory to complete an upgrade

178
nfv/nfv-tests/nfv_unit_tests/tests/test_kube_upgrade_strategy.py
View File

@ -269,8 +269,7 @@ class ApplyStageMixin(object):
'total_steps': 1,
'steps': [
{'name': 'kube-upgrade-start',
'success_state': 'upgrade-started',
'fail_state': 'upgrade-starting-failed'},
'success_state': 'upgrade-started'},
],
}
@ -285,6 +284,17 @@ class ApplyStageMixin(object):
],
}
def _kube_pre_application_update_stage(self):
return {
'name': 'kube-pre-application-update',
'total_steps': 1,
'steps': [
{'name': 'kube-pre-application-update',
'success_state': 'pre-updated-apps',
'fail_state': 'pre-updating-apps-failed'},
],
}
def _kube_host_cordon_stage(self, ver="N/A"):
return {
'name': 'kube-host-cordon',
@ -362,6 +372,17 @@ class ApplyStageMixin(object):
],
}
def _kube_post_application_update_stage(self):
return {
'name': 'kube-post-application-update',
'total_steps': 1,
'steps': [
{'name': 'kube-post-application-update',
'success_state': 'post-updated-apps',
'fail_state': 'post-updating-apps-failed'},
],
}
def _kube_upgrade_cleanup_stage(self):
return {
'name': 'kube-upgrade-cleanup',
@ -496,6 +517,7 @@ class ApplyStageMixin(object):
storage_list=None,
add_start=True,
add_download=True,
add_pre_app_update=True,
add_networking=True,
add_storage=True,
add_cordon=True,
@ -503,6 +525,7 @@ class ApplyStageMixin(object):
add_second_control_plane=True,
add_kubelets=True,
add_uncordon=True,
add_post_app_update=True,
add_complete=True,
add_cleanup=True):
"""The order of the host_list determines the kubelets"""
@ -518,6 +541,8 @@ class ApplyStageMixin(object):
stages.append(self._kube_upgrade_start_stage())
if add_download:
stages.append(self._kube_upgrade_download_images_stage())
if add_pre_app_update:
stages.append(self._kube_pre_application_update_stage())
if add_networking:
stages.append(self._kube_upgrade_networking_stage())
if add_storage:
@ -539,6 +564,8 @@ class ApplyStageMixin(object):
stages.append(self._kube_host_uncordon_stage())
if add_complete:
stages.append(self._kube_upgrade_complete_stage())
if add_post_app_update:
stages.append(self._kube_post_application_update_stage())
if add_cleanup:
stages.append(self._kube_upgrade_cleanup_stage())
return stages
@ -582,7 +609,7 @@ class ApplyStageMixin(object):
"""
Test the kube_upgrade strategy creation when the upgrade had previously
stopped after upgrade-completed.
It is expected to resume at the cleanup stage
It is expected to resume at the post application update stage
"""
kube_upgrade = self._create_kube_upgrade_obj(
KUBE_UPGRADE_STATE.KUBE_UPGRADE_COMPLETE,
@ -590,6 +617,7 @@ class ApplyStageMixin(object):
self.default_to_version)
# not using build_stage_list utility since the list of stages is small
stages = [
self._kube_post_application_update_stage(),
self._kube_upgrade_cleanup_stage(),
]
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
@ -624,39 +652,6 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
self.worker_list = []
self.storage_list = []
def test_resume_after_starting_failed(self):
"""
Test the kube_upgrade strategy creation when the upgrade had previously
stopped with 'upgrade-starting-failed'
It is expected to resume at the 'upgrade-starting' stage
"""
kube_upgrade = self._create_kube_upgrade_obj(
KUBE_UPGRADE_STATE.KUBE_UPGRADE_STARTING_FAILED,
self.default_from_version,
self.default_to_version)
stages = [
self._kube_upgrade_start_stage(),
self._kube_upgrade_download_images_stage(),
self._kube_upgrade_networking_stage(),
self._kube_upgrade_storage_stage(),
]
if self.is_simplex():
stages.append(self._kube_host_cordon_stage())
for ver in self.kube_versions:
stages.append(self._kube_upgrade_first_control_plane_stage(ver))
stages.extend(self._kube_upgrade_kubelet_stages(
ver,
self.std_controller_list,
self.aio_controller_list,
self.worker_list))
if self.is_simplex():
stages.append(self._kube_host_uncordon_stage())
stages.extend([
self._kube_upgrade_complete_stage(),
self._kube_upgrade_cleanup_stage(),
])
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
def test_resume_after_download_images_failed(self):
"""
Test the kube_upgrade strategy creation when the upgrade had previously
@ -669,6 +664,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
self.default_to_version)
stages = [
self._kube_upgrade_download_images_stage(),
self._kube_pre_application_update_stage(),
self._kube_upgrade_networking_stage(),
self._kube_upgrade_storage_stage(),
]
@ -685,6 +681,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
stages.append(self._kube_host_uncordon_stage())
stages.extend([
self._kube_upgrade_complete_stage(),
self._kube_post_application_update_stage(),
self._kube_upgrade_cleanup_stage(),
])
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
@ -693,12 +690,79 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
"""
Test the kube_upgrade strategy creation when the upgrade had previously
stopped with 'downloaded-images'
It is expected to resume at the 'first control plane' stage.
It is expected to resume at the 'pre application update' stage.
"""
kube_upgrade = self._create_kube_upgrade_obj(
KUBE_UPGRADE_STATE.KUBE_UPGRADE_DOWNLOADED_IMAGES,
self.default_from_version,
self.default_to_version)
stages = [
self._kube_pre_application_update_stage(),
self._kube_upgrade_networking_stage(),
self._kube_upgrade_storage_stage(),
]
if self.is_simplex():
stages.append(self._kube_host_cordon_stage())
for ver in self.kube_versions:
stages.append(self._kube_upgrade_first_control_plane_stage(
ver))
stages.extend(self._kube_upgrade_kubelet_stages(
ver,
self.std_controller_list,
self.aio_controller_list,
self.worker_list))
if self.is_simplex():
stages.append(self._kube_host_uncordon_stage())
stages.extend([
self._kube_upgrade_complete_stage(),
self._kube_post_application_update_stage(),
self._kube_upgrade_cleanup_stage(),
])
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
def test_resume_after_pre_app_update_failed(self):
"""
Test the kube_upgrade strategy creation when the upgrade had previously
stopped with 'pre-updating-apps-failed'.
It is expected to resume at the pre application update stage.
"""
kube_upgrade = self._create_kube_upgrade_obj(
KUBE_UPGRADE_STATE.KUBE_PRE_UPDATING_APPS_FAILED,
self.default_from_version,
self.default_to_version)
stages = [
self._kube_pre_application_update_stage(),
self._kube_upgrade_networking_stage(),
self._kube_upgrade_storage_stage(),
]
if self.is_simplex():
stages.append(self._kube_host_cordon_stage())
for ver in self.kube_versions:
stages.append(self._kube_upgrade_first_control_plane_stage(ver))
stages.extend(self._kube_upgrade_kubelet_stages(
ver,
self.std_controller_list,
self.aio_controller_list,
self.worker_list))
if self.is_simplex():
stages.append(self._kube_host_uncordon_stage())
stages.extend([
self._kube_upgrade_complete_stage(),
self._kube_post_application_update_stage(),
self._kube_upgrade_cleanup_stage(),
])
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
def test_resume_after_pre_app_update_succeeded(self):
"""
Test the kube_upgrade strategy creation when the upgrade had previously
stopped with 'pre-updated-apps'.
It is expected to resume at the upgrade networking stage.
"""
kube_upgrade = self._create_kube_upgrade_obj(
KUBE_UPGRADE_STATE.KUBE_PRE_UPDATED_APPS,
self.default_from_version,
self.default_to_version)
stages = [
self._kube_upgrade_networking_stage(),
self._kube_upgrade_storage_stage(),
@ -717,6 +781,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
stages.append(self._kube_host_uncordon_stage())
stages.extend([
self._kube_upgrade_complete_stage(),
self._kube_post_application_update_stage(),
self._kube_upgrade_cleanup_stage(),
])
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
@ -744,6 +809,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
stages.append(self._kube_host_uncordon_stage())
stages.extend([
self._kube_upgrade_complete_stage(),
self._kube_post_application_update_stage(),
self._kube_upgrade_cleanup_stage(),
])
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
@ -769,6 +835,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
stages.append(self._kube_host_uncordon_stage())
stages.extend([
self._kube_upgrade_complete_stage(),
self._kube_post_application_update_stage(),
self._kube_upgrade_cleanup_stage(),
])
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
@ -801,6 +868,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
stages.append(self._kube_host_uncordon_stage())
stages.extend([
self._kube_upgrade_complete_stage(),
self._kube_post_application_update_stage(),
self._kube_upgrade_cleanup_stage(),
])
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
@ -832,6 +900,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
stages.append(self._kube_host_uncordon_stage())
stages.extend([
self._kube_upgrade_complete_stage(),
self._kube_post_application_update_stage(),
self._kube_upgrade_cleanup_stage(),
])
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
@ -863,6 +932,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
stages.append(self._kube_host_uncordon_stage())
stages.extend([
self._kube_upgrade_complete_stage(),
self._kube_post_application_update_stage(),
self._kube_upgrade_cleanup_stage(),
])
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
@ -892,6 +962,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
stages.append(self._kube_host_uncordon_stage())
stages.extend([
self._kube_upgrade_complete_stage(),
self._kube_post_application_update_stage(),
self._kube_upgrade_cleanup_stage(),
])
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
@ -919,6 +990,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
stages.append(self._kube_host_uncordon_stage())
stages.extend([
self._kube_upgrade_complete_stage(),
self._kube_post_application_update_stage(),
self._kube_upgrade_cleanup_stage(),
])
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
@ -946,10 +1018,42 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
stages.append(self._kube_host_uncordon_stage())
stages.extend([
self._kube_upgrade_complete_stage(),
self._kube_post_application_update_stage(),
self._kube_upgrade_cleanup_stage(),
])
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
def test_resume_after_post_app_update_failed(self):
"""
Test the kube_upgrade strategy creation when the upgrade had previously
stopped with 'post-updating-apps-failed'
It is expected to resume at the post application update stage.
"""
kube_upgrade = self._create_kube_upgrade_obj(
KUBE_UPGRADE_STATE.KUBE_POST_UPDATING_APPS_FAILED,
self.default_from_version,
self.default_to_version)
stages = [
self._kube_post_application_update_stage(),
self._kube_upgrade_cleanup_stage(),
]
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
def test_resume_after_post_app_update_succeeded(self):
"""
Test the kube_upgrade strategy creation when the upgrade had previously
stopped with 'post-updated-apps'.
It is expected to resume at the cleanup stage.
"""
kube_upgrade = self._create_kube_upgrade_obj(
KUBE_UPGRADE_STATE.KUBE_POST_UPDATED_APPS,
self.default_from_version,
self.default_to_version)
stages = [
self._kube_upgrade_cleanup_stage(),
]
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
@mock.patch('nfv_vim.event_log._instance._event_issue',
sw_update_testcase.fake_event_issue)

2
nfv/nfv-vim/nfv_vim/nfvi/__init__.py
View File

@ -117,6 +117,8 @@ from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_host_cordon # no
from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_host_uncordon # noqa: F401
from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_host_upgrade_control_plane # noqa: F401
from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_host_upgrade_kubelet # noqa: F401
from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_post_application_update # noqa: F401
from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_pre_application_update # noqa: F401
from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_rootca_update_abort # noqa: F401
from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_rootca_update_complete # noqa: F401
from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_rootca_update_generate_cert # noqa: F401

20
nfv/nfv-vim/nfv_vim/nfvi/_nfvi_infrastructure_module.py
View File

@ -321,6 +321,26 @@ def nfvi_kube_upgrade_start(to_version, force, alarm_ignore_list, callback):
return cmd_id
def nfvi_kube_pre_application_update(callback):
"""
Kube Upgrade Pre Application Update
"""
cmd_id = _infrastructure_plugin.invoke_plugin(
'kube_pre_application_update',
callback=callback)
return cmd_id
def nfvi_kube_post_application_update(callback):
"""
Kube Upgrade Post Application Update
"""
cmd_id = _infrastructure_plugin.invoke_plugin(
'kube_post_application_update',
callback=callback)
return cmd_id
def nfvi_get_kube_host_upgrade_list(callback):
"""
Get kube host upgrade list

8
nfv/nfv-vim/nfv_vim/nfvi/objects/v1/_kube_upgrade.py
View File

@ -37,12 +37,16 @@ class KubeUpgradeState(Constants):
Maintaining the same order as defined in kubernetes.py
"""
KUBE_UPGRADE_STARTING = Constant('upgrade-starting')
KUBE_UPGRADE_STARTING_FAILED = Constant('upgrade-starting-failed')
KUBE_UPGRADE_STARTED = Constant('upgrade-started')
KUBE_UPGRADE_DOWNLOADING_IMAGES = Constant('downloading-images')
KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED = Constant('downloading-images-failed')
KUBE_UPGRADE_DOWNLOADED_IMAGES = Constant('downloaded-images')
KUBE_PRE_UPDATING_APPS = Constant('pre-updating-apps')
KUBE_PRE_UPDATING_APPS_FAILED = Constant('pre-updating-apps-failed')
KUBE_PRE_UPDATED_APPS = Constant('pre-updated-apps')
KUBE_POST_UPDATING_APPS = Constant('post-updating-apps')
KUBE_POST_UPDATING_APPS_FAILED = Constant('post-updating-apps-failed')
KUBE_POST_UPDATED_APPS = Constant('post-updated-apps')
KUBE_UPGRADING_NETWORKING = Constant('upgrading-networking')
KUBE_UPGRADING_NETWORKING_FAILED = Constant('upgrading-networking-failed')
KUBE_UPGRADED_NETWORKING = Constant('upgraded-networking')

2
nfv/nfv-vim/nfv_vim/strategy/__init__.py
View File

@ -21,6 +21,8 @@ from nfv_vim.strategy._strategy_steps import KubeHostCordonStep # noqa: F401
from nfv_vim.strategy._strategy_steps import KubeHostUncordonStep # noqa: F401
from nfv_vim.strategy._strategy_steps import KubeHostUpgradeControlPlaneStep # noqa: F401
from nfv_vim.strategy._strategy_steps import KubeHostUpgradeKubeletStep # noqa: F401
from nfv_vim.strategy._strategy_steps import KubePostApplicationUpdateStep # noqa: F401
from nfv_vim.strategy._strategy_steps import KubePreApplicationUpdateStep # noqa: F401
from nfv_vim.strategy._strategy_steps import KubeRootcaUpdateCompleteStep # noqa: F401
from nfv_vim.strategy._strategy_steps import KubeRootcaUpdateGenerateCertStep # noqa: F401
from nfv_vim.strategy._strategy_steps import KubeRootcaUpdateHostTrustBothcasStep # noqa: F401

62
nfv/nfv-vim/nfv_vim/strategy/_strategy.py
View File

@ -3211,7 +3211,23 @@ class KubeUpgradeStrategy(SwUpdateStrategy,
strategy.STRATEGY_STAGE_NAME.KUBE_UPGRADE_DOWNLOAD_IMAGES)
stage.add_step(strategy.KubeUpgradeDownloadImagesStep())
self.apply_phase.add_stage(stage)
# Next stage after download images is upgrade networking
# Next stage after download images is pre application update
self._add_kube_pre_application_update_stage()
def _add_kube_pre_application_update_stage(self):
"""
Add kube pre application update stage.
This stage only occurs after download images
It then proceeds to the next stage
"""
from nfv_vim import strategy
stage = strategy.StrategyStage(
strategy.STRATEGY_STAGE_NAME.KUBE_PRE_APPLICATION_UPDATE)
stage.add_step(strategy.KubePreApplicationUpdateStep())
self.apply_phase.add_stage(stage)
# Next stage after pre application update is upgrade networking
self._add_kube_upgrade_networking_stage()
def _add_kube_upgrade_networking_stage(self):
@ -3496,13 +3512,29 @@ class KubeUpgradeStrategy(SwUpdateStrategy,
strategy.STRATEGY_STAGE_NAME.KUBE_UPGRADE_COMPLETE)
stage.add_step(strategy.KubeUpgradeCompleteStep())
self.apply_phase.add_stage(stage)
# stage after kube upgrade complete stage, cleans up the kube upgrade
# Next stage after upgrade complete is post application update
self._add_kube_post_application_update_stage()
def _add_kube_post_application_update_stage(self):
"""
Add kube post application update stage.
This stage occurs after the upgrade is completed
It then proceeds to the next stage
"""
from nfv_vim import strategy
stage = strategy.StrategyStage(
strategy.STRATEGY_STAGE_NAME.KUBE_POST_APPLICATION_UPDATE)
stage.add_step(strategy.KubePostApplicationUpdateStep())
self.apply_phase.add_stage(stage)
# Next stage after post application update is upgrade cleanup
self._add_kube_upgrade_cleanup_stage()
def _add_kube_upgrade_cleanup_stage(self):
"""
kube upgrade cleanup stage deletes the kube upgrade.
This stage occurs after all kube upgrade is completed
This stage occurs after the post application update stage
"""
from nfv_vim import strategy
stage = strategy.StrategyStage(
@ -3563,10 +3595,6 @@ class KubeUpgradeStrategy(SwUpdateStrategy,
# Note: there are no resume states for actions that are still running
# ie: KUBE_UPGRADE_DOWNLOADING_IMAGES
RESUME_STATE = {
# If upgrade start failed, allow to restart
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_STARTING_FAILED:
self._add_kube_upgrade_start_stage,
# after upgrade-started -> download images
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_STARTED:
self._add_kube_upgrade_download_images_stage,
@ -3575,8 +3603,16 @@ class KubeUpgradeStrategy(SwUpdateStrategy,
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED:
self._add_kube_upgrade_download_images_stage,
# After downloading images -> upgrade networking
# After downloading images -> pre update applications
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_DOWNLOADED_IMAGES:
self._add_kube_pre_application_update_stage,
# if pre updating apps failed, resume at pre updating apps
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_PRE_UPDATING_APPS_FAILED:
self._add_kube_pre_application_update_stage,
# After pre updating apps -> upgrade networking
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_PRE_UPDATED_APPS:
self._add_kube_upgrade_networking_stage,
# if networking state failed, resync at networking state
@ -3631,8 +3667,16 @@ class KubeUpgradeStrategy(SwUpdateStrategy,
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_HOST_UNCORDON_COMPLETE:
self._add_kube_upgrade_complete_stage,
# upgrade is completed, delete the upgrade
# upgrade is completed, post update apps
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_COMPLETE:
self._add_kube_post_application_update_stage,
# If post updating apps failed, resume at post application update stage
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_POST_UPDATING_APPS_FAILED:
self._add_kube_post_application_update_stage,
# After post updating apps, delete the upgrade
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_POST_UPDATED_APPS:
self._add_kube_upgrade_cleanup_stage,
}

2
nfv/nfv-vim/nfv_vim/strategy/_strategy_stages.py
View File

@ -63,6 +63,7 @@ class StrategyStageNames(Constants):
KUBE_UPGRADE_QUERY = Constant('kube-upgrade-query')
KUBE_UPGRADE_START = Constant('kube-upgrade-start')
KUBE_UPGRADE_DOWNLOAD_IMAGES = Constant('kube-upgrade-download-images')
KUBE_PRE_APPLICATION_UPDATE = Constant('kube-pre-application-update')
KUBE_UPGRADE_FIRST_CONTROL_PLANE = \
Constant('kube-upgrade-first-control-plane')
KUBE_UPGRADE_NETWORKING = Constant('kube-upgrade-networking')
@ -73,6 +74,7 @@ class StrategyStageNames(Constants):
KUBE_UPGRADE_KUBELETS_CONTROLLERS = \
Constant('kube-upgrade-kubelets-controllers')
KUBE_UPGRADE_KUBELETS_WORKERS = Constant('kube-upgrade-kubelets-workers')
KUBE_POST_APPLICATION_UPDATE = Constant('kube-post-application-update')
KUBE_UPGRADE_COMPLETE = Constant('kube-upgrade-complete')
KUBE_UPGRADE_CLEANUP = Constant('kube-upgrade-cleanup')

87
nfv/nfv-vim/nfv_vim/strategy/_strategy_steps.py
View File

@ -88,6 +88,8 @@ class StrategyStepNames(Constants):
KUBE_HOST_UPGRADE_CONTROL_PLANE = \
Constant('kube-host-upgrade-control-plane')
KUBE_HOST_UPGRADE_KUBELET = Constant('kube-host-upgrade-kubelet')
KUBE_PRE_APPLICATION_UPDATE = Constant('kube-pre-application-update')
KUBE_POST_APPLICATION_UPDATE = Constant('kube-post-application-update')
# system config update specific steps
QUERY_SYSTEM_CONFIG_UPDATE_HOSTS = Constant('query-system-config-update-hosts')
SYSTEM_CONFIG_UPDATE_HOSTS = Constant('system-config-update-hosts')
@ -4134,8 +4136,7 @@ class KubeUpgradeStartStep(AbstractKubeUpgradeStep):
super(KubeUpgradeStartStep, self).__init__(
STRATEGY_STEP_NAME.KUBE_UPGRADE_START,
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_STARTED,
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_STARTING_FAILED,
timeout_in_secs=1800)
None) # there is no failure state if upgrade-start fails
# next 2 attributes must be persisted through from_dict/as_dict
self._to_version = to_version
self._force = force
@ -4171,9 +4172,13 @@ class KubeUpgradeStartStep(AbstractKubeUpgradeStep):
response = (yield)
DLOG.debug("%s callback response=%s." % (self._name, response))
# kube-upgrade-start will return a result when it completes,
# so we do not want to use handle_event
if response['completed']:
if self.strategy is not None:
self.strategy.nfvi_kube_upgrade = response['result-data']
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
self.stage.step_complete(result, "")
else:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, response['reason'])
@ -4363,6 +4368,82 @@ class KubeUpgradeDownloadImagesStep(AbstractKubeUpgradeStep):
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
class KubePreApplicationUpdateStep(AbstractKubeUpgradeStep):
"""Kube Pre Application Update - Strategy Step"""
def __init__(self):
from nfv_vim import nfvi
super(KubePreApplicationUpdateStep, self).__init__(
STRATEGY_STEP_NAME.KUBE_PRE_APPLICATION_UPDATE,
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_PRE_UPDATED_APPS,
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_PRE_UPDATING_APPS_FAILED,
timeout_in_secs=1800)
def abort(self):
"""
Returns the abort step related to this step
"""
return [KubeUpgradeAbortStep()]
@coroutine
def _response_callback(self):
"""Kube Pre Application Update - Callback"""
response = (yield)
DLOG.debug("%s callback response=%s." % (self._name, response))
if response['completed']:
if self.strategy is not None:
self.strategy.nfvi_kube_upgrade = response['result-data']
else:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, response['reason'])
def apply(self):
"""Kube Pre Application Update """
from nfv_vim import nfvi
DLOG.info("Step (%s) apply." % self._name)
nfvi.nfvi_kube_pre_application_update(self._response_callback())
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
class KubePostApplicationUpdateStep(AbstractKubeUpgradeStep):
"""Kube Post Application Update - Strategy Step"""
def __init__(self):
from nfv_vim import nfvi
super(KubePostApplicationUpdateStep, self).__init__(
STRATEGY_STEP_NAME.KUBE_POST_APPLICATION_UPDATE,
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_POST_UPDATED_APPS,
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_POST_UPDATING_APPS_FAILED,
timeout_in_secs=1800)
@coroutine
def _response_callback(self):
"""Kube Post Application Update - Callback"""
response = (yield)
DLOG.debug("%s callback response=%s." % (self._name, response))
if response['completed']:
if self.strategy is not None:
self.strategy.nfvi_kube_upgrade = response['result-data']
else:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, response['reason'])
def apply(self):
"""Kube Post Application Update """
from nfv_vim import nfvi
DLOG.info("Step (%s) apply." % self._name)
nfvi.nfvi_kube_post_application_update(self._response_callback())
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
class KubeUpgradeNetworkingStep(AbstractKubeUpgradeStep):
"""Kube Upgrade Networking - Strategy Step"""
@ -4894,6 +4975,8 @@ def strategy_step_rebuild_from_dict(data):
STRATEGY_STEP_NAME.KUBE_UPGRADE_NETWORKING: KubeUpgradeNetworkingStep,
STRATEGY_STEP_NAME.KUBE_UPGRADE_STORAGE: KubeUpgradeStorageStep,
STRATEGY_STEP_NAME.KUBE_UPGRADE_START: KubeUpgradeStartStep,
STRATEGY_STEP_NAME.KUBE_PRE_APPLICATION_UPDATE: KubePreApplicationUpdateStep,
STRATEGY_STEP_NAME.KUBE_POST_APPLICATION_UPDATE: KubePostApplicationUpdateStep,
STRATEGY_STEP_NAME.QUERY_KUBE_HOST_UPGRADE: QueryKubeHostUpgradeStep,
STRATEGY_STEP_NAME.QUERY_KUBE_UPGRADE: QueryKubeUpgradeStep,
STRATEGY_STEP_NAME.QUERY_KUBE_VERSIONS: QueryKubeVersionsStep,