starlingx
/
nfv
Code Issues Proposed changes

Merge "Nfv upgrade orchestration for kube-upgrade-storage"

This commit is contained in:
Zuul 2024-01-09 22:03:36 +00:00 committed by Gerrit Code Review
parent 42442c85d9 b90eeb1436
commit 8af76f954d
10 changed files with 221 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_infrastructure_api.py
View File

@ -2074,6 +2074,64 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
callback.send(response)
callback.close()
def kube_upgrade_storage(self, future, callback):
"""
Start kube upgrade storage
"""
response = dict()
response['completed'] = False
response['reason'] = ''
action_type = 'kube-upgrade-storage'
try:
future.set_timeouts(config.CONF.get('nfvi-timeouts', None))
if self._platform_token is None or \
self._platform_token.is_expired():
future.work(openstack.get_token, self._platform_directory)
future.result = (yield)
if not future.result.is_complete() or \
future.result.data is None:
DLOG.error("OpenStack get-token did not complete.")
return
self._platform_token = future.result.data
future.work(sysinv.kube_upgrade_storage, self._platform_token)
future.result = (yield)
if not future.result.is_complete():
DLOG.error("%s did not complete." % action_type)
return
kube_upgrade_data = future.result.data
kube_upgrade_obj = nfvi.objects.v1.KubeUpgrade(
kube_upgrade_data['state'],
kube_upgrade_data['from_version'],
kube_upgrade_data['to_version'])
response['result-data'] = kube_upgrade_obj
response['completed'] = True
except exceptions.OpenStackRestAPIException as e:
if httplib.UNAUTHORIZED == e.http_status_code:
response['error-code'] = nfvi.NFVI_ERROR_CODE.TOKEN_EXPIRED
if self._platform_token is not None:
self._platform_token.set_expired()
else:
DLOG.exception("Caught API exception while trying %s. error=%s"
% (action_type, e))
except Exception as e:
DLOG.exception("Caught exception while trying %s. error=%s"
% (action_type, e))
finally:
callback.send(response)
callback.close()
def kube_upgrade_start(self, future, to_version, force, alarm_ignore_list,
callback):
"""

7
nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/sysinv.py
View File

@ -443,6 +443,13 @@ def kube_upgrade_networking(token):
return _patch_kube_upgrade_state(token, "upgrading-networking")
def kube_upgrade_storage(token):
"""
Ask System Inventory to kube upgrade storage
"""
return _patch_kube_upgrade_state(token, "upgrading-storage")
def kube_host_cordon(token, hostname, force):
"""
system kube-host-cordon <host>

81
nfv/nfv-tests/nfv_unit_tests/tests/test_kube_upgrade_strategy.py
View File

@ -340,6 +340,17 @@ class ApplyStageMixin(object):
],
}
def _kube_upgrade_storage_stage(self):
return {
'name': 'kube-upgrade-storage',
'total_steps': 1,
'steps': [
{'name': 'kube-upgrade-storage',
'success_state': 'upgraded-storage',
'fail_state': 'upgrading-storage-failed'},
],
}
def _kube_upgrade_complete_stage(self):
return {
'name': 'kube-upgrade-complete',
@ -485,6 +496,7 @@ class ApplyStageMixin(object):
add_start=True,
add_download=True,
add_networking=True,
add_storage=True,
add_cordon=True,
add_first_control_plane=True,
add_second_control_plane=True,
@ -507,6 +519,8 @@ class ApplyStageMixin(object):
stages.append(self._kube_upgrade_download_images_stage())
if add_networking:
stages.append(self._kube_upgrade_networking_stage())
if add_storage:
stages.append(self._kube_upgrade_storage_stage())
if add_cordon:
stages.append(self._kube_host_cordon_stage())
for ver in self.kube_versions:
@ -622,6 +636,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
stages = [
self._kube_upgrade_download_images_stage(),
self._kube_upgrade_networking_stage(),
self._kube_upgrade_storage_stage(),
]
if self.is_simplex():
stages.append(self._kube_host_cordon_stage())
@ -652,6 +667,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
self.default_to_version)
stages = [
self._kube_upgrade_networking_stage(),
self._kube_upgrade_storage_stage(),
]
if self.is_simplex():
stages.append(self._kube_host_cordon_stage())
@ -735,6 +751,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
self.default_to_version)
stages = [
self._kube_upgrade_networking_stage(),
self._kube_upgrade_storage_stage()
]
if self.is_simplex():
stages.append(self._kube_host_cordon_stage())
@ -758,12 +775,74 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
"""
Test the kube_upgrade strategy creation when there is only a simplex
and the upgrade had previously stopped after successful networking.
It is expected to resume at the first control plane
It is expected to resume at the storage stage
"""
kube_upgrade = self._create_kube_upgrade_obj(
KUBE_UPGRADE_STATE.KUBE_UPGRADED_NETWORKING,
self.default_from_version,
self.default_to_version)
stages = [
self._kube_upgrade_storage_stage()
]
if self.is_simplex():
stages.append(self._kube_host_cordon_stage())
for ver in self.kube_versions:
stages.append(self._kube_upgrade_first_control_plane_stage(
ver))
stages.extend(self._kube_upgrade_kubelet_stages(
ver,
self.std_controller_list,
self.aio_controller_list,
self.worker_list))
if self.is_simplex():
stages.append(self._kube_host_uncordon_stage())
stages.extend([
self._kube_upgrade_complete_stage(),
self._kube_upgrade_cleanup_stage(),
])
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
def test_resume_after_storage_failed(self):
"""
Test the kube_upgrade strategy creation when there is only a simplex
and the upgrade had previously failed during storage.
It is expected to retry and resume at the storage stage
"""
kube_upgrade = self._create_kube_upgrade_obj(
KUBE_UPGRADE_STATE.KUBE_UPGRADING_STORAGE_FAILED,
self.default_from_version,
self.default_to_version)
stages = [
self._kube_upgrade_storage_stage(),
]
if self.is_simplex():
stages.append(self._kube_host_cordon_stage())
for ver in self.kube_versions:
stages.append(self._kube_upgrade_first_control_plane_stage(
ver))
stages.extend(self._kube_upgrade_kubelet_stages(
ver,
self.std_controller_list,
self.aio_controller_list,
self.worker_list))
if self.is_simplex():
stages.append(self._kube_host_uncordon_stage())
stages.extend([
self._kube_upgrade_complete_stage(),
self._kube_upgrade_cleanup_stage(),
])
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
def test_resume_after_storage_succeeded(self):
"""
Test the kube_upgrade strategy creation when there is only a simplex
and the upgrade had previously stopped after successful storage.
It is expected to resume at the first control plane
"""
kube_upgrade = self._create_kube_upgrade_obj(
KUBE_UPGRADE_STATE.KUBE_UPGRADED_STORAGE,
self.default_from_version,
self.default_to_version)
stages = []
if self.is_simplex():
stages.append(self._kube_host_cordon_stage())

1
nfv/nfv-vim/nfv_vim/nfvi/__init__.py
View File

@ -130,6 +130,7 @@ from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_upgrade_complete
from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_upgrade_download_images # noqa: F401
from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_upgrade_networking # noqa: F401
from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_upgrade_start # noqa: F401
from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_upgrade_storage # noqa: F401
from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_list_deployment_hosts # noqa: F401
from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_lock_host # noqa: F401
from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_notify_host_failed # noqa: F401

9
nfv/nfv-vim/nfv_vim/nfvi/_nfvi_infrastructure_module.py
View File

@ -299,6 +299,15 @@ def nfvi_kube_upgrade_networking(callback):
return cmd_id
def nfvi_kube_upgrade_storage(callback):
"""
Kube Upgrade Storage
"""
cmd_id = _infrastructure_plugin.invoke_plugin('kube_upgrade_storage',
callback=callback)
return cmd_id
def nfvi_kube_upgrade_start(to_version, force, alarm_ignore_list, callback):
"""
Kube Upgrade Start

3
nfv/nfv-vim/nfv_vim/nfvi/objects/v1/_kube_upgrade.py
View File

@ -44,6 +44,9 @@ class KubeUpgradeState(Constants):
KUBE_UPGRADING_NETWORKING = Constant('upgrading-networking')
KUBE_UPGRADING_NETWORKING_FAILED = Constant('upgrading-networking-failed')
KUBE_UPGRADED_NETWORKING = Constant('upgraded-networking')
KUBE_UPGRADING_STORAGE = Constant('upgrading-storage')
KUBE_UPGRADING_STORAGE_FAILED = Constant('upgrading-storage-failed')
KUBE_UPGRADED_STORAGE = Constant('upgraded-storage')
KUBE_UPGRADING_FIRST_MASTER = Constant('upgrading-first-master')
KUBE_UPGRADING_FIRST_MASTER_FAILED = Constant('upgrading-first-master-failed')
KUBE_UPGRADED_FIRST_MASTER = Constant('upgraded-first-master')

1
nfv/nfv-vim/nfv_vim/strategy/__init__.py
View File

@ -36,6 +36,7 @@ from nfv_vim.strategy._strategy_steps import KubeUpgradeCompleteStep # noqa: F4
from nfv_vim.strategy._strategy_steps import KubeUpgradeDownloadImagesStep # noqa: F401
from nfv_vim.strategy._strategy_steps import KubeUpgradeNetworkingStep # noqa: F401
from nfv_vim.strategy._strategy_steps import KubeUpgradeStartStep # noqa: F401
from nfv_vim.strategy._strategy_steps import KubeUpgradeStorageStep # noqa: F401
from nfv_vim.strategy._strategy_steps import LockHostsStep # noqa: F401
from nfv_vim.strategy._strategy_steps import MigrateInstancesFromHostStep # noqa: F401
from nfv_vim.strategy._strategy_steps import MigrateInstancesStep # noqa: F401

26
nfv/nfv-vim/nfv_vim/strategy/_strategy.py
View File

@ -3458,6 +3458,22 @@ class KubeUpgradeStrategy(SwUpdateStrategy,
stage.add_step(strategy.KubeUpgradeNetworkingStep())
self.apply_phase.add_stage(stage)
# Next stage after networking is upgrade storage
self._add_kube_upgrade_storage_stage()
def _add_kube_upgrade_storage_stage(self):
"""
Add kube upgrade storage stage.
This stage only occurs after upgrade networking
It then proceeds to the next stage
"""
from nfv_vim import strategy
stage = strategy.StrategyStage(
strategy.STRATEGY_STAGE_NAME.KUBE_UPGRADE_STORAGE)
stage.add_step(strategy.KubeUpgradeStorageStep())
self.apply_phase.add_stage(stage)
# Next stage after networking is cordon
self._add_kube_host_cordon_stage()
@ -3794,8 +3810,16 @@ class KubeUpgradeStrategy(SwUpdateStrategy,
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADING_NETWORKING_FAILED:
self._add_kube_upgrade_networking_stage,
# After networking -> cordon
# After networking -> upgrade storage
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADED_NETWORKING:
self._add_kube_upgrade_storage_stage,
# if storage state failed, resync at storage state
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADING_STORAGE_FAILED:
self._add_kube_upgrade_storage_stage,
# After storage -> cordon
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADED_STORAGE:
self._add_kube_host_cordon_stage,
# If the state is cordon-failed, resume at cordon stage

1
nfv/nfv-vim/nfv_vim/strategy/_strategy_stages.py
View File

@ -66,6 +66,7 @@ class StrategyStageNames(Constants):
KUBE_UPGRADE_FIRST_CONTROL_PLANE = \
Constant('kube-upgrade-first-control-plane')
KUBE_UPGRADE_NETWORKING = Constant('kube-upgrade-networking')
KUBE_UPGRADE_STORAGE = Constant('kube-upgrade-storage')
KUBE_UPGRADE_SECOND_CONTROL_PLANE = \
Constant('kube-upgrade-second-control-plane')
KUBE_UPGRADE_PATCH = Constant('kube-upgrade-patch')

36
nfv/nfv-vim/nfv_vim/strategy/_strategy_steps.py
View File

@ -83,6 +83,7 @@ class StrategyStepNames(Constants):
KUBE_UPGRADE_COMPLETE = Constant('kube-upgrade-complete')
KUBE_UPGRADE_DOWNLOAD_IMAGES = Constant('kube-upgrade-download-images')
KUBE_UPGRADE_NETWORKING = Constant('kube-upgrade-networking')
KUBE_UPGRADE_STORAGE = Constant('kube-upgrade-storage')
KUBE_HOST_UPGRADE_CONTROL_PLANE = \
Constant('kube-host-upgrade-control-plane')
KUBE_HOST_UPGRADE_KUBELET = Constant('kube-host-upgrade-kubelet')
@ -4407,6 +4408,40 @@ class KubeUpgradeNetworkingStep(AbstractKubeUpgradeStep):
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
class KubeUpgradeStorageStep(AbstractKubeUpgradeStep):
"""Kube Upgrade Storage - Strategy Step"""
def __init__(self):
from nfv_vim import nfvi
super(KubeUpgradeStorageStep, self).__init__(
STRATEGY_STEP_NAME.KUBE_UPGRADE_STORAGE,
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADED_STORAGE,
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADING_STORAGE_FAILED,
timeout_in_secs=900)
@coroutine
def _response_callback(self):
"""Kube Upgrade Storage - Callback"""
response = (yield)
DLOG.debug("%s callback response=%s." % (self._name, response))
if response['completed']:
if self.strategy is not None:
self.strategy.nfvi_kube_upgrade = response['result-data']
else:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, response['reason'])
def apply(self):
"""Kube Upgrade Storage"""
from nfv_vim import nfvi
nfvi.nfvi_kube_upgrade_storage(self._response_callback())
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
class AbstractKubeHostUpgradeStep(AbstractKubeUpgradeStep):
"""Kube Upgrade Host - Abstract Strategy Step
@ -4861,6 +4896,7 @@ def strategy_step_rebuild_from_dict(data):
STRATEGY_STEP_NAME.KUBE_UPGRADE_DOWNLOAD_IMAGES:
KubeUpgradeDownloadImagesStep,
STRATEGY_STEP_NAME.KUBE_UPGRADE_NETWORKING: KubeUpgradeNetworkingStep,
STRATEGY_STEP_NAME.KUBE_UPGRADE_STORAGE: KubeUpgradeStorageStep,
STRATEGY_STEP_NAME.KUBE_UPGRADE_START: KubeUpgradeStartStep,
STRATEGY_STEP_NAME.QUERY_KUBE_HOST_UPGRADE: QueryKubeHostUpgradeStep,
STRATEGY_STEP_NAME.QUERY_KUBE_UPGRADE: QueryKubeUpgradeStep,