starlingx
/
config
Code Issues Proposed changes

Merge "Steps for kube-upgrade-storage"

This commit is contained in:
Zuul 2024-01-09 22:25:06 +00:00 committed by Gerrit Code Review
parent 907c461db0 97f0d1b24d
commit d65514be34
8 changed files with 155 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
sysinv/cgts-client/cgts-client/cgtsclient/v1/kube_upgrade_shell.py
View File

@ -11,6 +11,7 @@ from cgtsclient.v1 import ihost as ihost_utils
# Kubernetes constants
KUBE_UPGRADE_STATE_DOWNLOADING_IMAGES = 'downloading-images'
KUBE_UPGRADE_STATE_UPGRADING_NETWORKING = 'upgrading-networking'
KUBE_UPGRADE_STATE_UPGRADING_STORAGE = 'upgrading-storage'
KUBE_UPGRADE_STATE_COMPLETE = 'upgrade-complete'
KUBE_UPGRADE_STATE_UPGRADING_FIRST_MASTER = 'upgrading-first-master'
KUBE_UPGRADE_STATE_UPGRADING_SECOND_MASTER = 'upgrading-second-master'
@ -131,6 +132,23 @@ def do_kube_upgrade_networking(cc, args):
_print_kube_upgrade_show(kube_upgrade)
def do_kube_upgrade_storage(cc, args):
"""Upgrade kubernetes storage."""
data = dict()
data['state'] = KUBE_UPGRADE_STATE_UPGRADING_STORAGE
patch = []
for (k, v) in data.items():
patch.append({'op': 'replace', 'path': '/' + k, 'value': v})
try:
kube_upgrade = cc.kube_upgrade.update(patch)
except exc.HTTPNotFound:
raise exc.CommandError('Kubernetes upgrade UUID not found')
_print_kube_upgrade_show(kube_upgrade)
def do_kube_upgrade_abort(cc, args):
"""Kubernetes upgrade aborting."""

6
sysinv/sysinv/sysinv/sysinv/api/controllers/v1/host.py
View File

@ -7178,12 +7178,12 @@ class HostController(rest.RestController):
if utils.get_system_mode() == constants.SYSTEM_MODE_SIMPLEX:
check_upgraded_state = [
kubernetes.KUBE_UPGRADED_NETWORKING,
kubernetes.KUBE_UPGRADED_STORAGE,
kubernetes.KUBE_UPGRADED_FIRST_MASTER,
kubernetes.KUBE_UPGRADE_CORDON_COMPLETE]
else:
check_upgraded_state = [
kubernetes.KUBE_UPGRADED_NETWORKING,
kubernetes.KUBE_UPGRADED_STORAGE,
kubernetes.KUBE_UPGRADED_FIRST_MASTER]
# Verify the upgrade is in the correct state
@ -7278,7 +7278,7 @@ class HostController(rest.RestController):
kube_host_upgrade_obj.save()
if kube_upgrade_obj.state in [
kubernetes.KUBE_UPGRADED_NETWORKING,
kubernetes.KUBE_UPGRADED_STORAGE,
kubernetes.KUBE_UPGRADING_FIRST_MASTER_FAILED,
kubernetes.KUBE_UPGRADE_CORDON_COMPLETE]:
# Update the upgrade state

35
sysinv/sysinv/sysinv/sysinv/api/controllers/v1/kube_upgrade.py
View File

@ -336,6 +336,7 @@ class KubeUpgradeController(rest.RestController):
kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES,
kubernetes.KUBE_UPGRADING_FIRST_MASTER,
kubernetes.KUBE_UPGRADING_SECOND_MASTER,
kubernetes.KUBE_UPGRADING_STORAGE,
kubernetes.KUBE_UPGRADING_NETWORKING]:
kube_upgrade_obj.state = updates['state']
kube_upgrade_obj.save()
@ -385,8 +386,8 @@ class KubeUpgradeController(rest.RestController):
kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES,
kubernetes.KUBE_UPGRADING_NETWORKING_FAILED]:
raise wsme.exc.ClientSideError(_(
"Kubernetes upgrade must be in %s or %s state to upgrade "
"networking" %
"Kubernetes upgrade must be in %s or %s state to "
"upgrade networking" %
(kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES,
kubernetes.KUBE_UPGRADING_NETWORKING_FAILED)))
@ -402,6 +403,30 @@ class KubeUpgradeController(rest.RestController):
kube_upgrade_obj.to_version)
return KubeUpgrade.convert_with_links(kube_upgrade_obj)
elif updates['state'] == kubernetes.KUBE_UPGRADING_STORAGE:
# Make sure upgrade is in the correct state to upgrade storage
if kube_upgrade_obj.state not in [
kubernetes.KUBE_UPGRADED_NETWORKING,
kubernetes.KUBE_UPGRADING_STORAGE_FAILED]:
raise wsme.exc.ClientSideError(_(
"Kubernetes upgrade must be in %s or %s state to "
"upgrade storage" %
(kubernetes.KUBE_UPGRADED_NETWORKING,
kubernetes.KUBE_UPGRADING_STORAGE_FAILED)))
# Update the upgrade state
kube_upgrade_obj.state = kubernetes.KUBE_UPGRADING_STORAGE
kube_upgrade_obj.save()
# Tell the conductor to upgrade storage
pecan.request.rpcapi.kube_upgrade_storage(
pecan.request.context, kube_upgrade_obj.to_version)
LOG.info("Upgrading kubernetes storage to version: %s" %
kube_upgrade_obj.to_version)
return KubeUpgrade.convert_with_links(kube_upgrade_obj)
elif updates['state'] == kubernetes.KUBE_UPGRADE_ABORTING:
system = pecan.request.dbapi.isystem_get_one()
if system.system_mode != constants.SYSTEM_MODE_SIMPLEX:
@ -460,11 +485,13 @@ class KubeUpgradeController(rest.RestController):
# Make sure upgrade is in the correct state to cordon
if kube_upgrade_obj.state not in [
kubernetes.KUBE_UPGRADED_NETWORKING,
kubernetes.KUBE_UPGRADED_STORAGE,
kubernetes.KUBE_UPGRADE_CORDON_FAILED]:
raise wsme.exc.ClientSideError(_(
"Kubernetes upgrade must be in %s or %s state to "
"cordon" %
"Kubernetes upgrade must be in %s, %s or %s state "
"to cordon" %
(kubernetes.KUBE_UPGRADED_NETWORKING,
kubernetes.KUBE_UPGRADED_STORAGE,
kubernetes.KUBE_UPGRADE_CORDON_FAILED)))
# Update the upgrade state

2
sysinv/sysinv/sysinv/sysinv/common/constants.py
View File

@ -2110,6 +2110,8 @@ ANSIBLE_RESTORE_ROOK_FLAG = os.path.join(tsc.VOLATILE_PATH, ".ansible_restore_ro
# Ansible playbooks
ANSIBLE_KUBE_NETWORKING_PLAYBOOK = \
'/usr/share/ansible/stx-ansible/playbooks/upgrade-k8s-networking.yml'
ANSIBLE_KUBE_STORAGE_PLAYBOOK = \
'/usr/share/ansible/stx-ansible/playbooks/upgrade-k8s-storage.yml'
ANSIBLE_KUBE_PUSH_IMAGES_PLAYBOOK = \
'/usr/share/ansible/stx-ansible/playbooks/push_k8s_images.yml'
ANSIBLE_PLATFORM_BACKUP_PLAYBOOK = \

16
sysinv/sysinv/sysinv/sysinv/common/kubernetes.py
View File

@ -85,6 +85,9 @@ KUBE_UPGRADE_DOWNLOADED_IMAGES = 'downloaded-images'
KUBE_UPGRADING_NETWORKING = 'upgrading-networking'
KUBE_UPGRADING_NETWORKING_FAILED = 'upgrading-networking-failed'
KUBE_UPGRADED_NETWORKING = 'upgraded-networking'
KUBE_UPGRADING_STORAGE = 'upgrading-storage'
KUBE_UPGRADING_STORAGE_FAILED = 'upgrading-storage-failed'
KUBE_UPGRADED_STORAGE = 'upgraded-storage'
KUBE_UPGRADING_FIRST_MASTER = 'upgrading-first-master'
KUBE_UPGRADING_FIRST_MASTER_FAILED = 'upgrading-first-master-failed'
KUBE_UPGRADED_FIRST_MASTER = 'upgraded-first-master'
@ -283,6 +286,19 @@ def get_kube_networking_upgrade_version(kube_upgrade):
return kube_upgrade.to_version
def get_kube_storage_upgrade_version(kube_upgrade):
"""Determine the version that kubernetes storage
should be upgraded to."""
if kube_upgrade.state in [
KUBE_UPGRADE_STARTED,
KUBE_UPGRADE_DOWNLOADING_IMAGES,
KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED,
KUBE_UPGRADE_DOWNLOADED_IMAGES]:
return kube_upgrade.from_version
else:
return kube_upgrade.to_version
def is_k8s_configured():
"""Check to see if the k8s admin config file exists."""
if os.path.isfile(KUBERNETES_ADMIN_CONF):

68
sysinv/sysinv/sysinv/sysinv/conductor/manager.py
View File

@ -7939,6 +7939,7 @@ class ConductorManager(service.PeriodicService):
return
self._upgrade_downgrade_static_images()
self._upgrade_downgrade_kube_networking()
self._upgrade_downgrade_kube_storage()
@retry(retry_on_result=lambda x: x is False,
wait_fixed=(CONF.conductor.kube_upgrade_downgrade_retry_interval * 1000))
@ -7977,6 +7978,43 @@ class ConductorManager(service.PeriodicService):
return True
@retry(retry_on_result=lambda x: x is False,
wait_fixed=(CONF.conductor.kube_upgrade_downgrade_retry_interval * 1000))
@cutils.synchronized(LOCK_IMAGE_PULL)
def _upgrade_downgrade_kube_storage(self):
try:
# Get the kubernetes version from the upgrade table
# if an upgrade exists
kube_upgrade = self.dbapi.kube_upgrade_get_one()
kube_version = \
kubernetes.get_kube_storage_upgrade_version(kube_upgrade)
except exception.NotFound:
# Not upgrading kubernetes, get the kubernetes version
# from the kubeadm config map
kube_version = self._kube.kube_get_kubernetes_version()
if not kube_version:
LOG.error("Unable to get the current kubernetes version.")
return False
try:
LOG.info("_upgrade_downgrade_kube_storage executing"
" playbook: %s for version %s" %
(constants.ANSIBLE_KUBE_STORAGE_PLAYBOOK, kube_version))
playbook_cmd = ['ansible-playbook', '-e', 'kubernetes_version=%s' % kube_version,
constants.ANSIBLE_KUBE_STORAGE_PLAYBOOK]
returncode = cutils.run_playbook(playbook_cmd)
if returncode:
raise Exception("ansible-playbook returned an error: %s" % returncode)
except Exception as e:
LOG.error("Failed to upgrade/downgrade kubernetes "
"storage images: {}".format(e))
return False
return True
@retry(retry_on_result=lambda x: x is False,
wait_fixed=(CONF.conductor.kube_upgrade_downgrade_retry_interval * 1000))
@cutils.synchronized(LOCK_IMAGE_PULL)
@ -15982,6 +16020,8 @@ class ConductorManager(service.PeriodicService):
fail_state = kubernetes.KUBE_UPGRADING_FIRST_MASTER_FAILED
elif kube_upgrade.state == kubernetes.KUBE_UPGRADING_NETWORKING:
fail_state = kubernetes.KUBE_UPGRADING_NETWORKING_FAILED
elif kube_upgrade.state == kubernetes.KUBE_UPGRADING_STORAGE:
fail_state = kubernetes.KUBE_UPGRADING_STORAGE_FAILED
elif kube_upgrade.state == kubernetes.KUBE_UPGRADING_SECOND_MASTER:
fail_state = kubernetes.KUBE_UPGRADING_SECOND_MASTER_FAILED
@ -16469,6 +16509,30 @@ class ConductorManager(service.PeriodicService):
kube_upgrade_obj.state = kubernetes.KUBE_UPGRADED_NETWORKING
kube_upgrade_obj.save()
def kube_upgrade_storage(self, context, kube_version):
"""Upgrade kubernetes storage for this kubernetes version"""
LOG.info("executing playbook: %s for version %s" %
(constants.ANSIBLE_KUBE_STORAGE_PLAYBOOK, kube_version))
playbook_cmd = ['ansible-playbook', '-e', 'kubernetes_version=%s' % kube_version,
constants.ANSIBLE_KUBE_STORAGE_PLAYBOOK]
returncode = cutils.run_playbook(playbook_cmd)
if returncode:
LOG.warning("ansible-playbook returned an error: %s" %
returncode)
# Update the upgrade state
kube_upgrade_obj = objects.kube_upgrade.get_one(context)
kube_upgrade_obj.state = \
kubernetes.KUBE_UPGRADING_STORAGE_FAILED
kube_upgrade_obj.save()
return
# Indicate that storage upgrade is complete
kube_upgrade_obj = objects.kube_upgrade.get_one(context)
kube_upgrade_obj.state = kubernetes.KUBE_UPGRADED_STORAGE
kube_upgrade_obj.save()
def kube_upgrade_abort(self, context, kube_state):
"""
This is an abort procedure we call via 'system kube-upgrade-abort'
@ -16508,7 +16572,9 @@ class ConductorManager(service.PeriodicService):
not os.path.exists(kubernetes.KUBE_CONTROL_PLANE_STATIC_PODS_BACKUP_PATH):
LOG.info("Kubernetes control plane backup path doesn't exists.")
if kube_state in [kubernetes.KUBE_UPGRADING_NETWORKING,
kubernetes.KUBE_UPGRADING_NETWORKING_FAILED]:
kubernetes.KUBE_UPGRADING_NETWORKING_FAILED,
kubernetes.KUBE_UPGRADING_STORAGE,
kubernetes.KUBE_UPGRADING_STORAGE_FAILED]:
# Indicate that kubernetes upgrade is aborted
for host_obj in controller_hosts:
kube_host_upgrade_obj = objects.kube_host_upgrade.get_by_host_id(

10
sysinv/sysinv/sysinv/sysinv/conductor/rpcapi.py
View File

@ -1972,6 +1972,16 @@ class ConductorAPI(sysinv.openstack.common.rpc.proxy.RpcProxy):
return self.cast(context, self.make_msg('kube_upgrade_networking',
kube_version=kube_version))
def kube_upgrade_storage(self, context, kube_version):
"""Asynchronously, have the conductor upgrade storage for this
new version.
:param context: request context
:param kube_version: kubernetes version being upgraded to
"""
return self.cast(context, self.make_msg('kube_upgrade_storage',
kube_version=kube_version))
def kube_upgrade_abort(self, context, kube_state):
"""Asynchronously, have the conductor abort the kubernetes
upgrade.

16
sysinv/sysinv/sysinv/sysinv/tests/api/test_host.py
View File

@ -481,7 +481,7 @@ class TestPostKubeUpgrades(TestHost):
kube_upgrade = dbutils.create_test_kube_upgrade(
from_version='v1.42.1',
to_version='v1.42.2',
state=kubernetes.KUBE_UPGRADED_NETWORKING,
state=kubernetes.KUBE_UPGRADED_STORAGE,
)
# Upgrade the control plane
@ -574,7 +574,7 @@ class TestPostKubeUpgrades(TestHost):
kube_upgrade = dbutils.create_test_kube_upgrade(
from_version='v1.42.1',
to_version='v1.42.2',
state=kubernetes.KUBE_UPGRADED_NETWORKING,
state=kubernetes.KUBE_UPGRADED_STORAGE,
)
# Upgrade the control plane
@ -816,7 +816,7 @@ class TestPostKubeUpgrades(TestHost):
dbutils.create_test_kube_upgrade(
from_version='v1.42.1',
to_version='v1.42.2',
state=kubernetes.KUBE_UPGRADED_NETWORKING,
state=kubernetes.KUBE_UPGRADED_STORAGE,
)
# Fake the missing patches
@ -865,7 +865,7 @@ class TestPostKubeUpgrades(TestHost):
dbutils.create_test_kube_upgrade(
from_version='v1.42.1',
to_version='v1.42.2',
state=kubernetes.KUBE_UPGRADED_NETWORKING,
state=kubernetes.KUBE_UPGRADED_STORAGE,
)
# Fake the missing patches
@ -918,7 +918,7 @@ class TestPostKubeUpgrades(TestHost):
dbutils.create_test_kube_upgrade(
from_version='v1.42.1',
to_version='v1.42.2',
state=kubernetes.KUBE_UPGRADED_NETWORKING,
state=kubernetes.KUBE_UPGRADED_STORAGE,
)
# Upgrade the control plane
@ -948,7 +948,7 @@ class TestPostKubeUpgrades(TestHost):
dbutils.create_test_kube_upgrade(
from_version='v1.42.1',
to_version='v1.42.2',
state=kubernetes.KUBE_UPGRADED_NETWORKING,
state=kubernetes.KUBE_UPGRADED_STORAGE,
)
# No control plane version for this controller
@ -983,7 +983,7 @@ class TestPostKubeUpgrades(TestHost):
dbutils.create_test_kube_upgrade(
from_version='v1.42.1',
to_version='v1.42.2',
state=kubernetes.KUBE_UPGRADED_NETWORKING,
state=kubernetes.KUBE_UPGRADED_STORAGE,
)
# Upgrade the control plane
@ -1013,7 +1013,7 @@ class TestPostKubeUpgrades(TestHost):
dbutils.create_test_kube_upgrade(
from_version='v1.42.1',
to_version='v1.42.2',
state=kubernetes.KUBE_UPGRADED_NETWORKING,
state=kubernetes.KUBE_UPGRADED_STORAGE,
)
# Mark the kube host as already upgrading