starlingx
/
config
Code Issues Proposed changes

Update apps during Kubernetes upgrade 

Update StarlingX applications during Kubernetes upgrades according to
their metadata.

Applications are updated if they have "k8s_upgrades:auto_update" set to
true on their metadata.yaml file. The ones that have
"k8s_upgrades:timing" set to "pre" are updated during the
"kube-upgrade-start" phase. The ones that set "k8s_upgrades:timing" to
"post" are updated during the "kube-upgrade-complete" phase.

In order to better support application updates during
kube-upgrade-start, two new statuses were added: 'upgrade-starting' and
'upgrade-starting-failed'. The 'upgrade-starting' state is the new
initial state when triggering a Kubernetes upgrade. If starting the
upgrade fails, then the status is updated to 'upgrade-starting-failed'
and users can either abort the upgrade (only available in simplex loads)
or try starting it again. No changes were made to kube-upgrade-complete
in that regard because at that point a new Kubernetes version is already
in place. A review was raised to nfv-vim to reflect the new statuses on
the Kubernetes orchestrated upgrade code:
https://review.opendev.org/c/starlingx/nfv/+/906594

Application auto updates can be retried when restarting a Kubernetes
upgrade that previously failed due to a failing app.

A bug that was preventing the k8s_upgrade metadata section from being
parsed was fixed by this commit as well.

Test Plan:
PASS: build-pkgs -a && build-image
PASS: Create a new platform-integ-apps tarball adding
      "k8s_upgrades:auto_update=true" and "k8s_upgrades:timing=pre" to
      metadata.yaml.
      Add the new tarball to /usr/local/share/applications/helm/.
      Run kube-upgrade-start.
      Check if platform-integ-apps was correctly updated.
      Check if no other apps were updated.
PASS: Create a new metrics-server tarball adding
      "k8s_upgrades:auto_update=true" and "k8s_upgrades:timing=post" to
      to metadata.yaml.
      Add the new tarball to /usr/local/share/applications/helm/.
      Run kube-upgrade-complete.
      Check if metrics-server was correctly updated.
      Check if no other apps were updated.
PASS: Create a new platform-integ-apps tarball adding
      "k8s_upgrades:auto_update=true" and "k8s_upgrades:timing=pre" to
      metadata.yaml.
      Add the new tarball to /usr/local/share/applications/helm/.
      Restart sysinv to update the database.
      Replace the platform-integ-apps tarball with another tarball that
      does not have a metadata.yaml file.
      Check if an error is logged when running kube-upgrade-start
      reporting that platform-integ-apps failed to be updated.
      Confirm that the Kubernetes upgrade was not started.
      Abort Kubernetes upgrade
      Check if upgrade was successfully aborted
PASS: Create a new metrics-server tarball adding
      "k8s_upgrades:auto_update=true" and "k8s_upgrades:timing=post" to
      to metadata.yaml.
      Add the new tarball to /usr/local/share/applications/helm/.
      Restart sysinv to update the database.
      Replace the snmp tarball with another tarball that does not have
      a metadata.yaml file.
      Check if an error is logged when running kube-upgrade-complete
      reporting that metrics-server failed to be updated.
      Check if the Kubernetes upgrade was marked as completed.
PASS: AIO-SX fresh install
      Manual upgrade to Kubernetes v1.27.5
      Check if upgrade was successfuly done
PASS: AIO-SX fresh install
      Orchestrated upgrade to Kubernetes v1.27.5
      Check if upgrade was successfuly done
PASS: AIO-SX fresh install with Kubernetes v1.24.4
      Orchestrated upgrade to Kubernetes v1.27.5
      Check if upgrade was successfuly done

Story: 2010929
Task: 49416

Change-Id: I31333bf44501c7ad1688635b75c7fcef11513026
Signed-off-by: Igor Soares <Igor.PiresSoares@windriver.com>
This commit is contained in:
Igor Soares 2024-01-08 12:39:35 -03:00
parent 0e8ca81672
commit 46f5ccfc55
11 changed files with 497 additions and 201 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
sysinv/sysinv/sysinv/sysinv/api/controllers/v1/kube_app.py
View File

@ -665,7 +665,7 @@ class KubeAppHelper(object):
"Error while reporting the patch dependencies "
"to patch-controller.")
def _check_app_compatibility(self, app_name, app_version):
def _check_app_compatibility(self, app_name, app_version, target_kube_version=None):
"""Checks whether the application is compatible
with the current k8s version"""
@ -675,14 +675,25 @@ class KubeAppHelper(object):
if not kube_min_version and not kube_max_version:
return
version_states = self._kube_operator.kube_get_version_states()
for kube_version, state in version_states.items():
if state in [kubernetes.KUBE_STATE_ACTIVE,
kubernetes.KUBE_STATE_PARTIAL]:
if not kubernetes.is_kube_version_supported(
kube_version, kube_min_version, kube_max_version):
raise exception.IncompatibleKubeVersion(
name=app_name, version=app_version, kube_version=kube_version)
if target_kube_version is None:
version_states = self._kube_operator.kube_get_version_states()
for kube_version, state in version_states.items():
if state in [kubernetes.KUBE_STATE_ACTIVE,
kubernetes.KUBE_STATE_PARTIAL]:
if not kubernetes.is_kube_version_supported(
kube_version, kube_min_version, kube_max_version):
LOG.error("Application {} is incompatible with Kubernetes version {}."
.format(app_name, kube_version))
raise exception.IncompatibleKubeVersion(
name=app_name, version=app_version, kube_version=kube_version)
elif not kubernetes.is_kube_version_supported(target_kube_version,
kube_min_version,
kube_max_version):
LOG.error("Application {} is incompatible with target Kubernetes version {}."
.format(app_name, target_kube_version))
raise exception.IncompatibleKubeVersion(name=app_name,
version=app_version,
kube_version=target_kube_version)
def _find_manifest(self, app_path, app_name):
""" Find the required application manifest elements

139
sysinv/sysinv/sysinv/sysinv/api/controllers/v1/kube_upgrade.py
View File

@ -152,24 +152,6 @@ class KubeUpgradeController(rest.RestController):
"the kubernetes upgrade: %s" %
available_patches))
@staticmethod
def _check_installed_apps_compatibility(apps, kube_version):
"""Checks whether all installed applications are compatible
with the new k8s version"""
for app in apps:
if app.status != constants.APP_APPLY_SUCCESS:
continue
kube_min_version, kube_max_version = \
cutils.get_app_supported_kube_version(app.name, app.app_version)
if not kubernetes.is_kube_version_supported(
kube_version, kube_min_version, kube_max_version):
raise wsme.exc.ClientSideError(_(
"The installed Application %s (%s) is incompatible with the "
"new Kubernetes version %s." % (app.name, app.app_version, kube_version)))
@wsme_pecan.wsexpose(KubeUpgradeCollection)
def get_all(self):
"""Retrieve a list of kubernetes upgrades."""
@ -193,6 +175,7 @@ class KubeUpgradeController(rest.RestController):
force = body.get('force', False) is True
alarm_ignore_list = body.get('alarm_ignore_list')
system = pecan.request.dbapi.isystem_get_one()
retry = False
# There must not be a platform upgrade in progress
try:
@ -206,12 +189,21 @@ class KubeUpgradeController(rest.RestController):
# There must not already be a kubernetes upgrade in progress
try:
pecan.request.dbapi.kube_upgrade_get_one()
kube_upgrade_obj = objects.kube_upgrade.get_one(pecan.request.context)
except exception.NotFound:
pass
else:
raise wsme.exc.ClientSideError(_(
"A kubernetes upgrade is already in progress"))
# Allow retrying the new Kubernetes upgrade if the current
# state is 'upgrade-starting-failed'.
if (kube_upgrade_obj.state == kubernetes.KUBE_UPGRADE_STARTING_FAILED and
kube_upgrade_obj.to_version == to_version):
retry = True
if alarm_ignore_list is None:
alarm_ignore_list = []
alarm_ignore_list.append(fm_constants.FM_ALARM_ID_KUBE_UPGRADE_IN_PROGRESS)
else:
raise wsme.exc.ClientSideError(_(
"A kubernetes upgrade is already in progress"))
# Check whether target version is available or not
try:
@ -252,10 +244,6 @@ class KubeUpgradeController(rest.RestController):
applied_patches=target_version_obj.applied_patches,
available_patches=target_version_obj.available_patches)
# Check that all installed applications support new k8s version
apps = pecan.request.dbapi.kube_app_get_all()
self._check_installed_apps_compatibility(apps, to_version)
# The system must be healthy
success, output = pecan.request.rpcapi.get_system_health(
pecan.request.context,
@ -272,48 +260,63 @@ class KubeUpgradeController(rest.RestController):
"System is not in a valid state for kubernetes upgrade. "
"Run system health-query-kube-upgrade for more details."))
# Create upgrade record.
create_values = {'from_version': current_kube_version,
'to_version': to_version,
'state': kubernetes.KUBE_UPGRADE_STARTED}
new_upgrade = pecan.request.dbapi.kube_upgrade_create(create_values)
if retry:
# Update upgrade record
kube_upgrade_obj.state = kubernetes.KUBE_UPGRADE_STARTING
kube_upgrade_obj.save()
else:
# Create upgrade record.
create_values = {'from_version': current_kube_version,
'to_version': to_version,
'state': kubernetes.KUBE_UPGRADE_STARTING}
kube_upgrade_obj = pecan.request.dbapi.kube_upgrade_create(create_values)
# Set the target version for each host to the current version
update_values = {'target_version': current_kube_version}
kube_host_upgrades = pecan.request.dbapi.kube_host_upgrade_get_list()
for kube_host_upgrade in kube_host_upgrades:
pecan.request.dbapi.kube_host_upgrade_update(kube_host_upgrade.id,
update_values)
# Raise alarm to show a kubernetes upgrade is in progress
entity_instance_id = "%s=%s" % (fm_constants.FM_ENTITY_TYPE_HOST,
constants.CONTROLLER_HOSTNAME)
fault = fm_api.Fault(
alarm_id=fm_constants.FM_ALARM_ID_KUBE_UPGRADE_IN_PROGRESS,
alarm_state=fm_constants.FM_ALARM_STATE_SET,
entity_type_id=fm_constants.FM_ENTITY_TYPE_HOST,
entity_instance_id=entity_instance_id,
severity=fm_constants.FM_ALARM_SEVERITY_MINOR,
reason_text="Kubernetes upgrade in progress.",
# operational
alarm_type=fm_constants.FM_ALARM_TYPE_7,
# congestion
probable_cause=fm_constants.ALARM_PROBABLE_CAUSE_8,
proposed_repair_action="No action required.",
service_affecting=False)
fm_api.FaultAPIs().set_fault(fault)
try:
# Set the target version for each host to the current version
update_values = {'target_version': current_kube_version}
kube_host_upgrades = pecan.request.dbapi.kube_host_upgrade_get_list()
for kube_host_upgrade in kube_host_upgrades:
pecan.request.dbapi.kube_host_upgrade_update(kube_host_upgrade.id,
update_values)
# Raise alarm to show a kubernetes upgrade is in progress
entity_instance_id = "%s=%s" % (fm_constants.FM_ENTITY_TYPE_HOST,
constants.CONTROLLER_HOSTNAME)
fault = fm_api.Fault(
alarm_id=fm_constants.FM_ALARM_ID_KUBE_UPGRADE_IN_PROGRESS,
alarm_state=fm_constants.FM_ALARM_STATE_SET,
entity_type_id=fm_constants.FM_ENTITY_TYPE_HOST,
entity_instance_id=entity_instance_id,
severity=fm_constants.FM_ALARM_SEVERITY_MINOR,
reason_text="Kubernetes upgrade in progress.",
# operational
alarm_type=fm_constants.FM_ALARM_TYPE_7,
# congestion
probable_cause=fm_constants.ALARM_PROBABLE_CAUSE_8,
proposed_repair_action="No action required.",
service_affecting=False)
fm_api.FaultAPIs().set_fault(fault)
# Set the new kubeadm version in the DB.
# This will not actually change the bind mounts until we apply a
# puppet manifest that makes use of it.
kube_cmd_versions = objects.kube_cmd_version.get(
pecan.request.context)
kube_cmd_versions.kubeadm_version = to_version.lstrip('v')
kube_cmd_versions.save()
# Set the new kubeadm version in the DB.
# This will not actually change the bind mounts until we apply a
# puppet manifest that makes use of it.
kube_cmd_versions = objects.kube_cmd_version.get(
pecan.request.context)
kube_cmd_versions.kubeadm_version = to_version.lstrip('v')
kube_cmd_versions.save()
LOG.info("Started kubernetes upgrade from version: %s to version: %s"
% (current_kube_version, to_version))
LOG.info("Starting kubernetes upgrade from version: %s to version: %s"
% (current_kube_version, to_version))
return KubeUpgrade.convert_with_links(new_upgrade)
# Tell the conductor to update the required apps and mark the upgrade as started
pecan.request.rpcapi.kube_upgrade_start(
pecan.request.context,
to_version)
except Exception as e:
LOG.exception("Failed to start Kubernetes upgrade: %s" % e)
kube_upgrade_obj.state = kubernetes.KUBE_UPGRADE_STARTING_FAILED
kube_upgrade_obj.save()
return KubeUpgrade.convert_with_links(kube_upgrade_obj)
@cutils.synchronized(LOCK_NAME)
@wsme.validate([KubeUpgradePatchType])
@ -458,7 +461,9 @@ class KubeUpgradeController(rest.RestController):
# Update the state as aborted for these states since no actual k8s changes done
# so we don't need to do anything more to complete the abort.
if kube_upgrade_obj.state in [kubernetes.KUBE_UPGRADE_STARTED,
if kube_upgrade_obj.state in [kubernetes.KUBE_UPGRADE_STARTING,
kubernetes.KUBE_UPGRADE_STARTING_FAILED,
kubernetes.KUBE_UPGRADE_STARTED,
kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES,
kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED,
kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES]:
@ -622,6 +627,12 @@ class KubeUpgradeController(rest.RestController):
if role == constants.DISTRIBUTED_CLOUD_ROLE_SYSTEMCONTROLLER:
dc_api.notify_dcmanager_kubernetes_upgrade_completed()
# Update apps that contain 'k8s_upgrade.timing = post' metadata
pecan.request.rpcapi.update_apps_based_on_k8s_version_async(
pecan.request.context,
kube_upgrade_obj.to_version,
constants.APP_METADATA_TIMING_POST)
# Check if apps need to be reapplied
pecan.request.rpcapi.evaluate_apps_reapply(
pecan.request.context,

4
sysinv/sysinv/sysinv/sysinv/common/app_metadata.py
View File

@ -611,11 +611,11 @@ def extract_bundle_metadata(file_path):
LOG.warning("k8s_upgrades section missing from {} metadata"
.format(file_path))
else:
k8s_auto_update = tarball.metadata.get(
k8s_auto_update = metadata.get(
constants.APP_METADATA_K8S_UPGRADES).get(
constants.APP_METADATA_AUTO_UPDATE,
constants.APP_METADATA_K8S_AUTO_UPDATE_DEFAULT_VALUE)
k8s_update_timing = tarball.metadata.get(
k8s_update_timing = metadata.get(
constants.APP_METADATA_K8S_UPGRADES).get(
constants.APP_METADATA_TIMING,
constants.APP_METADATA_TIMING_DEFAULT_VALUE)

2
sysinv/sysinv/sysinv/sysinv/common/kubernetes.py
View File

@ -78,6 +78,8 @@ KUBE_CONTROLLER_MANAGER = 'kube-controller-manager'
KUBE_SCHEDULER = 'kube-scheduler'
# Kubernetes upgrade states
KUBE_UPGRADE_STARTING = 'upgrade-starting'
KUBE_UPGRADE_STARTING_FAILED = 'upgrade-starting-failed'
KUBE_UPGRADE_STARTED = 'upgrade-started'
KUBE_UPGRADE_DOWNLOADING_IMAGES = 'downloading-images'
KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED = 'downloading-images-failed'

38
sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py
View File

@ -2704,7 +2704,7 @@ class AppOperator(object):
def perform_app_update(self, from_rpc_app, to_rpc_app, tarfile,
operation, lifecycle_hook_info_app_update, reuse_user_overrides=None,
reuse_attributes=None):
reuse_attributes=None, k8s_version=None):
"""Process application update request
This method leverages the existing application upload workflow to
@ -2733,7 +2733,8 @@ class AppOperator(object):
:param lifecycle_hook_info_app_update: LifecycleHookInfo object
:param reuse_user_overrides: (optional) True or False
:param reuse_attributes: (optional) True or False
:param k8s_version: (optional) Target Kubernetes version
:return: True if the update to the new version was successful. False otherwise.
"""
from_app = AppOperator.Application(from_rpc_app)
@ -2777,20 +2778,25 @@ class AppOperator(object):
except exception.LifecycleSemanticCheckException as e:
LOG.info("App {} rejected operation {} for reason: {}"
"".format(to_app.name, constants.APP_UPDATE_OP, str(e)))
return self._perform_app_recover(to_rpc_app, from_app, to_app,
lifecycle_hook_info_app_update,
fluxcd_process_required=False)
self._perform_app_recover(to_rpc_app, from_app, to_app,
lifecycle_hook_info_app_update,
fluxcd_process_required=False)
return False
except Exception as e:
LOG.error("App {} operation {} semantic check error: {}"
"".format(to_app.name, constants.APP_UPDATE_OP, str(e)))
return self._perform_app_recover(to_rpc_app, from_app, to_app,
lifecycle_hook_info_app_update,
fluxcd_process_required=False)
self._perform_app_recover(to_rpc_app, from_app, to_app,
lifecycle_hook_info_app_update,
fluxcd_process_required=False)
return False
self.load_application_metadata_from_file(to_rpc_app)
# Check whether the new application is compatible with the current k8s version
self._utils._check_app_compatibility(to_app.name, to_app.version)
# Check whether the new application is compatible with the given k8s version.
# If k8s_version is none the check is performed against the active version.
self._utils._check_app_compatibility(to_app.name,
to_app.version,
k8s_version)
self._update_app_status(to_app, constants.APP_UPDATE_IN_PROGRESS)
@ -2853,8 +2859,9 @@ class AppOperator(object):
if do_recovery:
LOG.error("Application %s update from version %s to version "
"%s aborted." % (to_app.name, from_app.version, to_app.version))
return self._perform_app_recover(to_rpc_app, from_app, to_app,
lifecycle_hook_info_app_update)
self._perform_app_recover(to_rpc_app, from_app, to_app,
lifecycle_hook_info_app_update)
return False
self._update_app_status(to_app, constants.APP_UPDATE_IN_PROGRESS,
"cleanup application version {}".format(from_app.version))
@ -2928,9 +2935,10 @@ class AppOperator(object):
# ie.images download/k8s resource creation failure
# Start recovering without trigger fluxcd process
LOG.exception(e)
return self._perform_app_recover(to_rpc_app, from_app, to_app,
lifecycle_hook_info_app_update,
fluxcd_process_required=False)
self._perform_app_recover(to_rpc_app, from_app, to_app,
lifecycle_hook_info_app_update,
fluxcd_process_required=False)
return False
except Exception as e:
# Application update successfully(fluxcd apply/rollback)
# Error occurs during cleanup old app

272
sysinv/sysinv/sysinv/sysinv/conductor/manager.py
View File

@ -64,6 +64,7 @@ from controllerconfig.upgrades import management as upgrades_management
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric import rsa
from eventlet import greenpool
from eventlet import greenthread
# Make subprocess module greenthread friendly
from eventlet.green import subprocess
@ -275,13 +276,11 @@ class KubeAppBundleDatabase(KubeAppBundleStorageFactory):
"""Check if the table is empty."""
return self.dbapi.kube_app_bundle_is_empty()
def get_all(self):
def get_all(self, name=None, k8s_auto_update=None, k8s_timing=None):
"""Get a list containing all bundles."""
return self.dbapi.kube_app_bundle_get_all()
def get_by_name(self, app_name):
"""Get a list of bundles by their name."""
return self.dbapi.kube_app_bundle_get_by_name(app_name)
return self.dbapi.kube_app_bundle_get_all(name=name,
k8s_auto_update=k8s_auto_update,
k8s_timing=k8s_timing)
def destroy_all(self):
"""Prune all bundle metadata."""
@ -7425,18 +7424,117 @@ class ConductorManager(service.PeriodicService):
self._inner_sync_auto_apply(context, app_name)
def _get_app_bundle_for_update(self, app):
def update_apps_based_on_k8s_version_sync(self, context, k8s_version, k8s_upgrade_timing):
""" Update applications based on a given Kubernetes version (blocking).
:param context: Context of the request
:param k8s_version: Kubernetes target version.
:param k8s_upgrade_timing: When applications should be updated.
:return: True if all apps were successfully updated.
False if any apps failed to be updated.
"""
LOG.info("Checking available application updates for Kubernetes version {}."
.format(k8s_version))
update_candidates = [app_name for app_name in
self.apps_metadata[constants.APP_METADATA_APPS].keys()]
# Launch a thread for each update candidate, then wait for all applications
# to finish updating.
threadpool = greenpool.GreenPool(len(update_candidates))
threads = {}
result = True
for app_name in update_candidates:
try:
app = kubeapp_obj.get_by_name(context, app_name)
except exception.KubeAppNotFound:
continue
# Apps should be either in 'applied' or 'apply-failure' state.
# Applied apps are selected to be updated since they are currently in use.
# If the app is in 'apply-failure' state we give it a chance to be
# successfully applied via the update process.
if (app.status == constants.APP_APPLY_SUCCESS or
app.status == constants.APP_APPLY_FAILURE):
threads[app.name] = threadpool.spawn(self._auto_update_app,
context,
app_name,
k8s_version,
k8s_upgrade_timing,
async_update=False)
# Wait for all updates to finish
threadpool.waitall()
# Check result values
for app_name, thread in threads.items():
if thread.wait() is False:
LOG.error("Failed to update {} to match target Kubernetes version {}"
.format(app_name, k8s_version))
result = False
return result
def update_apps_based_on_k8s_version_async(self, context, k8s_version, k8s_upgrade_timing):
""" Update applications based on a given Kubernetes version (non-blocking).
:param context: Context of the request
:param k8s_version: Kubernetes target version.
:param k8s_upgrade_timing: When applications should be updated.
"""
update_candidates = [app_name for app_name in
self.apps_metadata[constants.APP_METADATA_APPS].keys()]
LOG.info("Checking available application updates for Kubernetes version {}."
.format(k8s_version))
for app_name in update_candidates:
try:
app = kubeapp_obj.get_by_name(context, app_name)
except exception.KubeAppNotFound:
continue
# Apps should be either in 'applied' or 'apply-failure' state.
# Applied apps are selected to be updated since they are currently in use.
# If the app is in 'apply-failure' state we give it a chance to be
# successfully applied via the update process.
if (app.status == constants.APP_APPLY_SUCCESS or
app.status == constants.APP_APPLY_FAILURE):
if self._auto_update_app(context,
app_name,
k8s_version,
k8s_upgrade_timing) is False:
LOG.error("Failed to update {} to match Kubernetes version {}"
.format(app_name, k8s_version))
def _get_app_bundle_for_update(self, app, k8s_version=None, k8s_upgrade_timing=None):
""" Retrieve metadata from the most updated application bundle
that can be used to update the given app.
:param app: The application to be updated
:param k8s_version: Target Kubernetes version
:param k8s_upgrade_timing: When applications should be updated during Kubernetes upgrades
:return The bundle metadata from the new version of the app
"""
bundle_metadata_list = self._kube_app_bundle_storage.get_by_name(app.name)
if k8s_upgrade_timing is None:
bundle_metadata_list = self._kube_app_bundle_storage.get_all(app.name)
else:
# Filter bundle list by the application name, k8s_auto_update = True and
# the given k8s_upgrade_timing.
bundle_metadata_list = self._kube_app_bundle_storage.get_all(app.name,
True,
k8s_upgrade_timing)
latest_version_bundle = None
k8s_version = self._kube.kube_get_kubernetes_version().strip().lstrip('v')
if k8s_version is None:
k8s_version = self._kube.kube_get_kubernetes_version().strip().lstrip('v')
else:
k8s_version = k8s_version.strip().lstrip('v')
for bundle_metadata in bundle_metadata_list:
if LooseVersion(bundle_metadata.version) <= LooseVersion(app.app_version):
LOG.debug("Bundle {} version {} lower than installed app version ({})"
@ -7446,9 +7544,6 @@ class ConductorManager(service.PeriodicService):
elif not bundle_metadata.auto_update:
LOG.debug("Application auto update disabled for bundle {}"
.format(bundle_metadata.file_path))
elif not bundle_metadata.k8s_auto_update:
LOG.debug("Kubernetes application auto update disabled for bundle {}"
.format(bundle_metadata.file_path))
elif LooseVersion(k8s_version) < LooseVersion(bundle_metadata.k8s_minimum_version):
LOG.debug("Kubernetes version {} is lower than {} which is "
"the minimum required for bundle {}"
@ -7471,17 +7566,33 @@ class ConductorManager(service.PeriodicService):
return latest_version_bundle
def _auto_update_app(self, context, app_name):
"""Auto update applications"""
def _auto_update_app(self,
context,
app_name,
k8s_version=None,
k8s_upgrade_timing=None,
async_update=True):
"""Auto update applications
:param context: Context of the request.
:param app_name: Name of the application to be updated.
:param k8s_version: Kubernetes target version.
:param k8s_upgrade_timing: When applications should be updated.
:param async_update: Update asynchronously if True. Update synchronously if False.
:return: True if the update successfully started when running asynchronously.
True if the app was successfully updated when running synchronously.
False if an error has occurred.
None if there is not an updated version available for the given app.
"""
try:
app = kubeapp_obj.get_by_name(context, app_name)
except exception.KubeAppNotFound as e:
LOG.exception(e)
return
return False
if app.status != constants.APP_APPLY_SUCCESS:
# In case the previous re-apply fails
return
return False
try:
hook_info = LifecycleHookInfo()
@ -7493,28 +7604,27 @@ class ConductorManager(service.PeriodicService):
self.app_lifecycle_actions(context, app, hook_info)
except exception.LifecycleSemanticCheckException as e:
LOG.info("Auto-update failed prerequisites for {}: {}".format(app.name, e))
return
return False
except exception.LifecycleSemanticCheckOperationNotSupported as e:
LOG.debug(e)
return
return False
except exception.SysinvException:
LOG.exception("Internal sysinv error while checking automatic "
"updates for {}"
.format(app.name))
return
return False
except Exception as e:
LOG.exception("Automatic operation:{} "
"for app {} failed with: {}".format(hook_info,
app.name,
e))
return
return False
if self._patching_operation_is_occurring():
return
return False
LOG.debug("Application %s: Checking "
"for update ..." % app_name)
app_bundle = self._get_app_bundle_for_update(app)
app_bundle = self._get_app_bundle_for_update(app, k8s_version, k8s_upgrade_timing)
if app_bundle is None:
# Skip if no bundles are found
LOG.debug("No bundle found for updating %s" % app_name)
@ -7529,23 +7639,30 @@ class ConductorManager(service.PeriodicService):
(tarball.manifest_name is None) or
(tarball.manifest_file is None)):
# Skip if tarball check fails
return
return False
if app_bundle.version in \
app.app_metadata.get(
constants.APP_METADATA_UPGRADES, {}).get(
constants.APP_METADATA_FAILED_VERSIONS, []):
constants.APP_METADATA_FAILED_VERSIONS, []) and \
k8s_version is None:
# Skip if this version was previously failed to
# be updated
# be updated. Allow retrying only if a Kubernetes version is
# defined, meaning that Kubernetes upgrade is in progress.
LOG.error("Application %s with version %s was previously "
"failed to be updated from version %s by auto-update"
% (app.name, tarball.app_version, app.app_version))
return
return False
self._inner_sync_auto_update(context, app, tarball)
return self._inner_sync_auto_update(context, app, tarball, k8s_version, async_update)
@cutils.synchronized(LOCK_APP_AUTO_MANAGE)
def _inner_sync_auto_update(self, context, applied_app, tarball):
def _inner_sync_auto_update(self,
context,
applied_app,
tarball,
k8s_version=None,
async_update=True):
# Check no other app is in progress of apply/update/recovery
for other_app in self.dbapi.kube_app_get_all():
if other_app.status in [constants.APP_APPLY_IN_PROGRESS,
@ -7555,7 +7672,7 @@ class ConductorManager(service.PeriodicService):
"is in progress of apply/update/recovery. "
"Will retry on next audit",
applied_app.name, other_app.name)
return
return False
# Set the status for the current applied app to inactive
applied_app.status = constants.APP_INACTIVE_STATE
@ -7590,14 +7707,36 @@ class ConductorManager(service.PeriodicService):
applied_app.progress = constants.APP_PROGRESS_COMPLETED
applied_app.save()
LOG.exception(e)
return
return False
LOG.info("Platform managed application %s: "
"Auto updating..." % target_app.name)
hook_info = LifecycleHookInfo()
hook_info.mode = constants.APP_LIFECYCLE_MODE_AUTO
greenthread.spawn(self.perform_app_update, context, applied_app,
target_app, tarball.tarball_name, operation, hook_info)
if async_update:
greenthread.spawn(self.perform_app_update,
context,
applied_app,
target_app,
tarball.tarball_name,
operation,
hook_info,
None,
None,
k8s_version)
else:
return self.perform_app_update(context,
applied_app,
target_app,
tarball.tarball_name,
operation,
hook_info,
None,
None,
k8s_version)
return True
def _search_tarfile(self, app_name, managed_app):
"""Search a specified application tarfile from the directory
@ -16044,7 +16183,7 @@ class ConductorManager(service.PeriodicService):
def perform_app_update(self, context, from_rpc_app, to_rpc_app, tarfile,
operation, lifecycle_hook_info_app_update, reuse_user_overrides=None,
reuse_attributes=None):
reuse_attributes=None, k8s_version=None):
"""Handling of application update request (via AppOperator)
:param context: request context.
@ -16061,9 +16200,14 @@ class ConductorManager(service.PeriodicService):
"""
lifecycle_hook_info_app_update.operation = constants.APP_UPDATE_OP
self._app.perform_app_update(from_rpc_app, to_rpc_app, tarfile,
operation, lifecycle_hook_info_app_update, reuse_user_overrides,
reuse_attributes)
return self._app.perform_app_update(from_rpc_app,
to_rpc_app,
tarfile,
operation,
lifecycle_hook_info_app_update,
reuse_user_overrides,
reuse_attributes,
k8s_version)
def perform_app_remove(self, context, rpc_app, lifecycle_hook_info_app_remove, force=False):
"""Handling of application removal request (via AppOperator)
@ -16332,6 +16476,60 @@ class ConductorManager(service.PeriodicService):
LOG.info("Successfully completed k8s control plane backup.")
def _check_installed_apps_compatibility(self, kube_version):
"""Checks whether all installed applications are compatible
with the new k8s version
:param kube_version: Target Kubernetes version
:return: True if all apps are compatible with the given Kubernetes version
False if any apps are incompatible with the given Kubernetes version
"""
# Check that all installed applications support new k8s version
apps = self.dbapi.kube_app_get_all()
success = True
for app in apps:
if app.status != constants.APP_APPLY_SUCCESS:
continue
kube_min_version, kube_max_version = \
cutils.get_app_supported_kube_version(app.name, app.app_version)
if not kubernetes.is_kube_version_supported(
kube_version, kube_min_version, kube_max_version):
LOG.error("The installed Application {} ({}) is incompatible with the "
"new Kubernetes version {}.".format(app.name,
app.app_version,
kube_version))
success = False
return success
def kube_upgrade_start(self, context, k8s_version):
""" Start a Kubernetes upgrade by updating all required apps.
:param context: Context of the request.
:param k8s_version: Kubernetes target version.
:param k8s_upgrade_timing: When apps should be updated.
"""
kube_upgrade_obj = objects.kube_upgrade.get_one(context)
if (self.update_apps_based_on_k8s_version_sync(context,
k8s_version,
constants.APP_METADATA_TIMING_PRE) and
self._check_installed_apps_compatibility(k8s_version)):
kube_upgrade_obj.state = kubernetes.KUBE_UPGRADE_STARTED
LOG.info("Started kubernetes upgrade from version: %s to version: %s"
% (kube_upgrade_obj.from_version, kube_upgrade_obj.to_version))
else:
kube_upgrade_obj.state = kubernetes.KUBE_UPGRADE_STARTING_FAILED
LOG.info("Failed to start kubernetes upgrade from version: %s to version: %s"
% (kube_upgrade_obj.from_version, kube_upgrade_obj.to_version))
kube_upgrade_obj.save()
def kube_download_images(self, context, kube_version):
"""Download the kubernetes images for this version"""

26
sysinv/sysinv/sysinv/sysinv/conductor/rpcapi.py
View File

@ -1716,6 +1716,32 @@ class ConductorAPI(sysinv.openstack.common.rpc.proxy.RpcProxy):
return self.call(context, self.make_msg('get_fernet_keys',
key_id=key_id))
def kube_upgrade_start(self, context, k8s_version):
"""Asynchronously, start a Kubernetes upgrade to the given version.
:param context: Context of the request
:param k8s_version: Kubernetes target version
:param k8s_upgrade_timing: When applications should be updated
"""
return self.cast(context, self.make_msg('kube_upgrade_start',
k8s_version=k8s_version))
def update_apps_based_on_k8s_version_async(self,
context,
k8s_version,
k8s_upgrade_timing):
"""Asynchronously, update all applications based on a given Kubernetes version.
:param context: Context of the request
:param k8s_version: Kubernetes target version
:param k8s_upgrade_timing: When applications should be updated
"""
return self.cast(context, self.make_msg('update_apps_based_on_k8s_version_async',
k8s_version=k8s_version,
k8s_upgrade_timing=k8s_upgrade_timing))
def evaluate_apps_reapply(self, context, trigger):
"""Synchronously, determine whether an application
re-apply is needed, and if so, raise the re-apply flag.

26
sysinv/sysinv/sysinv/sysinv/db/api.py
View File

@ -5122,6 +5122,8 @@ class Connection(object):
@abc.abstractmethod
def kube_app_bundle_get_all(self,
name=None,
k8s_auto_update=None,
timing=None,
limit=None,
marker=None,
sort_key=None,
@ -5130,25 +5132,11 @@ class Connection(object):
given filter.
:param name: Application name.
:param limit: Maximum number of entries to return.
:param marker: The last item of the previous page; we return the next
result set.
:param sort_key: Attribute by which results should be sorted.
:param sort_dir: Direction in which results should be sorted.
(asc, desc)
:returns: A list of kube_app_bundle entries with the given name.
"""
@abc.abstractmethod
def kube_app_bundle_get_by_name(self,
name,
limit=None,
marker=None,
sort_key=None,
sort_dir=None):
"""Get kube_app_bundle entries that match a given name
:param name: Application name.
:param k8s_auto_update: Whether automatically updating the application
is enabled when upgrading Kubernetes.
:param timing: Application update timing during Kubernetes upgrade
"pre": during kube-upgrade-start.
"post": during kube-upgrade-complete.
:param limit: Maximum number of entries to return.
:param marker: The last item of the previous page; we return the next
result set.

16
sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/api.py
View File

@ -9455,24 +9455,20 @@ class Connection(api.Connection):
return result is None
@db_objects.objectify(objects.kube_app_bundle)
def kube_app_bundle_get_all(self, name=None,
limit=None, marker=None,
def kube_app_bundle_get_all(self, name=None, k8s_auto_update=None,
k8s_timing=None, limit=None, marker=None,
sort_key=None, sort_dir=None):
query = model_query(models.KubeAppBundle)
if name:
query = query.filter_by(name=name)
if k8s_auto_update:
query = query.filter_by(k8s_auto_update=k8s_auto_update)
if k8s_timing:
query = query.filter_by(k8s_timing=k8s_timing)
return _paginate_query(models.KubeAppBundle, limit, marker,
sort_key, sort_dir, query)
@db_objects.objectify(objects.kube_app_bundle)
def kube_app_bundle_get_by_name(self, name,
limit=None, marker=None,
sort_key=None, sort_dir=None):
return self.kube_app_bundle_get_all(name, limit, marker,
sort_key, sort_dir)
def kube_app_bundle_destroy_all(self, file_path=None):
with _session_for_write() as session:
query = model_query(models.KubeAppBundle, session=session)

72
sysinv/sysinv/sysinv/sysinv/tests/api/test_kube_upgrade.py
View File

@ -72,9 +72,11 @@ class FakeFmClient(object):
class FakeConductorAPI(object):
def __init__(self):
self.kube_upgrade_start = mock.MagicMock()
self.kube_download_images = mock.MagicMock()
self.kube_upgrade_networking = mock.MagicMock()
self.kube_upgrade_abort = mock.MagicMock()
self.update_apps_based_on_k8s_version_async = mock.MagicMock()
self.evaluate_apps_reapply = mock.MagicMock()
self.remove_kube_control_plane_backup = mock.MagicMock()
self.kube_delete_container_images = mock.MagicMock()
@ -170,22 +172,6 @@ class TestKubeUpgrade(base.FunctionalTest):
self.mocked_kube_get_version_states.start()
self.addCleanup(self.mocked_kube_get_version_states.stop)
# Mock utility function
self.kube_min_version_result, self.kube_max_version_result = 'v1.42.1', 'v1.43.1'
def mock_get_app_supported_kube_version(app_name, app_version):
return self.kube_min_version_result, self.kube_max_version_result
self.mocked_kube_min_version = mock.patch(
'sysinv.common.utils.get_app_supported_kube_version',
mock_get_app_supported_kube_version)
self.mocked_kube_max_version = mock.patch(
'sysinv.common.utils.get_app_supported_kube_version',
mock_get_app_supported_kube_version)
self.mocked_kube_min_version.start()
self.mocked_kube_max_version.start()
self.addCleanup(self.mocked_kube_min_version.stop)
self.addCleanup(self.mocked_kube_max_version.stop)
self.setup_health_mocked_calls()
def setup_health_mocked_calls(self):
@ -288,11 +274,15 @@ class TestPostKubeUpgradeSimplex(TestKubeUpgrade,
result = self.post_json('/kube_upgrade', create_dict,
headers={'User-Agent': 'sysinv-test'})
# Verify that the upgrade was started
self.fake_conductor_api.kube_upgrade_start.\
assert_called_with(mock.ANY, 'v1.43.3')
# Verify that the upgrade has the expected attributes
self.assertEqual(result.json['from_version'], 'v1.42.1')
self.assertEqual(result.json['to_version'], 'v1.43.3')
self.assertEqual(result.json['state'],
kubernetes.KUBE_UPGRADE_STARTED)
kubernetes.KUBE_UPGRADE_STARTING)
# see if kubeadm_version was changed in DB
kube_cmd_version = self.dbapi.kube_cmd_version_get()
@ -390,11 +380,15 @@ class TestPostKubeUpgrade(TestKubeUpgrade,
result = self.post_json('/kube_upgrade', create_dict,
headers={'User-Agent': 'sysinv-test'})
# Verify that the upgrade was started
self.fake_conductor_api.kube_upgrade_start.\
assert_called_with(mock.ANY, 'v1.43.2')
# Verify that the upgrade has the expected attributes
self.assertEqual(result.json['from_version'], 'v1.43.1')
self.assertEqual(result.json['to_version'], 'v1.43.2')
self.assertEqual(result.json['state'],
kubernetes.KUBE_UPGRADE_STARTED)
kubernetes.KUBE_UPGRADE_STARTING)
# see if kubeadm_version was changed in DB
kube_cmd_version = self.dbapi.kube_cmd_version_get()
@ -485,30 +479,6 @@ class TestPostKubeUpgrade(TestKubeUpgrade,
self.assertIn("version v1.43.1 is not active",
result.json['error_message'])
def test_create_installed_app_not_compatible(self):
# Test creation of upgrade when the installed application isn't
# compatible with the new kubernetes version
# Create application
dbutils.create_test_app(
name='stx-openstack',
app_version='1.0-19',
manifest_name='manifest',
manifest_file='stx-openstack.yaml',
status='applied',
active=True)
create_dict = dbutils.post_get_test_kube_upgrade(to_version='v1.43.2')
result = self.post_json('/kube_upgrade', create_dict,
headers={'User-Agent': 'sysinv-test'},
expect_errors=True)
# Verify the failure
self.assertEqual(result.content_type, 'application/json')
self.assertEqual(http_client.BAD_REQUEST, result.status_int)
self.assertIn("incompatible with the new Kubernetes version v1.43.2",
result.json['error_message'])
@mock.patch('sysinv.common.health.Health._check_trident_compatibility', lambda x: True)
def test_create_system_unhealthy_from_alarms(self):
"""Test creation of a kube upgrade while there are alarms"""
@ -542,11 +512,15 @@ class TestPostKubeUpgrade(TestKubeUpgrade,
result = self.post_json('/kube_upgrade', create_dict,
headers={'User-Agent': 'sysinv-test'})
# Verify that the upgrade was started
self.fake_conductor_api.kube_upgrade_start.\
assert_called_with(mock.ANY, 'v1.43.2')
# Verify that the upgrade has the expected attributes
self.assertEqual(result.json['from_version'], 'v1.43.1')
self.assertEqual(result.json['to_version'], 'v1.43.2')
self.assertEqual(result.json['state'],
kubernetes.KUBE_UPGRADE_STARTED)
kubernetes.KUBE_UPGRADE_STARTING)
@mock.patch('sysinv.common.health.Health._check_trident_compatibility', lambda x: True)
def test_force_create_system_unhealthy_from_mgmt_affecting_alarms(self):
@ -583,11 +557,15 @@ class TestPostKubeUpgrade(TestKubeUpgrade,
result = self.post_json('/kube_upgrade', create_dict,
headers={'User-Agent': 'sysinv-test'})
# Verify that the upgrade was started
self.fake_conductor_api.kube_upgrade_start.\
assert_called_with(mock.ANY, 'v1.43.2')
# Verify that the upgrade has the expected attributes
self.assertEqual(result.json['from_version'], 'v1.43.1')
self.assertEqual(result.json['to_version'], 'v1.43.2')
self.assertEqual(result.json['state'],
kubernetes.KUBE_UPGRADE_STARTED)
kubernetes.KUBE_UPGRADE_STARTING)
@mock.patch('sysinv.common.health.Health._check_trident_compatibility', lambda x: True)
def test_create_system_unhealthy_from_bad_apps(self):
@ -629,11 +607,15 @@ class TestPostKubeUpgrade(TestKubeUpgrade,
result = self.post_json('/kube_upgrade', create_dict,
headers={'User-Agent': 'sysinv-test'})
# Verify that the upgrade was started
self.fake_conductor_api.kube_upgrade_start.\
assert_called_with(mock.ANY, 'v1.43.3')
# Verify that the upgrade has the expected attributes
self.assertEqual(result.json['from_version'], 'v1.43.2')
self.assertEqual(result.json['to_version'], 'v1.43.3')
self.assertEqual(result.json['state'],
kubernetes.KUBE_UPGRADE_STARTED)
kubernetes.KUBE_UPGRADE_STARTING)
def test_create_applied_patch_missing(self):
# Test creation of upgrade when applied patch is missing

74
sysinv/sysinv/sysinv/sysinv/tests/conductor/test_manager.py
View File

@ -49,6 +49,7 @@ from sysinv.db import api as dbapi
from sysinv.loads.loads import LoadImport
from sysinv.objects.load import Load
from sysinv.puppet import common as puppet_common
from sysinv.tests.db import utils as dbutils
from sysinv import objects
from sysinv.tests.db import base
@ -497,6 +498,22 @@ class ManagerTestCase(base.DbTestCase):
self.alarm_raised = False
self.kernel_alarms = {}
# Mock utility function
self.kube_min_version_result, self.kube_max_version_result = 'v1.42.1', 'v1.43.1'
def mock_get_app_supported_kube_version(app_name, app_version):
return self.kube_min_version_result, self.kube_max_version_result
self.mocked_kube_min_version = mock.patch(
'sysinv.common.utils.get_app_supported_kube_version',
mock_get_app_supported_kube_version)
self.mocked_kube_max_version = mock.patch(
'sysinv.common.utils.get_app_supported_kube_version',
mock_get_app_supported_kube_version)
self.mocked_kube_min_version.start()
self.mocked_kube_max_version.start()
self.addCleanup(self.mocked_kube_min_version.stop)
self.addCleanup(self.mocked_kube_max_version.stop)
def tearDown(self):
super(ManagerTestCase, self).tearDown()
@ -931,6 +948,63 @@ class ManagerTestCase(base.DbTestCase):
ret = self.service.platform_interfaces(self.context, ihost['id'] + 1)
self.assertEqual(ret, [])
def test_kube_upgrade_start(self):
# Create application
dbutils.create_test_app(
name='stx-openstack',
app_version='1.0-19',
manifest_name='manifest',
manifest_file='stx-openstack.yaml',
status='applied',
active=True)
# Create an upgrade
from_version = 'v1.42.1'
to_version = 'v1.43.1'
utils.create_test_kube_upgrade(
from_version=from_version,
to_version=to_version,
state=kubernetes.KUBE_UPGRADE_STARTING,
)
# Start upgrade
self.service.kube_upgrade_start(self.context, to_version)
# Verify that the upgrade state was updated
updated_upgrade = self.dbapi.kube_upgrade_get_one()
self.assertEqual(updated_upgrade.state,
kubernetes.KUBE_UPGRADE_STARTED)
def test_kube_upgrade_start_app_not_compatible(self):
# Test creation of upgrade when the installed application isn't
# compatible with the new kubernetes version
# Create application
dbutils.create_test_app(
name='stx-openstack',
app_version='1.0-19',
manifest_name='manifest',
manifest_file='stx-openstack.yaml',
status='applied',
active=True)
# Create an upgrade
from_version = 'v1.42.1'
to_version = 'v1.43.2'
utils.create_test_kube_upgrade(
from_version=from_version,
to_version=to_version,
state=kubernetes.KUBE_UPGRADE_STARTING,
)
# Start upgrade
self.service.kube_upgrade_start(self.context, to_version)
# Verify that the upgrade state was updated
updated_upgrade = self.dbapi.kube_upgrade_get_one()
self.assertEqual(updated_upgrade.state,
kubernetes.KUBE_UPGRADE_STARTING_FAILED)
def test_kube_download_images(self):
# Create controller-0
config_uuid = str(uuid.uuid4())