From 46f5ccfc55a9c5792c5604412d1a3362792ace20 Mon Sep 17 00:00:00 2001 From: Igor Soares Date: Mon, 8 Jan 2024 12:39:35 -0300 Subject: [PATCH] Update apps during Kubernetes upgrade Update StarlingX applications during Kubernetes upgrades according to their metadata. Applications are updated if they have "k8s_upgrades:auto_update" set to true on their metadata.yaml file. The ones that have "k8s_upgrades:timing" set to "pre" are updated during the "kube-upgrade-start" phase. The ones that set "k8s_upgrades:timing" to "post" are updated during the "kube-upgrade-complete" phase. In order to better support application updates during kube-upgrade-start, two new statuses were added: 'upgrade-starting' and 'upgrade-starting-failed'. The 'upgrade-starting' state is the new initial state when triggering a Kubernetes upgrade. If starting the upgrade fails, then the status is updated to 'upgrade-starting-failed' and users can either abort the upgrade (only available in simplex loads) or try starting it again. No changes were made to kube-upgrade-complete in that regard because at that point a new Kubernetes version is already in place. A review was raised to nfv-vim to reflect the new statuses on the Kubernetes orchestrated upgrade code: https://review.opendev.org/c/starlingx/nfv/+/906594 Application auto updates can be retried when restarting a Kubernetes upgrade that previously failed due to a failing app. A bug that was preventing the k8s_upgrade metadata section from being parsed was fixed by this commit as well. Test Plan: PASS: build-pkgs -a && build-image PASS: Create a new platform-integ-apps tarball adding "k8s_upgrades:auto_update=true" and "k8s_upgrades:timing=pre" to metadata.yaml. Add the new tarball to /usr/local/share/applications/helm/. Run kube-upgrade-start. Check if platform-integ-apps was correctly updated. Check if no other apps were updated. PASS: Create a new metrics-server tarball adding "k8s_upgrades:auto_update=true" and "k8s_upgrades:timing=post" to to metadata.yaml. Add the new tarball to /usr/local/share/applications/helm/. Run kube-upgrade-complete. Check if metrics-server was correctly updated. Check if no other apps were updated. PASS: Create a new platform-integ-apps tarball adding "k8s_upgrades:auto_update=true" and "k8s_upgrades:timing=pre" to metadata.yaml. Add the new tarball to /usr/local/share/applications/helm/. Restart sysinv to update the database. Replace the platform-integ-apps tarball with another tarball that does not have a metadata.yaml file. Check if an error is logged when running kube-upgrade-start reporting that platform-integ-apps failed to be updated. Confirm that the Kubernetes upgrade was not started. Abort Kubernetes upgrade Check if upgrade was successfully aborted PASS: Create a new metrics-server tarball adding "k8s_upgrades:auto_update=true" and "k8s_upgrades:timing=post" to to metadata.yaml. Add the new tarball to /usr/local/share/applications/helm/. Restart sysinv to update the database. Replace the snmp tarball with another tarball that does not have a metadata.yaml file. Check if an error is logged when running kube-upgrade-complete reporting that metrics-server failed to be updated. Check if the Kubernetes upgrade was marked as completed. PASS: AIO-SX fresh install Manual upgrade to Kubernetes v1.27.5 Check if upgrade was successfuly done PASS: AIO-SX fresh install Orchestrated upgrade to Kubernetes v1.27.5 Check if upgrade was successfuly done PASS: AIO-SX fresh install with Kubernetes v1.24.4 Orchestrated upgrade to Kubernetes v1.27.5 Check if upgrade was successfuly done Story: 2010929 Task: 49416 Change-Id: I31333bf44501c7ad1688635b75c7fcef11513026 Signed-off-by: Igor Soares --- .../sysinv/api/controllers/v1/kube_app.py | 29 +- .../sysinv/api/controllers/v1/kube_upgrade.py | 139 ++++----- .../sysinv/sysinv/common/app_metadata.py | 4 +- .../sysinv/sysinv/sysinv/common/kubernetes.py | 2 + .../sysinv/sysinv/conductor/kube_app.py | 38 ++- .../sysinv/sysinv/sysinv/conductor/manager.py | 272 +++++++++++++++--- .../sysinv/sysinv/sysinv/conductor/rpcapi.py | 26 ++ sysinv/sysinv/sysinv/sysinv/db/api.py | 26 +- .../sysinv/sysinv/sysinv/db/sqlalchemy/api.py | 16 +- .../sysinv/tests/api/test_kube_upgrade.py | 72 ++--- .../sysinv/tests/conductor/test_manager.py | 74 +++++ 11 files changed, 497 insertions(+), 201 deletions(-) diff --git a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/kube_app.py b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/kube_app.py index ce0f300fb0..1ddc9f9e20 100644 --- a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/kube_app.py +++ b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/kube_app.py @@ -665,7 +665,7 @@ class KubeAppHelper(object): "Error while reporting the patch dependencies " "to patch-controller.") - def _check_app_compatibility(self, app_name, app_version): + def _check_app_compatibility(self, app_name, app_version, target_kube_version=None): """Checks whether the application is compatible with the current k8s version""" @@ -675,14 +675,25 @@ class KubeAppHelper(object): if not kube_min_version and not kube_max_version: return - version_states = self._kube_operator.kube_get_version_states() - for kube_version, state in version_states.items(): - if state in [kubernetes.KUBE_STATE_ACTIVE, - kubernetes.KUBE_STATE_PARTIAL]: - if not kubernetes.is_kube_version_supported( - kube_version, kube_min_version, kube_max_version): - raise exception.IncompatibleKubeVersion( - name=app_name, version=app_version, kube_version=kube_version) + if target_kube_version is None: + version_states = self._kube_operator.kube_get_version_states() + for kube_version, state in version_states.items(): + if state in [kubernetes.KUBE_STATE_ACTIVE, + kubernetes.KUBE_STATE_PARTIAL]: + if not kubernetes.is_kube_version_supported( + kube_version, kube_min_version, kube_max_version): + LOG.error("Application {} is incompatible with Kubernetes version {}." + .format(app_name, kube_version)) + raise exception.IncompatibleKubeVersion( + name=app_name, version=app_version, kube_version=kube_version) + elif not kubernetes.is_kube_version_supported(target_kube_version, + kube_min_version, + kube_max_version): + LOG.error("Application {} is incompatible with target Kubernetes version {}." + .format(app_name, target_kube_version)) + raise exception.IncompatibleKubeVersion(name=app_name, + version=app_version, + kube_version=target_kube_version) def _find_manifest(self, app_path, app_name): """ Find the required application manifest elements diff --git a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/kube_upgrade.py b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/kube_upgrade.py index 0d203bf014..be0f4d816f 100755 --- a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/kube_upgrade.py +++ b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/kube_upgrade.py @@ -152,24 +152,6 @@ class KubeUpgradeController(rest.RestController): "the kubernetes upgrade: %s" % available_patches)) - @staticmethod - def _check_installed_apps_compatibility(apps, kube_version): - """Checks whether all installed applications are compatible - with the new k8s version""" - - for app in apps: - if app.status != constants.APP_APPLY_SUCCESS: - continue - - kube_min_version, kube_max_version = \ - cutils.get_app_supported_kube_version(app.name, app.app_version) - - if not kubernetes.is_kube_version_supported( - kube_version, kube_min_version, kube_max_version): - raise wsme.exc.ClientSideError(_( - "The installed Application %s (%s) is incompatible with the " - "new Kubernetes version %s." % (app.name, app.app_version, kube_version))) - @wsme_pecan.wsexpose(KubeUpgradeCollection) def get_all(self): """Retrieve a list of kubernetes upgrades.""" @@ -193,6 +175,7 @@ class KubeUpgradeController(rest.RestController): force = body.get('force', False) is True alarm_ignore_list = body.get('alarm_ignore_list') system = pecan.request.dbapi.isystem_get_one() + retry = False # There must not be a platform upgrade in progress try: @@ -206,12 +189,21 @@ class KubeUpgradeController(rest.RestController): # There must not already be a kubernetes upgrade in progress try: - pecan.request.dbapi.kube_upgrade_get_one() + kube_upgrade_obj = objects.kube_upgrade.get_one(pecan.request.context) except exception.NotFound: pass else: - raise wsme.exc.ClientSideError(_( - "A kubernetes upgrade is already in progress")) + # Allow retrying the new Kubernetes upgrade if the current + # state is 'upgrade-starting-failed'. + if (kube_upgrade_obj.state == kubernetes.KUBE_UPGRADE_STARTING_FAILED and + kube_upgrade_obj.to_version == to_version): + retry = True + if alarm_ignore_list is None: + alarm_ignore_list = [] + alarm_ignore_list.append(fm_constants.FM_ALARM_ID_KUBE_UPGRADE_IN_PROGRESS) + else: + raise wsme.exc.ClientSideError(_( + "A kubernetes upgrade is already in progress")) # Check whether target version is available or not try: @@ -252,10 +244,6 @@ class KubeUpgradeController(rest.RestController): applied_patches=target_version_obj.applied_patches, available_patches=target_version_obj.available_patches) - # Check that all installed applications support new k8s version - apps = pecan.request.dbapi.kube_app_get_all() - self._check_installed_apps_compatibility(apps, to_version) - # The system must be healthy success, output = pecan.request.rpcapi.get_system_health( pecan.request.context, @@ -272,48 +260,63 @@ class KubeUpgradeController(rest.RestController): "System is not in a valid state for kubernetes upgrade. " "Run system health-query-kube-upgrade for more details.")) - # Create upgrade record. - create_values = {'from_version': current_kube_version, - 'to_version': to_version, - 'state': kubernetes.KUBE_UPGRADE_STARTED} - new_upgrade = pecan.request.dbapi.kube_upgrade_create(create_values) + if retry: + # Update upgrade record + kube_upgrade_obj.state = kubernetes.KUBE_UPGRADE_STARTING + kube_upgrade_obj.save() + else: + # Create upgrade record. + create_values = {'from_version': current_kube_version, + 'to_version': to_version, + 'state': kubernetes.KUBE_UPGRADE_STARTING} + kube_upgrade_obj = pecan.request.dbapi.kube_upgrade_create(create_values) - # Set the target version for each host to the current version - update_values = {'target_version': current_kube_version} - kube_host_upgrades = pecan.request.dbapi.kube_host_upgrade_get_list() - for kube_host_upgrade in kube_host_upgrades: - pecan.request.dbapi.kube_host_upgrade_update(kube_host_upgrade.id, - update_values) - # Raise alarm to show a kubernetes upgrade is in progress - entity_instance_id = "%s=%s" % (fm_constants.FM_ENTITY_TYPE_HOST, - constants.CONTROLLER_HOSTNAME) - fault = fm_api.Fault( - alarm_id=fm_constants.FM_ALARM_ID_KUBE_UPGRADE_IN_PROGRESS, - alarm_state=fm_constants.FM_ALARM_STATE_SET, - entity_type_id=fm_constants.FM_ENTITY_TYPE_HOST, - entity_instance_id=entity_instance_id, - severity=fm_constants.FM_ALARM_SEVERITY_MINOR, - reason_text="Kubernetes upgrade in progress.", - # operational - alarm_type=fm_constants.FM_ALARM_TYPE_7, - # congestion - probable_cause=fm_constants.ALARM_PROBABLE_CAUSE_8, - proposed_repair_action="No action required.", - service_affecting=False) - fm_api.FaultAPIs().set_fault(fault) + try: + # Set the target version for each host to the current version + update_values = {'target_version': current_kube_version} + kube_host_upgrades = pecan.request.dbapi.kube_host_upgrade_get_list() + for kube_host_upgrade in kube_host_upgrades: + pecan.request.dbapi.kube_host_upgrade_update(kube_host_upgrade.id, + update_values) + # Raise alarm to show a kubernetes upgrade is in progress + entity_instance_id = "%s=%s" % (fm_constants.FM_ENTITY_TYPE_HOST, + constants.CONTROLLER_HOSTNAME) + fault = fm_api.Fault( + alarm_id=fm_constants.FM_ALARM_ID_KUBE_UPGRADE_IN_PROGRESS, + alarm_state=fm_constants.FM_ALARM_STATE_SET, + entity_type_id=fm_constants.FM_ENTITY_TYPE_HOST, + entity_instance_id=entity_instance_id, + severity=fm_constants.FM_ALARM_SEVERITY_MINOR, + reason_text="Kubernetes upgrade in progress.", + # operational + alarm_type=fm_constants.FM_ALARM_TYPE_7, + # congestion + probable_cause=fm_constants.ALARM_PROBABLE_CAUSE_8, + proposed_repair_action="No action required.", + service_affecting=False) + fm_api.FaultAPIs().set_fault(fault) - # Set the new kubeadm version in the DB. - # This will not actually change the bind mounts until we apply a - # puppet manifest that makes use of it. - kube_cmd_versions = objects.kube_cmd_version.get( - pecan.request.context) - kube_cmd_versions.kubeadm_version = to_version.lstrip('v') - kube_cmd_versions.save() + # Set the new kubeadm version in the DB. + # This will not actually change the bind mounts until we apply a + # puppet manifest that makes use of it. + kube_cmd_versions = objects.kube_cmd_version.get( + pecan.request.context) + kube_cmd_versions.kubeadm_version = to_version.lstrip('v') + kube_cmd_versions.save() - LOG.info("Started kubernetes upgrade from version: %s to version: %s" - % (current_kube_version, to_version)) + LOG.info("Starting kubernetes upgrade from version: %s to version: %s" + % (current_kube_version, to_version)) - return KubeUpgrade.convert_with_links(new_upgrade) + # Tell the conductor to update the required apps and mark the upgrade as started + pecan.request.rpcapi.kube_upgrade_start( + pecan.request.context, + to_version) + except Exception as e: + LOG.exception("Failed to start Kubernetes upgrade: %s" % e) + kube_upgrade_obj.state = kubernetes.KUBE_UPGRADE_STARTING_FAILED + kube_upgrade_obj.save() + + return KubeUpgrade.convert_with_links(kube_upgrade_obj) @cutils.synchronized(LOCK_NAME) @wsme.validate([KubeUpgradePatchType]) @@ -458,7 +461,9 @@ class KubeUpgradeController(rest.RestController): # Update the state as aborted for these states since no actual k8s changes done # so we don't need to do anything more to complete the abort. - if kube_upgrade_obj.state in [kubernetes.KUBE_UPGRADE_STARTED, + if kube_upgrade_obj.state in [kubernetes.KUBE_UPGRADE_STARTING, + kubernetes.KUBE_UPGRADE_STARTING_FAILED, + kubernetes.KUBE_UPGRADE_STARTED, kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES, kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED, kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES]: @@ -622,6 +627,12 @@ class KubeUpgradeController(rest.RestController): if role == constants.DISTRIBUTED_CLOUD_ROLE_SYSTEMCONTROLLER: dc_api.notify_dcmanager_kubernetes_upgrade_completed() + # Update apps that contain 'k8s_upgrade.timing = post' metadata + pecan.request.rpcapi.update_apps_based_on_k8s_version_async( + pecan.request.context, + kube_upgrade_obj.to_version, + constants.APP_METADATA_TIMING_POST) + # Check if apps need to be reapplied pecan.request.rpcapi.evaluate_apps_reapply( pecan.request.context, diff --git a/sysinv/sysinv/sysinv/sysinv/common/app_metadata.py b/sysinv/sysinv/sysinv/sysinv/common/app_metadata.py index e0aab54a11..f8e6de7d6f 100644 --- a/sysinv/sysinv/sysinv/sysinv/common/app_metadata.py +++ b/sysinv/sysinv/sysinv/sysinv/common/app_metadata.py @@ -611,11 +611,11 @@ def extract_bundle_metadata(file_path): LOG.warning("k8s_upgrades section missing from {} metadata" .format(file_path)) else: - k8s_auto_update = tarball.metadata.get( + k8s_auto_update = metadata.get( constants.APP_METADATA_K8S_UPGRADES).get( constants.APP_METADATA_AUTO_UPDATE, constants.APP_METADATA_K8S_AUTO_UPDATE_DEFAULT_VALUE) - k8s_update_timing = tarball.metadata.get( + k8s_update_timing = metadata.get( constants.APP_METADATA_K8S_UPGRADES).get( constants.APP_METADATA_TIMING, constants.APP_METADATA_TIMING_DEFAULT_VALUE) diff --git a/sysinv/sysinv/sysinv/sysinv/common/kubernetes.py b/sysinv/sysinv/sysinv/sysinv/common/kubernetes.py index 4e5af410b9..611dc07f19 100644 --- a/sysinv/sysinv/sysinv/sysinv/common/kubernetes.py +++ b/sysinv/sysinv/sysinv/sysinv/common/kubernetes.py @@ -78,6 +78,8 @@ KUBE_CONTROLLER_MANAGER = 'kube-controller-manager' KUBE_SCHEDULER = 'kube-scheduler' # Kubernetes upgrade states +KUBE_UPGRADE_STARTING = 'upgrade-starting' +KUBE_UPGRADE_STARTING_FAILED = 'upgrade-starting-failed' KUBE_UPGRADE_STARTED = 'upgrade-started' KUBE_UPGRADE_DOWNLOADING_IMAGES = 'downloading-images' KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED = 'downloading-images-failed' diff --git a/sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py b/sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py index b447f201d6..31d8f625e7 100644 --- a/sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py +++ b/sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py @@ -2704,7 +2704,7 @@ class AppOperator(object): def perform_app_update(self, from_rpc_app, to_rpc_app, tarfile, operation, lifecycle_hook_info_app_update, reuse_user_overrides=None, - reuse_attributes=None): + reuse_attributes=None, k8s_version=None): """Process application update request This method leverages the existing application upload workflow to @@ -2733,7 +2733,8 @@ class AppOperator(object): :param lifecycle_hook_info_app_update: LifecycleHookInfo object :param reuse_user_overrides: (optional) True or False :param reuse_attributes: (optional) True or False - + :param k8s_version: (optional) Target Kubernetes version + :return: True if the update to the new version was successful. False otherwise. """ from_app = AppOperator.Application(from_rpc_app) @@ -2777,20 +2778,25 @@ class AppOperator(object): except exception.LifecycleSemanticCheckException as e: LOG.info("App {} rejected operation {} for reason: {}" "".format(to_app.name, constants.APP_UPDATE_OP, str(e))) - return self._perform_app_recover(to_rpc_app, from_app, to_app, - lifecycle_hook_info_app_update, - fluxcd_process_required=False) + self._perform_app_recover(to_rpc_app, from_app, to_app, + lifecycle_hook_info_app_update, + fluxcd_process_required=False) + return False except Exception as e: LOG.error("App {} operation {} semantic check error: {}" "".format(to_app.name, constants.APP_UPDATE_OP, str(e))) - return self._perform_app_recover(to_rpc_app, from_app, to_app, - lifecycle_hook_info_app_update, - fluxcd_process_required=False) + self._perform_app_recover(to_rpc_app, from_app, to_app, + lifecycle_hook_info_app_update, + fluxcd_process_required=False) + return False self.load_application_metadata_from_file(to_rpc_app) - # Check whether the new application is compatible with the current k8s version - self._utils._check_app_compatibility(to_app.name, to_app.version) + # Check whether the new application is compatible with the given k8s version. + # If k8s_version is none the check is performed against the active version. + self._utils._check_app_compatibility(to_app.name, + to_app.version, + k8s_version) self._update_app_status(to_app, constants.APP_UPDATE_IN_PROGRESS) @@ -2853,8 +2859,9 @@ class AppOperator(object): if do_recovery: LOG.error("Application %s update from version %s to version " "%s aborted." % (to_app.name, from_app.version, to_app.version)) - return self._perform_app_recover(to_rpc_app, from_app, to_app, - lifecycle_hook_info_app_update) + self._perform_app_recover(to_rpc_app, from_app, to_app, + lifecycle_hook_info_app_update) + return False self._update_app_status(to_app, constants.APP_UPDATE_IN_PROGRESS, "cleanup application version {}".format(from_app.version)) @@ -2928,9 +2935,10 @@ class AppOperator(object): # ie.images download/k8s resource creation failure # Start recovering without trigger fluxcd process LOG.exception(e) - return self._perform_app_recover(to_rpc_app, from_app, to_app, - lifecycle_hook_info_app_update, - fluxcd_process_required=False) + self._perform_app_recover(to_rpc_app, from_app, to_app, + lifecycle_hook_info_app_update, + fluxcd_process_required=False) + return False except Exception as e: # Application update successfully(fluxcd apply/rollback) # Error occurs during cleanup old app diff --git a/sysinv/sysinv/sysinv/sysinv/conductor/manager.py b/sysinv/sysinv/sysinv/sysinv/conductor/manager.py index 000ec0541d..ea001d2902 100644 --- a/sysinv/sysinv/sysinv/sysinv/conductor/manager.py +++ b/sysinv/sysinv/sysinv/sysinv/conductor/manager.py @@ -64,6 +64,7 @@ from controllerconfig.upgrades import management as upgrades_management from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric import rsa +from eventlet import greenpool from eventlet import greenthread # Make subprocess module greenthread friendly from eventlet.green import subprocess @@ -275,13 +276,11 @@ class KubeAppBundleDatabase(KubeAppBundleStorageFactory): """Check if the table is empty.""" return self.dbapi.kube_app_bundle_is_empty() - def get_all(self): + def get_all(self, name=None, k8s_auto_update=None, k8s_timing=None): """Get a list containing all bundles.""" - return self.dbapi.kube_app_bundle_get_all() - - def get_by_name(self, app_name): - """Get a list of bundles by their name.""" - return self.dbapi.kube_app_bundle_get_by_name(app_name) + return self.dbapi.kube_app_bundle_get_all(name=name, + k8s_auto_update=k8s_auto_update, + k8s_timing=k8s_timing) def destroy_all(self): """Prune all bundle metadata.""" @@ -7425,18 +7424,117 @@ class ConductorManager(service.PeriodicService): self._inner_sync_auto_apply(context, app_name) - def _get_app_bundle_for_update(self, app): + def update_apps_based_on_k8s_version_sync(self, context, k8s_version, k8s_upgrade_timing): + """ Update applications based on a given Kubernetes version (blocking). + + :param context: Context of the request + :param k8s_version: Kubernetes target version. + :param k8s_upgrade_timing: When applications should be updated. + :return: True if all apps were successfully updated. + False if any apps failed to be updated. + """ + + LOG.info("Checking available application updates for Kubernetes version {}." + .format(k8s_version)) + + update_candidates = [app_name for app_name in + self.apps_metadata[constants.APP_METADATA_APPS].keys()] + + # Launch a thread for each update candidate, then wait for all applications + # to finish updating. + threadpool = greenpool.GreenPool(len(update_candidates)) + threads = {} + result = True + for app_name in update_candidates: + + try: + app = kubeapp_obj.get_by_name(context, app_name) + except exception.KubeAppNotFound: + continue + + # Apps should be either in 'applied' or 'apply-failure' state. + # Applied apps are selected to be updated since they are currently in use. + # If the app is in 'apply-failure' state we give it a chance to be + # successfully applied via the update process. + if (app.status == constants.APP_APPLY_SUCCESS or + app.status == constants.APP_APPLY_FAILURE): + threads[app.name] = threadpool.spawn(self._auto_update_app, + context, + app_name, + k8s_version, + k8s_upgrade_timing, + async_update=False) + + # Wait for all updates to finish + threadpool.waitall() + + # Check result values + for app_name, thread in threads.items(): + if thread.wait() is False: + LOG.error("Failed to update {} to match target Kubernetes version {}" + .format(app_name, k8s_version)) + result = False + + return result + + def update_apps_based_on_k8s_version_async(self, context, k8s_version, k8s_upgrade_timing): + """ Update applications based on a given Kubernetes version (non-blocking). + + :param context: Context of the request + :param k8s_version: Kubernetes target version. + :param k8s_upgrade_timing: When applications should be updated. + """ + update_candidates = [app_name for app_name in + self.apps_metadata[constants.APP_METADATA_APPS].keys()] + + LOG.info("Checking available application updates for Kubernetes version {}." + .format(k8s_version)) + + for app_name in update_candidates: + try: + app = kubeapp_obj.get_by_name(context, app_name) + except exception.KubeAppNotFound: + continue + + # Apps should be either in 'applied' or 'apply-failure' state. + # Applied apps are selected to be updated since they are currently in use. + # If the app is in 'apply-failure' state we give it a chance to be + # successfully applied via the update process. + if (app.status == constants.APP_APPLY_SUCCESS or + app.status == constants.APP_APPLY_FAILURE): + if self._auto_update_app(context, + app_name, + k8s_version, + k8s_upgrade_timing) is False: + LOG.error("Failed to update {} to match Kubernetes version {}" + .format(app_name, k8s_version)) + + def _get_app_bundle_for_update(self, app, k8s_version=None, k8s_upgrade_timing=None): """ Retrieve metadata from the most updated application bundle that can be used to update the given app. :param app: The application to be updated + :param k8s_version: Target Kubernetes version + :param k8s_upgrade_timing: When applications should be updated during Kubernetes upgrades :return The bundle metadata from the new version of the app """ - bundle_metadata_list = self._kube_app_bundle_storage.get_by_name(app.name) + if k8s_upgrade_timing is None: + bundle_metadata_list = self._kube_app_bundle_storage.get_all(app.name) + else: + # Filter bundle list by the application name, k8s_auto_update = True and + # the given k8s_upgrade_timing. + bundle_metadata_list = self._kube_app_bundle_storage.get_all(app.name, + True, + k8s_upgrade_timing) latest_version_bundle = None - k8s_version = self._kube.kube_get_kubernetes_version().strip().lstrip('v') + + if k8s_version is None: + k8s_version = self._kube.kube_get_kubernetes_version().strip().lstrip('v') + else: + k8s_version = k8s_version.strip().lstrip('v') + for bundle_metadata in bundle_metadata_list: if LooseVersion(bundle_metadata.version) <= LooseVersion(app.app_version): LOG.debug("Bundle {} version {} lower than installed app version ({})" @@ -7446,9 +7544,6 @@ class ConductorManager(service.PeriodicService): elif not bundle_metadata.auto_update: LOG.debug("Application auto update disabled for bundle {}" .format(bundle_metadata.file_path)) - elif not bundle_metadata.k8s_auto_update: - LOG.debug("Kubernetes application auto update disabled for bundle {}" - .format(bundle_metadata.file_path)) elif LooseVersion(k8s_version) < LooseVersion(bundle_metadata.k8s_minimum_version): LOG.debug("Kubernetes version {} is lower than {} which is " "the minimum required for bundle {}" @@ -7471,17 +7566,33 @@ class ConductorManager(service.PeriodicService): return latest_version_bundle - def _auto_update_app(self, context, app_name): - """Auto update applications""" + def _auto_update_app(self, + context, + app_name, + k8s_version=None, + k8s_upgrade_timing=None, + async_update=True): + """Auto update applications + + :param context: Context of the request. + :param app_name: Name of the application to be updated. + :param k8s_version: Kubernetes target version. + :param k8s_upgrade_timing: When applications should be updated. + :param async_update: Update asynchronously if True. Update synchronously if False. + :return: True if the update successfully started when running asynchronously. + True if the app was successfully updated when running synchronously. + False if an error has occurred. + None if there is not an updated version available for the given app. + """ try: app = kubeapp_obj.get_by_name(context, app_name) except exception.KubeAppNotFound as e: LOG.exception(e) - return + return False if app.status != constants.APP_APPLY_SUCCESS: # In case the previous re-apply fails - return + return False try: hook_info = LifecycleHookInfo() @@ -7493,28 +7604,27 @@ class ConductorManager(service.PeriodicService): self.app_lifecycle_actions(context, app, hook_info) except exception.LifecycleSemanticCheckException as e: LOG.info("Auto-update failed prerequisites for {}: {}".format(app.name, e)) - return + return False except exception.LifecycleSemanticCheckOperationNotSupported as e: LOG.debug(e) - return + return False except exception.SysinvException: LOG.exception("Internal sysinv error while checking automatic " "updates for {}" .format(app.name)) - return + return False except Exception as e: LOG.exception("Automatic operation:{} " "for app {} failed with: {}".format(hook_info, app.name, e)) - return + return False if self._patching_operation_is_occurring(): - return - + return False LOG.debug("Application %s: Checking " "for update ..." % app_name) - app_bundle = self._get_app_bundle_for_update(app) + app_bundle = self._get_app_bundle_for_update(app, k8s_version, k8s_upgrade_timing) if app_bundle is None: # Skip if no bundles are found LOG.debug("No bundle found for updating %s" % app_name) @@ -7529,23 +7639,30 @@ class ConductorManager(service.PeriodicService): (tarball.manifest_name is None) or (tarball.manifest_file is None)): # Skip if tarball check fails - return + return False if app_bundle.version in \ app.app_metadata.get( constants.APP_METADATA_UPGRADES, {}).get( - constants.APP_METADATA_FAILED_VERSIONS, []): + constants.APP_METADATA_FAILED_VERSIONS, []) and \ + k8s_version is None: # Skip if this version was previously failed to - # be updated + # be updated. Allow retrying only if a Kubernetes version is + # defined, meaning that Kubernetes upgrade is in progress. LOG.error("Application %s with version %s was previously " "failed to be updated from version %s by auto-update" % (app.name, tarball.app_version, app.app_version)) - return + return False - self._inner_sync_auto_update(context, app, tarball) + return self._inner_sync_auto_update(context, app, tarball, k8s_version, async_update) @cutils.synchronized(LOCK_APP_AUTO_MANAGE) - def _inner_sync_auto_update(self, context, applied_app, tarball): + def _inner_sync_auto_update(self, + context, + applied_app, + tarball, + k8s_version=None, + async_update=True): # Check no other app is in progress of apply/update/recovery for other_app in self.dbapi.kube_app_get_all(): if other_app.status in [constants.APP_APPLY_IN_PROGRESS, @@ -7555,7 +7672,7 @@ class ConductorManager(service.PeriodicService): "is in progress of apply/update/recovery. " "Will retry on next audit", applied_app.name, other_app.name) - return + return False # Set the status for the current applied app to inactive applied_app.status = constants.APP_INACTIVE_STATE @@ -7590,14 +7707,36 @@ class ConductorManager(service.PeriodicService): applied_app.progress = constants.APP_PROGRESS_COMPLETED applied_app.save() LOG.exception(e) - return + return False LOG.info("Platform managed application %s: " "Auto updating..." % target_app.name) hook_info = LifecycleHookInfo() hook_info.mode = constants.APP_LIFECYCLE_MODE_AUTO - greenthread.spawn(self.perform_app_update, context, applied_app, - target_app, tarball.tarball_name, operation, hook_info) + + if async_update: + greenthread.spawn(self.perform_app_update, + context, + applied_app, + target_app, + tarball.tarball_name, + operation, + hook_info, + None, + None, + k8s_version) + else: + return self.perform_app_update(context, + applied_app, + target_app, + tarball.tarball_name, + operation, + hook_info, + None, + None, + k8s_version) + + return True def _search_tarfile(self, app_name, managed_app): """Search a specified application tarfile from the directory @@ -16044,7 +16183,7 @@ class ConductorManager(service.PeriodicService): def perform_app_update(self, context, from_rpc_app, to_rpc_app, tarfile, operation, lifecycle_hook_info_app_update, reuse_user_overrides=None, - reuse_attributes=None): + reuse_attributes=None, k8s_version=None): """Handling of application update request (via AppOperator) :param context: request context. @@ -16061,9 +16200,14 @@ class ConductorManager(service.PeriodicService): """ lifecycle_hook_info_app_update.operation = constants.APP_UPDATE_OP - self._app.perform_app_update(from_rpc_app, to_rpc_app, tarfile, - operation, lifecycle_hook_info_app_update, reuse_user_overrides, - reuse_attributes) + return self._app.perform_app_update(from_rpc_app, + to_rpc_app, + tarfile, + operation, + lifecycle_hook_info_app_update, + reuse_user_overrides, + reuse_attributes, + k8s_version) def perform_app_remove(self, context, rpc_app, lifecycle_hook_info_app_remove, force=False): """Handling of application removal request (via AppOperator) @@ -16332,6 +16476,60 @@ class ConductorManager(service.PeriodicService): LOG.info("Successfully completed k8s control plane backup.") + def _check_installed_apps_compatibility(self, kube_version): + """Checks whether all installed applications are compatible + with the new k8s version + + :param kube_version: Target Kubernetes version + :return: True if all apps are compatible with the given Kubernetes version + False if any apps are incompatible with the given Kubernetes version + """ + + # Check that all installed applications support new k8s version + apps = self.dbapi.kube_app_get_all() + + success = True + for app in apps: + if app.status != constants.APP_APPLY_SUCCESS: + continue + + kube_min_version, kube_max_version = \ + cutils.get_app_supported_kube_version(app.name, app.app_version) + + if not kubernetes.is_kube_version_supported( + kube_version, kube_min_version, kube_max_version): + LOG.error("The installed Application {} ({}) is incompatible with the " + "new Kubernetes version {}.".format(app.name, + app.app_version, + kube_version)) + success = False + + return success + + def kube_upgrade_start(self, context, k8s_version): + """ Start a Kubernetes upgrade by updating all required apps. + + :param context: Context of the request. + :param k8s_version: Kubernetes target version. + :param k8s_upgrade_timing: When apps should be updated. + """ + + kube_upgrade_obj = objects.kube_upgrade.get_one(context) + + if (self.update_apps_based_on_k8s_version_sync(context, + k8s_version, + constants.APP_METADATA_TIMING_PRE) and + self._check_installed_apps_compatibility(k8s_version)): + kube_upgrade_obj.state = kubernetes.KUBE_UPGRADE_STARTED + LOG.info("Started kubernetes upgrade from version: %s to version: %s" + % (kube_upgrade_obj.from_version, kube_upgrade_obj.to_version)) + else: + kube_upgrade_obj.state = kubernetes.KUBE_UPGRADE_STARTING_FAILED + LOG.info("Failed to start kubernetes upgrade from version: %s to version: %s" + % (kube_upgrade_obj.from_version, kube_upgrade_obj.to_version)) + + kube_upgrade_obj.save() + def kube_download_images(self, context, kube_version): """Download the kubernetes images for this version""" diff --git a/sysinv/sysinv/sysinv/sysinv/conductor/rpcapi.py b/sysinv/sysinv/sysinv/sysinv/conductor/rpcapi.py index d78989fc3f..9c737d3a5a 100644 --- a/sysinv/sysinv/sysinv/sysinv/conductor/rpcapi.py +++ b/sysinv/sysinv/sysinv/sysinv/conductor/rpcapi.py @@ -1716,6 +1716,32 @@ class ConductorAPI(sysinv.openstack.common.rpc.proxy.RpcProxy): return self.call(context, self.make_msg('get_fernet_keys', key_id=key_id)) + def kube_upgrade_start(self, context, k8s_version): + """Asynchronously, start a Kubernetes upgrade to the given version. + + :param context: Context of the request + :param k8s_version: Kubernetes target version + :param k8s_upgrade_timing: When applications should be updated + """ + + return self.cast(context, self.make_msg('kube_upgrade_start', + k8s_version=k8s_version)) + + def update_apps_based_on_k8s_version_async(self, + context, + k8s_version, + k8s_upgrade_timing): + """Asynchronously, update all applications based on a given Kubernetes version. + + :param context: Context of the request + :param k8s_version: Kubernetes target version + :param k8s_upgrade_timing: When applications should be updated + """ + + return self.cast(context, self.make_msg('update_apps_based_on_k8s_version_async', + k8s_version=k8s_version, + k8s_upgrade_timing=k8s_upgrade_timing)) + def evaluate_apps_reapply(self, context, trigger): """Synchronously, determine whether an application re-apply is needed, and if so, raise the re-apply flag. diff --git a/sysinv/sysinv/sysinv/sysinv/db/api.py b/sysinv/sysinv/sysinv/sysinv/db/api.py index 59f0b3d3e3..e7e7d8c4da 100644 --- a/sysinv/sysinv/sysinv/sysinv/db/api.py +++ b/sysinv/sysinv/sysinv/sysinv/db/api.py @@ -5122,6 +5122,8 @@ class Connection(object): @abc.abstractmethod def kube_app_bundle_get_all(self, name=None, + k8s_auto_update=None, + timing=None, limit=None, marker=None, sort_key=None, @@ -5130,25 +5132,11 @@ class Connection(object): given filter. :param name: Application name. - :param limit: Maximum number of entries to return. - :param marker: The last item of the previous page; we return the next - result set. - :param sort_key: Attribute by which results should be sorted. - :param sort_dir: Direction in which results should be sorted. - (asc, desc) - :returns: A list of kube_app_bundle entries with the given name. - """ - - @abc.abstractmethod - def kube_app_bundle_get_by_name(self, - name, - limit=None, - marker=None, - sort_key=None, - sort_dir=None): - """Get kube_app_bundle entries that match a given name - - :param name: Application name. + :param k8s_auto_update: Whether automatically updating the application + is enabled when upgrading Kubernetes. + :param timing: Application update timing during Kubernetes upgrade + "pre": during kube-upgrade-start. + "post": during kube-upgrade-complete. :param limit: Maximum number of entries to return. :param marker: The last item of the previous page; we return the next result set. diff --git a/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/api.py b/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/api.py index ffa41d3f26..5c6099d322 100644 --- a/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/api.py +++ b/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/api.py @@ -9455,24 +9455,20 @@ class Connection(api.Connection): return result is None @db_objects.objectify(objects.kube_app_bundle) - def kube_app_bundle_get_all(self, name=None, - limit=None, marker=None, + def kube_app_bundle_get_all(self, name=None, k8s_auto_update=None, + k8s_timing=None, limit=None, marker=None, sort_key=None, sort_dir=None): query = model_query(models.KubeAppBundle) if name: query = query.filter_by(name=name) + if k8s_auto_update: + query = query.filter_by(k8s_auto_update=k8s_auto_update) + if k8s_timing: + query = query.filter_by(k8s_timing=k8s_timing) return _paginate_query(models.KubeAppBundle, limit, marker, sort_key, sort_dir, query) - @db_objects.objectify(objects.kube_app_bundle) - def kube_app_bundle_get_by_name(self, name, - limit=None, marker=None, - sort_key=None, sort_dir=None): - - return self.kube_app_bundle_get_all(name, limit, marker, - sort_key, sort_dir) - def kube_app_bundle_destroy_all(self, file_path=None): with _session_for_write() as session: query = model_query(models.KubeAppBundle, session=session) diff --git a/sysinv/sysinv/sysinv/sysinv/tests/api/test_kube_upgrade.py b/sysinv/sysinv/sysinv/sysinv/tests/api/test_kube_upgrade.py index 4cebdec48a..16cb48555c 100644 --- a/sysinv/sysinv/sysinv/sysinv/tests/api/test_kube_upgrade.py +++ b/sysinv/sysinv/sysinv/sysinv/tests/api/test_kube_upgrade.py @@ -72,9 +72,11 @@ class FakeFmClient(object): class FakeConductorAPI(object): def __init__(self): + self.kube_upgrade_start = mock.MagicMock() self.kube_download_images = mock.MagicMock() self.kube_upgrade_networking = mock.MagicMock() self.kube_upgrade_abort = mock.MagicMock() + self.update_apps_based_on_k8s_version_async = mock.MagicMock() self.evaluate_apps_reapply = mock.MagicMock() self.remove_kube_control_plane_backup = mock.MagicMock() self.kube_delete_container_images = mock.MagicMock() @@ -170,22 +172,6 @@ class TestKubeUpgrade(base.FunctionalTest): self.mocked_kube_get_version_states.start() self.addCleanup(self.mocked_kube_get_version_states.stop) - # Mock utility function - self.kube_min_version_result, self.kube_max_version_result = 'v1.42.1', 'v1.43.1' - - def mock_get_app_supported_kube_version(app_name, app_version): - return self.kube_min_version_result, self.kube_max_version_result - self.mocked_kube_min_version = mock.patch( - 'sysinv.common.utils.get_app_supported_kube_version', - mock_get_app_supported_kube_version) - self.mocked_kube_max_version = mock.patch( - 'sysinv.common.utils.get_app_supported_kube_version', - mock_get_app_supported_kube_version) - self.mocked_kube_min_version.start() - self.mocked_kube_max_version.start() - self.addCleanup(self.mocked_kube_min_version.stop) - self.addCleanup(self.mocked_kube_max_version.stop) - self.setup_health_mocked_calls() def setup_health_mocked_calls(self): @@ -288,11 +274,15 @@ class TestPostKubeUpgradeSimplex(TestKubeUpgrade, result = self.post_json('/kube_upgrade', create_dict, headers={'User-Agent': 'sysinv-test'}) + # Verify that the upgrade was started + self.fake_conductor_api.kube_upgrade_start.\ + assert_called_with(mock.ANY, 'v1.43.3') + # Verify that the upgrade has the expected attributes self.assertEqual(result.json['from_version'], 'v1.42.1') self.assertEqual(result.json['to_version'], 'v1.43.3') self.assertEqual(result.json['state'], - kubernetes.KUBE_UPGRADE_STARTED) + kubernetes.KUBE_UPGRADE_STARTING) # see if kubeadm_version was changed in DB kube_cmd_version = self.dbapi.kube_cmd_version_get() @@ -390,11 +380,15 @@ class TestPostKubeUpgrade(TestKubeUpgrade, result = self.post_json('/kube_upgrade', create_dict, headers={'User-Agent': 'sysinv-test'}) + # Verify that the upgrade was started + self.fake_conductor_api.kube_upgrade_start.\ + assert_called_with(mock.ANY, 'v1.43.2') + # Verify that the upgrade has the expected attributes self.assertEqual(result.json['from_version'], 'v1.43.1') self.assertEqual(result.json['to_version'], 'v1.43.2') self.assertEqual(result.json['state'], - kubernetes.KUBE_UPGRADE_STARTED) + kubernetes.KUBE_UPGRADE_STARTING) # see if kubeadm_version was changed in DB kube_cmd_version = self.dbapi.kube_cmd_version_get() @@ -485,30 +479,6 @@ class TestPostKubeUpgrade(TestKubeUpgrade, self.assertIn("version v1.43.1 is not active", result.json['error_message']) - def test_create_installed_app_not_compatible(self): - # Test creation of upgrade when the installed application isn't - # compatible with the new kubernetes version - - # Create application - dbutils.create_test_app( - name='stx-openstack', - app_version='1.0-19', - manifest_name='manifest', - manifest_file='stx-openstack.yaml', - status='applied', - active=True) - - create_dict = dbutils.post_get_test_kube_upgrade(to_version='v1.43.2') - result = self.post_json('/kube_upgrade', create_dict, - headers={'User-Agent': 'sysinv-test'}, - expect_errors=True) - - # Verify the failure - self.assertEqual(result.content_type, 'application/json') - self.assertEqual(http_client.BAD_REQUEST, result.status_int) - self.assertIn("incompatible with the new Kubernetes version v1.43.2", - result.json['error_message']) - @mock.patch('sysinv.common.health.Health._check_trident_compatibility', lambda x: True) def test_create_system_unhealthy_from_alarms(self): """Test creation of a kube upgrade while there are alarms""" @@ -542,11 +512,15 @@ class TestPostKubeUpgrade(TestKubeUpgrade, result = self.post_json('/kube_upgrade', create_dict, headers={'User-Agent': 'sysinv-test'}) + # Verify that the upgrade was started + self.fake_conductor_api.kube_upgrade_start.\ + assert_called_with(mock.ANY, 'v1.43.2') + # Verify that the upgrade has the expected attributes self.assertEqual(result.json['from_version'], 'v1.43.1') self.assertEqual(result.json['to_version'], 'v1.43.2') self.assertEqual(result.json['state'], - kubernetes.KUBE_UPGRADE_STARTED) + kubernetes.KUBE_UPGRADE_STARTING) @mock.patch('sysinv.common.health.Health._check_trident_compatibility', lambda x: True) def test_force_create_system_unhealthy_from_mgmt_affecting_alarms(self): @@ -583,11 +557,15 @@ class TestPostKubeUpgrade(TestKubeUpgrade, result = self.post_json('/kube_upgrade', create_dict, headers={'User-Agent': 'sysinv-test'}) + # Verify that the upgrade was started + self.fake_conductor_api.kube_upgrade_start.\ + assert_called_with(mock.ANY, 'v1.43.2') + # Verify that the upgrade has the expected attributes self.assertEqual(result.json['from_version'], 'v1.43.1') self.assertEqual(result.json['to_version'], 'v1.43.2') self.assertEqual(result.json['state'], - kubernetes.KUBE_UPGRADE_STARTED) + kubernetes.KUBE_UPGRADE_STARTING) @mock.patch('sysinv.common.health.Health._check_trident_compatibility', lambda x: True) def test_create_system_unhealthy_from_bad_apps(self): @@ -629,11 +607,15 @@ class TestPostKubeUpgrade(TestKubeUpgrade, result = self.post_json('/kube_upgrade', create_dict, headers={'User-Agent': 'sysinv-test'}) + # Verify that the upgrade was started + self.fake_conductor_api.kube_upgrade_start.\ + assert_called_with(mock.ANY, 'v1.43.3') + # Verify that the upgrade has the expected attributes self.assertEqual(result.json['from_version'], 'v1.43.2') self.assertEqual(result.json['to_version'], 'v1.43.3') self.assertEqual(result.json['state'], - kubernetes.KUBE_UPGRADE_STARTED) + kubernetes.KUBE_UPGRADE_STARTING) def test_create_applied_patch_missing(self): # Test creation of upgrade when applied patch is missing diff --git a/sysinv/sysinv/sysinv/sysinv/tests/conductor/test_manager.py b/sysinv/sysinv/sysinv/sysinv/tests/conductor/test_manager.py index 470ff000b5..70a09caaa9 100644 --- a/sysinv/sysinv/sysinv/sysinv/tests/conductor/test_manager.py +++ b/sysinv/sysinv/sysinv/sysinv/tests/conductor/test_manager.py @@ -49,6 +49,7 @@ from sysinv.db import api as dbapi from sysinv.loads.loads import LoadImport from sysinv.objects.load import Load from sysinv.puppet import common as puppet_common +from sysinv.tests.db import utils as dbutils from sysinv import objects from sysinv.tests.db import base @@ -497,6 +498,22 @@ class ManagerTestCase(base.DbTestCase): self.alarm_raised = False self.kernel_alarms = {} + # Mock utility function + self.kube_min_version_result, self.kube_max_version_result = 'v1.42.1', 'v1.43.1' + + def mock_get_app_supported_kube_version(app_name, app_version): + return self.kube_min_version_result, self.kube_max_version_result + self.mocked_kube_min_version = mock.patch( + 'sysinv.common.utils.get_app_supported_kube_version', + mock_get_app_supported_kube_version) + self.mocked_kube_max_version = mock.patch( + 'sysinv.common.utils.get_app_supported_kube_version', + mock_get_app_supported_kube_version) + self.mocked_kube_min_version.start() + self.mocked_kube_max_version.start() + self.addCleanup(self.mocked_kube_min_version.stop) + self.addCleanup(self.mocked_kube_max_version.stop) + def tearDown(self): super(ManagerTestCase, self).tearDown() @@ -931,6 +948,63 @@ class ManagerTestCase(base.DbTestCase): ret = self.service.platform_interfaces(self.context, ihost['id'] + 1) self.assertEqual(ret, []) + def test_kube_upgrade_start(self): + # Create application + dbutils.create_test_app( + name='stx-openstack', + app_version='1.0-19', + manifest_name='manifest', + manifest_file='stx-openstack.yaml', + status='applied', + active=True) + + # Create an upgrade + from_version = 'v1.42.1' + to_version = 'v1.43.1' + utils.create_test_kube_upgrade( + from_version=from_version, + to_version=to_version, + state=kubernetes.KUBE_UPGRADE_STARTING, + ) + + # Start upgrade + self.service.kube_upgrade_start(self.context, to_version) + + # Verify that the upgrade state was updated + updated_upgrade = self.dbapi.kube_upgrade_get_one() + self.assertEqual(updated_upgrade.state, + kubernetes.KUBE_UPGRADE_STARTED) + + def test_kube_upgrade_start_app_not_compatible(self): + # Test creation of upgrade when the installed application isn't + # compatible with the new kubernetes version + + # Create application + dbutils.create_test_app( + name='stx-openstack', + app_version='1.0-19', + manifest_name='manifest', + manifest_file='stx-openstack.yaml', + status='applied', + active=True) + + # Create an upgrade + from_version = 'v1.42.1' + to_version = 'v1.43.2' + utils.create_test_kube_upgrade( + from_version=from_version, + to_version=to_version, + state=kubernetes.KUBE_UPGRADE_STARTING, + ) + + # Start upgrade + self.service.kube_upgrade_start(self.context, to_version) + + # Verify that the upgrade state was updated + updated_upgrade = self.dbapi.kube_upgrade_get_one() + self.assertEqual(updated_upgrade.state, + kubernetes.KUBE_UPGRADE_STARTING_FAILED) + def test_kube_download_images(self): # Create controller-0 config_uuid = str(uuid.uuid4())