From 7bd361b69b7f6c14c777a68c923b242eaa3b4148 Mon Sep 17 00:00:00 2001 From: Igor Soares Date: Thu, 15 Feb 2024 19:39:07 -0300 Subject: [PATCH] Enhance app updates during Kubernetes upgrades Add the folowing enhancements to application updates during Kubernetes upgrades: 1) Move the pre application update logic from kube-upgrade-start step to a specific separated step called via a new command line option named kube-pre-application-update, which can be triggered after the download images step and before upgrading networking. 2) Move the post application update logic from kube-upgrade-complete step to a specific separated step called via a new command line option named kube-post-application-update, which can be triggered after the kube-upgrade-complete stage and before the upgrade is deleted. 3) Introduce validation logic to kube-upgrade-start step to check if all applied apps have available versions compatible with intermediate and target Kubernetes versions. Upgrades are blocked if apps marked to be pre updated are incompatible with current and target Kubernetes versions. Upgrades are also blocked if apps marked to be post updated are incompatible with the target Kubernetes version. 4) Delete uploaded applications incompatible with the target Kubernetes version and upload one that is compatible if available. 5) Restore kube-upgrade-start and kube-upgrade-complete to their original logic before application updates during Kubernetes upgrades was implemented on task 49416. The kube-upgrade-start step is synchronous as it used to be before that change. 6) Update sysinv and cgts-client unit tests to account for the new Kubernetes upgrade steps. 7) Create a helper function called "patch_kube_upgrade" to improve code reuse when creating patch requests for new shell commands related to Kubernetes upgrades. Test Plan: AIO-SX Test Cases: PASS: Fresh install. PASS: Successful Kubernetes single version upgrade with no apps that need to be updated. PASS: Successful Kubernetes multi-version upgrade with no apps that need to be updated. PASS: Successful Kubernetes upgrade with apps that need to be updated before and after the new version is deployed. PASS: Check if the upgrade is blocked if an app is incompatible with a Kubernetes intermediate version during a multi-version upgrade. PASS: Check if the upgrade is blocked if an app marked to be pre updated is incompatible with the Kubernetes target version. PASS: Check if the upgrade is blocked if an app marked to be post updated is incompatible with the Kubernetes target version. PASS: Check if uploaded apps have been replaced by compatible versions. PASS: Check if uploaded apps that do not have compatible versions were removed. PASS: Failure to run kube-pre-application-update and successful retry. PASS: Failure to run kube-post-application-update and successful retry. PASS: Abort during kube-pre-application-update and start over. PASS: Reject aborting Kubernetes upgrade after post-updated-apps state. AIO-DX Test Cases: PASS: Fresh install. PASS Successful Kubernetes upgrade with no apps that need to be updated. PASS: Successful Kubernetes upgrade with apps that need to be updated before and after the new version is deployed. PASS: Check if the upgrade is blocked if an app marked to be pre updated is incompatible with the Kubernetes target version. PASS: Check if the upgrade is blocked if an app marked to be post updated is incompatible with the Kubernetes target version. Story: 2010929 Task: 49595 Change-Id: I9b48567c39c9a12b7563d56ab90fbfe9dd7082aa Signed-off-by: Igor Soares --- .../tests/v1/test_kube_upgrade_shell.py | 54 +++ .../cgtsclient/v1/kube_upgrade_shell.py | 94 ++---- .../sysinv/api/controllers/v1/kube_upgrade.py | 301 ++++++++++++----- .../sysinv/sysinv/sysinv/common/kubernetes.py | 8 +- .../sysinv/sysinv/sysinv/conductor/manager.py | 309 ++++++++++++------ .../sysinv/sysinv/sysinv/conductor/rpcapi.py | 34 +- sysinv/sysinv/sysinv/sysinv/db/api.py | 35 +- .../sysinv/sysinv/sysinv/db/sqlalchemy/api.py | 47 +++ .../sysinv/tests/api/test_kube_upgrade.py | 222 +++++++++++-- .../sysinv/tests/conductor/test_manager.py | 79 ++++- 10 files changed, 874 insertions(+), 309 deletions(-) diff --git a/sysinv/cgts-client/cgts-client/cgtsclient/tests/v1/test_kube_upgrade_shell.py b/sysinv/cgts-client/cgts-client/cgtsclient/tests/v1/test_kube_upgrade_shell.py index b22f105047..41506186e3 100644 --- a/sysinv/cgts-client/cgts-client/cgtsclient/tests/v1/test_kube_upgrade_shell.py +++ b/sysinv/cgts-client/cgts-client/cgtsclient/tests/v1/test_kube_upgrade_shell.py @@ -85,6 +85,33 @@ class KubeUpgradeTest(test_shell.ShellTest): self.assertIn(fake_kube_upgrade['created_at'], results) self.assertIn(fake_kube_upgrade['updated_at'], results) + @mock.patch('cgtsclient.v1.kube_upgrade.KubeUpgradeManager.update') + def test_kube_pre_application_update(self, mock_update): + fake_kube_upgrade = {'from_version': 'v1.42.1', + 'to_version': 'v1.42.2', + 'state': 'pre-updating-apps', + 'uuid': 'cb737aba-1820-4184-b0dc-9b073822af48', + 'created_at': 'fake-created-time', + 'updated_at': 'fake-updated-time', + } + mock_update.return_value = KubeUpgrade(None, fake_kube_upgrade, True) + + self.make_env() + results = self.shell("kube-pre-application-update") + + patch = {'op': 'replace', + 'path': '/state', + 'value': 'pre-updating-apps' + } + mock_update.assert_called_once_with([patch]) + + self.assertIn(fake_kube_upgrade['from_version'], results) + self.assertIn(fake_kube_upgrade['to_version'], results) + self.assertIn(fake_kube_upgrade['state'], results) + self.assertIn(fake_kube_upgrade['uuid'], results) + self.assertIn(fake_kube_upgrade['created_at'], results) + self.assertIn(fake_kube_upgrade['updated_at'], results) + @mock.patch('cgtsclient.v1.kube_upgrade.KubeUpgradeManager.update') def test_kube_upgrade_download_images(self, mock_update): fake_kube_upgrade = {'from_version': 'v1.42.1', @@ -125,6 +152,33 @@ class KubeUpgradeTest(test_shell.ShellTest): self.assertIn(fake_kube_upgrade['created_at'], results) self.assertIn(fake_kube_upgrade['updated_at'], results) + @mock.patch('cgtsclient.v1.kube_upgrade.KubeUpgradeManager.update') + def test_kube_post_application_update(self, mock_update): + fake_kube_upgrade = {'from_version': 'v1.42.1', + 'to_version': 'v1.42.2', + 'state': 'post-updating-apps', + 'uuid': 'cb737aba-1820-4184-b0dc-9b073822af48', + 'created_at': 'fake-created-time', + 'updated_at': 'fake-updated-time', + } + mock_update.return_value = KubeUpgrade(None, fake_kube_upgrade, True) + + self.make_env() + results = self.shell("kube-post-application-update") + + patch = {'op': 'replace', + 'path': '/state', + 'value': 'post-updating-apps' + } + mock_update.assert_called_once_with([patch]) + + self.assertIn(fake_kube_upgrade['from_version'], results) + self.assertIn(fake_kube_upgrade['to_version'], results) + self.assertIn(fake_kube_upgrade['state'], results) + self.assertIn(fake_kube_upgrade['uuid'], results) + self.assertIn(fake_kube_upgrade['created_at'], results) + self.assertIn(fake_kube_upgrade['updated_at'], results) + @mock.patch('cgtsclient.v1.kube_upgrade.KubeUpgradeManager.update') def test_kube_upgrade_complete(self, mock_update): fake_kube_upgrade = {'from_version': 'v1.42.1', diff --git a/sysinv/cgts-client/cgts-client/cgtsclient/v1/kube_upgrade_shell.py b/sysinv/cgts-client/cgts-client/cgtsclient/v1/kube_upgrade_shell.py index 8aacd884ee..b40fc5b330 100755 --- a/sysinv/cgts-client/cgts-client/cgtsclient/v1/kube_upgrade_shell.py +++ b/sysinv/cgts-client/cgts-client/cgtsclient/v1/kube_upgrade_shell.py @@ -9,6 +9,7 @@ from cgtsclient import exc from cgtsclient.v1 import ihost as ihost_utils # Kubernetes constants +KUBE_UPGRADE_STATE_PRE_UPDATING_APPS = 'pre-updating-apps' KUBE_UPGRADE_STATE_DOWNLOADING_IMAGES = 'downloading-images' KUBE_UPGRADE_STATE_UPGRADING_NETWORKING = 'upgrading-networking' KUBE_UPGRADE_STATE_UPGRADING_STORAGE = 'upgrading-storage' @@ -18,6 +19,7 @@ KUBE_UPGRADE_STATE_UPGRADING_SECOND_MASTER = 'upgrading-second-master' KUBE_UPGRADE_STATE_ABORTING = 'upgrade-aborting' KUBE_UPGRADE_STATE_CORDON = 'cordon-started' KUBE_UPGRADE_STATE_UNCORDON = 'uncordon-started' +KUBE_UPGRADE_STATE_POST_UPDATING_APPS = 'post-updating-apps' def _print_kube_upgrade_show(obj): @@ -56,11 +58,8 @@ def do_kube_upgrade_start(cc, args): _print_kube_upgrade_show(kube_upgrade) -def do_kube_upgrade_download_images(cc, args): - """Download kubernetes images.""" - - data = dict() - data['state'] = KUBE_UPGRADE_STATE_DOWNLOADING_IMAGES +def patch_kube_upgrade(cc, data): + """" Call patch HTTP method for kube upgrades""" patch = [] for (k, v) in data.items(): @@ -73,6 +72,24 @@ def do_kube_upgrade_download_images(cc, args): _print_kube_upgrade_show(kube_upgrade) +def do_kube_upgrade_download_images(cc, args): + """Download kubernetes images.""" + + data = dict() + data['state'] = KUBE_UPGRADE_STATE_DOWNLOADING_IMAGES + + patch_kube_upgrade(cc, data) + + +def do_kube_pre_application_update(cc, args): + """Update applications before Kubernetes is upgraded.""" + + data = dict() + data['state'] = KUBE_UPGRADE_STATE_PRE_UPDATING_APPS + + patch_kube_upgrade(cc, data) + + @utils.arg('hostid', metavar='', help="Name or ID of host") def do_kube_host_cordon(cc, args): @@ -83,15 +100,7 @@ def do_kube_host_cordon(cc, args): data['hostname'] = ihost.hostname data['state'] = KUBE_UPGRADE_STATE_CORDON - patch = [] - for (k, v) in data.items(): - patch.append({'op': 'replace', 'path': '/' + k, 'value': v}) - try: - kube_upgrade = cc.kube_upgrade.update(patch) - except exc.HTTPNotFound: - raise exc.CommandError('Kubernetes upgrade UUID not found') - - _print_kube_upgrade_show(kube_upgrade) + patch_kube_upgrade(cc, data) @utils.arg('hostid', metavar='', @@ -104,15 +113,7 @@ def do_kube_host_uncordon(cc, args): data['hostname'] = ihost.hostname data['state'] = KUBE_UPGRADE_STATE_UNCORDON - patch = [] - for (k, v) in data.items(): - patch.append({'op': 'replace', 'path': '/' + k, 'value': v}) - try: - kube_upgrade = cc.kube_upgrade.update(patch) - except exc.HTTPNotFound: - raise exc.CommandError('Kubernetes upgrade UUID not found') - - _print_kube_upgrade_show(kube_upgrade) + patch_kube_upgrade(cc, data) def do_kube_upgrade_networking(cc, args): @@ -121,15 +122,7 @@ def do_kube_upgrade_networking(cc, args): data = dict() data['state'] = KUBE_UPGRADE_STATE_UPGRADING_NETWORKING - patch = [] - for (k, v) in data.items(): - patch.append({'op': 'replace', 'path': '/' + k, 'value': v}) - try: - kube_upgrade = cc.kube_upgrade.update(patch) - except exc.HTTPNotFound: - raise exc.CommandError('Kubernetes upgrade UUID not found') - - _print_kube_upgrade_show(kube_upgrade) + patch_kube_upgrade(cc, data) def do_kube_upgrade_storage(cc, args): @@ -138,15 +131,16 @@ def do_kube_upgrade_storage(cc, args): data = dict() data['state'] = KUBE_UPGRADE_STATE_UPGRADING_STORAGE - patch = [] - for (k, v) in data.items(): - patch.append({'op': 'replace', 'path': '/' + k, 'value': v}) - try: - kube_upgrade = cc.kube_upgrade.update(patch) - except exc.HTTPNotFound: - raise exc.CommandError('Kubernetes upgrade UUID not found') + patch_kube_upgrade(cc, data) - _print_kube_upgrade_show(kube_upgrade) + +def do_kube_post_application_update(cc, args): + """Update applications after Kubernetes is upgraded.""" + + data = dict() + data['state'] = KUBE_UPGRADE_STATE_POST_UPDATING_APPS + + patch_kube_upgrade(cc, data) def do_kube_upgrade_abort(cc, args): @@ -155,15 +149,7 @@ def do_kube_upgrade_abort(cc, args): data = dict() data['state'] = KUBE_UPGRADE_STATE_ABORTING - patch = [] - for (k, v) in data.items(): - patch.append({'op': 'replace', 'path': '/' + k, 'value': v}) - try: - kube_upgrade = cc.kube_upgrade.update(patch) - except exc.HTTPNotFound: - raise exc.CommandError('Kubernetes upgrade not found') - - _print_kube_upgrade_show(kube_upgrade) + patch_kube_upgrade(cc, data) def do_kube_upgrade_complete(cc, args): @@ -172,15 +158,7 @@ def do_kube_upgrade_complete(cc, args): data = dict() data['state'] = KUBE_UPGRADE_STATE_COMPLETE - patch = [] - for (k, v) in data.items(): - patch.append({'op': 'replace', 'path': '/' + k, 'value': v}) - try: - kube_upgrade = cc.kube_upgrade.update(patch) - except exc.HTTPNotFound: - raise exc.CommandError('Kubernetes upgrade UUID not found') - - _print_kube_upgrade_show(kube_upgrade) + patch_kube_upgrade(cc, data) def do_kube_upgrade_delete(cc, args): diff --git a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/kube_upgrade.py b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/kube_upgrade.py index be0f4d816f..20797ba2f9 100755 --- a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/kube_upgrade.py +++ b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/kube_upgrade.py @@ -7,6 +7,7 @@ from fm_api import constants as fm_constants from fm_api import fm_api +from distutils.version import LooseVersion import pecan from pecan import rest import os @@ -152,6 +153,116 @@ class KubeUpgradeController(rest.RestController): "the kubernetes upgrade: %s" % available_patches)) + def _check_applied_apps_compatibility(self, from_version, to_version): + """Ensure that applied applications are compatible with + Kubernetes versions across the upgrade process + + :param from_version: Initial Kubernetes version + :param to_version: Target Kubernetes version + """ + + system = pecan.request.dbapi.isystem_get_one() + if system.system_mode == constants.SYSTEM_MODE_SIMPLEX: + next_versions = self._kube_operator.kube_get_higher_patch_version(from_version, + to_version) + else: + next_versions = [to_version] + + if not next_versions: + raise wsme.exc.ClientSideError(_("Error while retrieving Kubernetes intermediate " + "versions")) + + incompatible_apps = set() + lower_k8s_version_from_incompatible_apps = None + from_version = from_version.lstrip('v') + to_version = to_version.lstrip('v') + next_versions = [x.lstrip('v') for x in next_versions] + apps = pecan.request.dbapi.kube_app_get_all() + for app in apps: + if app.status != constants.APP_APPLY_SUCCESS: + continue + + # Applications with timing=pre need to be compatible with the current, + # intermediate and target k8s versions: + pre_update_compatible = pecan.request.dbapi.kube_app_bundle_is_k8s_compatible( + name=app.name, + k8s_timing=constants.APP_METADATA_TIMING_PRE, + current_k8s_version=from_version, + target_k8s_version=to_version) + if not pre_update_compatible: + LOG.debug("Unable to find a version of application {} to be pre updated." + .format(app.name)) + + # Applications with timing=post should be compatible with the target k8s version. + # Compatibility with current or intermediate versions is not required: + post_update_compatible = \ + pecan.request.dbapi.kube_app_bundle_is_k8s_compatible( + name=app.name, + k8s_timing=constants.APP_METADATA_TIMING_POST, + target_k8s_version=to_version) + if not post_update_compatible: + LOG.debug("Unable to find a version of application {} to be post updated." + .format(app.name)) + + if not pre_update_compatible and not post_update_compatible: + # If the app cannot be pre or post updated, check if we can proceed with + # the current applied version. + applied_app_kube_min_version, applied_app_kube_max_version = \ + cutils.get_app_supported_kube_version(app.name, app.app_version) + if kubernetes.is_kube_version_supported( + to_version, applied_app_kube_min_version, applied_app_kube_max_version): + LOG.info("No updates found for application {} during Kubernetes upgrade " + "to {} but current applied version {} is supported.".format( + app.name, + to_version, + app.app_version)) + continue + + max_compatible_version = \ + pecan.request.dbapi.kube_app_bundle_max_k8s_compatible_by_name( + name=app.name, + current_k8s_version=from_version, + target_k8s_version=to_version) + if max_compatible_version is None: + max_compatible_version = from_version + if (LooseVersion(applied_app_kube_max_version) > + LooseVersion(max_compatible_version)): + max_compatible_version = applied_app_kube_max_version + + incompatible_versions = [x for x in next_versions if LooseVersion(x) > + LooseVersion(max_compatible_version)] + LOG.error("Unable to find a suitable version of application {} " + "compatible with the following Kubernetes versions: {}." + .format(app.name, ', '.join(str(s) for s in incompatible_versions))) + + incompatible_apps.add(app.name) + if (lower_k8s_version_from_incompatible_apps is None or + LooseVersion(max_compatible_version) < + LooseVersion(lower_k8s_version_from_incompatible_apps)): + lower_k8s_version_from_incompatible_apps = max_compatible_version + + # If the lowest compatible version found amongst all apps is out of the version range we + # support then there is no upgrade path available. + # If the lowest compatible version found amongst all apps is within the range we support + # then inform the highest supported version. + if (lower_k8s_version_from_incompatible_apps and + LooseVersion(lower_k8s_version_from_incompatible_apps) < + LooseVersion(next_versions[0])): + raise wsme.exc.ClientSideError(_( + "The following apps are incompatible with intermediate/target Kubernetes " + "versions: {}. No upgrade path available to Kubernetes version {}." + .format(', '.join(str(s) for s in incompatible_apps), to_version))) + elif lower_k8s_version_from_incompatible_apps: + highest_supported_version = next( + x for x in list(reversed(next_versions)) + if (LooseVersion(x) <= + LooseVersion(lower_k8s_version_from_incompatible_apps))) + + raise wsme.exc.ClientSideError(_( + "The following apps are incompatible with intermediate/target Kubernetes " + "versions: {}. The system can be upgraded up to Kubernetes {}." + .format(', '.join(str(s) for s in incompatible_apps), highest_supported_version))) + @wsme_pecan.wsexpose(KubeUpgradeCollection) def get_all(self): """Retrieve a list of kubernetes upgrades.""" @@ -175,7 +286,6 @@ class KubeUpgradeController(rest.RestController): force = body.get('force', False) is True alarm_ignore_list = body.get('alarm_ignore_list') system = pecan.request.dbapi.isystem_get_one() - retry = False # There must not be a platform upgrade in progress try: @@ -189,21 +299,12 @@ class KubeUpgradeController(rest.RestController): # There must not already be a kubernetes upgrade in progress try: - kube_upgrade_obj = objects.kube_upgrade.get_one(pecan.request.context) + pecan.request.dbapi.kube_upgrade_get_one() except exception.NotFound: pass else: - # Allow retrying the new Kubernetes upgrade if the current - # state is 'upgrade-starting-failed'. - if (kube_upgrade_obj.state == kubernetes.KUBE_UPGRADE_STARTING_FAILED and - kube_upgrade_obj.to_version == to_version): - retry = True - if alarm_ignore_list is None: - alarm_ignore_list = [] - alarm_ignore_list.append(fm_constants.FM_ALARM_ID_KUBE_UPGRADE_IN_PROGRESS) - else: - raise wsme.exc.ClientSideError(_( - "A kubernetes upgrade is already in progress")) + raise wsme.exc.ClientSideError(_( + "A kubernetes upgrade is already in progress")) # Check whether target version is available or not try: @@ -260,63 +361,51 @@ class KubeUpgradeController(rest.RestController): "System is not in a valid state for kubernetes upgrade. " "Run system health-query-kube-upgrade for more details.")) - if retry: - # Update upgrade record - kube_upgrade_obj.state = kubernetes.KUBE_UPGRADE_STARTING - kube_upgrade_obj.save() - else: - # Create upgrade record. - create_values = {'from_version': current_kube_version, - 'to_version': to_version, - 'state': kubernetes.KUBE_UPGRADE_STARTING} - kube_upgrade_obj = pecan.request.dbapi.kube_upgrade_create(create_values) + # Check app compatibility + self._check_applied_apps_compatibility(current_kube_version, to_version) - try: - # Set the target version for each host to the current version - update_values = {'target_version': current_kube_version} - kube_host_upgrades = pecan.request.dbapi.kube_host_upgrade_get_list() - for kube_host_upgrade in kube_host_upgrades: - pecan.request.dbapi.kube_host_upgrade_update(kube_host_upgrade.id, - update_values) - # Raise alarm to show a kubernetes upgrade is in progress - entity_instance_id = "%s=%s" % (fm_constants.FM_ENTITY_TYPE_HOST, - constants.CONTROLLER_HOSTNAME) - fault = fm_api.Fault( - alarm_id=fm_constants.FM_ALARM_ID_KUBE_UPGRADE_IN_PROGRESS, - alarm_state=fm_constants.FM_ALARM_STATE_SET, - entity_type_id=fm_constants.FM_ENTITY_TYPE_HOST, - entity_instance_id=entity_instance_id, - severity=fm_constants.FM_ALARM_SEVERITY_MINOR, - reason_text="Kubernetes upgrade in progress.", - # operational - alarm_type=fm_constants.FM_ALARM_TYPE_7, - # congestion - probable_cause=fm_constants.ALARM_PROBABLE_CAUSE_8, - proposed_repair_action="No action required.", - service_affecting=False) - fm_api.FaultAPIs().set_fault(fault) + # Create upgrade record. + create_values = {'from_version': current_kube_version, + 'to_version': to_version, + 'state': kubernetes.KUBE_UPGRADE_STARTED} + new_upgrade = pecan.request.dbapi.kube_upgrade_create(create_values) - # Set the new kubeadm version in the DB. - # This will not actually change the bind mounts until we apply a - # puppet manifest that makes use of it. - kube_cmd_versions = objects.kube_cmd_version.get( - pecan.request.context) - kube_cmd_versions.kubeadm_version = to_version.lstrip('v') - kube_cmd_versions.save() + # Set the target version for each host to the current version + update_values = {'target_version': current_kube_version} + kube_host_upgrades = pecan.request.dbapi.kube_host_upgrade_get_list() + for kube_host_upgrade in kube_host_upgrades: + pecan.request.dbapi.kube_host_upgrade_update(kube_host_upgrade.id, + update_values) + # Raise alarm to show a kubernetes upgrade is in progress + entity_instance_id = "%s=%s" % (fm_constants.FM_ENTITY_TYPE_HOST, + constants.CONTROLLER_HOSTNAME) + fault = fm_api.Fault( + alarm_id=fm_constants.FM_ALARM_ID_KUBE_UPGRADE_IN_PROGRESS, + alarm_state=fm_constants.FM_ALARM_STATE_SET, + entity_type_id=fm_constants.FM_ENTITY_TYPE_HOST, + entity_instance_id=entity_instance_id, + severity=fm_constants.FM_ALARM_SEVERITY_MINOR, + reason_text="Kubernetes upgrade in progress.", + # operational + alarm_type=fm_constants.FM_ALARM_TYPE_7, + # congestion + probable_cause=fm_constants.ALARM_PROBABLE_CAUSE_8, + proposed_repair_action="No action required.", + service_affecting=False) + fm_api.FaultAPIs().set_fault(fault) - LOG.info("Starting kubernetes upgrade from version: %s to version: %s" - % (current_kube_version, to_version)) + # Set the new kubeadm version in the DB. + # This will not actually change the executable version until we apply a + # puppet manifest that makes use of it. + kube_cmd_versions = objects.kube_cmd_version.get( + pecan.request.context) + kube_cmd_versions.kubeadm_version = to_version.lstrip('v') + kube_cmd_versions.save() - # Tell the conductor to update the required apps and mark the upgrade as started - pecan.request.rpcapi.kube_upgrade_start( - pecan.request.context, - to_version) - except Exception as e: - LOG.exception("Failed to start Kubernetes upgrade: %s" % e) - kube_upgrade_obj.state = kubernetes.KUBE_UPGRADE_STARTING_FAILED - kube_upgrade_obj.save() + LOG.info("Started kubernetes upgrade from version: %s to version: %s" + % (current_kube_version, to_version)) - return KubeUpgrade.convert_with_links(kube_upgrade_obj) + return KubeUpgrade.convert_with_links(new_upgrade) @cutils.synchronized(LOCK_NAME) @wsme.validate([KubeUpgradePatchType]) @@ -337,10 +426,12 @@ class KubeUpgradeController(rest.RestController): if updates['state'] and updates['state'].split('-')[-1] == 'failed': if kube_upgrade_obj.state in [ kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES, + kubernetes.KUBE_PRE_UPDATING_APPS, kubernetes.KUBE_UPGRADING_FIRST_MASTER, kubernetes.KUBE_UPGRADING_SECOND_MASTER, kubernetes.KUBE_UPGRADING_STORAGE, - kubernetes.KUBE_UPGRADING_NETWORKING]: + kubernetes.KUBE_UPGRADING_NETWORKING, + kubernetes.KUBE_POST_UPDATING_APPS]: kube_upgrade_obj.state = updates['state'] kube_upgrade_obj.save() LOG.info("Kubernetes upgrade state is changed to %s" % updates['state']) @@ -383,15 +474,37 @@ class KubeUpgradeController(rest.RestController): kube_upgrade_obj.to_version) return KubeUpgrade.convert_with_links(kube_upgrade_obj) + elif updates['state'] == kubernetes.KUBE_PRE_UPDATING_APPS: + # Make sure upgrade is in the correct state to update apps + if kube_upgrade_obj.state not in [ + kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES, + kubernetes.KUBE_PRE_UPDATING_APPS_FAILED]: + raise wsme.exc.ClientSideError(_( + "Kubernetes upgrade must be in %s or %s state to " + "update applications" % + (kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES, + kubernetes.KUBE_PRE_UPDATING_APPS_FAILED))) + + # Update the upgrade state + kube_upgrade_obj.state = kubernetes.KUBE_PRE_UPDATING_APPS + kube_upgrade_obj.save() + + # Tell the conductor to update the required apps + pecan.request.rpcapi.kube_pre_application_update(pecan.request.context) + + LOG.info("Updating applications to match target Kubernetes version %s" % + kube_upgrade_obj.to_version) + return KubeUpgrade.convert_with_links(kube_upgrade_obj) + elif updates['state'] == kubernetes.KUBE_UPGRADING_NETWORKING: # Make sure upgrade is in the correct state to upgrade networking if kube_upgrade_obj.state not in [ - kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES, + kubernetes.KUBE_PRE_UPDATED_APPS, kubernetes.KUBE_UPGRADING_NETWORKING_FAILED]: raise wsme.exc.ClientSideError(_( "Kubernetes upgrade must be in %s or %s state to " "upgrade networking" % - (kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES, + (kubernetes.KUBE_PRE_UPDATED_APPS, kubernetes.KUBE_UPGRADING_NETWORKING_FAILED))) # Update the upgrade state @@ -438,7 +551,11 @@ class KubeUpgradeController(rest.RestController): "in %s" % system.system_mode)) if kube_upgrade_obj.state in [kubernetes.KUBE_UPGRADE_ABORTING, kubernetes.KUBE_UPGRADE_ABORTED, - kubernetes.KUBE_UPGRADE_COMPLETE]: + kubernetes.KUBE_UPGRADE_COMPLETE, + kubernetes.KUBE_POST_UPDATING_APPS, + kubernetes.KUBE_POST_UPDATING_APPS_FAILED, + kubernetes.KUBE_POST_UPDATED_APPS + ]: raise wsme.exc.ClientSideError(_( "Cannot abort the kubernetes upgrade it is in %s state" % (kube_upgrade_obj.state))) @@ -461,12 +578,13 @@ class KubeUpgradeController(rest.RestController): # Update the state as aborted for these states since no actual k8s changes done # so we don't need to do anything more to complete the abort. - if kube_upgrade_obj.state in [kubernetes.KUBE_UPGRADE_STARTING, - kubernetes.KUBE_UPGRADE_STARTING_FAILED, - kubernetes.KUBE_UPGRADE_STARTED, + if kube_upgrade_obj.state in [kubernetes.KUBE_UPGRADE_STARTED, kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES, kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED, - kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES]: + kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES, + kubernetes.KUBE_PRE_UPDATING_APPS, + kubernetes.KUBE_PRE_UPDATING_APPS_FAILED, + kubernetes.KUBE_PRE_UPDATED_APPS]: kube_upgrade_obj.state = kubernetes.KUBE_UPGRADE_ABORTED kube_upgrade_obj.save() else: @@ -627,12 +745,6 @@ class KubeUpgradeController(rest.RestController): if role == constants.DISTRIBUTED_CLOUD_ROLE_SYSTEMCONTROLLER: dc_api.notify_dcmanager_kubernetes_upgrade_completed() - # Update apps that contain 'k8s_upgrade.timing = post' metadata - pecan.request.rpcapi.update_apps_based_on_k8s_version_async( - pecan.request.context, - kube_upgrade_obj.to_version, - constants.APP_METADATA_TIMING_POST) - # Check if apps need to be reapplied pecan.request.rpcapi.evaluate_apps_reapply( pecan.request.context, @@ -640,6 +752,27 @@ class KubeUpgradeController(rest.RestController): return KubeUpgrade.convert_with_links(kube_upgrade_obj) + elif updates['state'] == kubernetes.KUBE_POST_UPDATING_APPS: + # Make sure upgrade is in the correct state to update apps + if kube_upgrade_obj.state not in [ + kubernetes.KUBE_UPGRADE_COMPLETE, + kubernetes.KUBE_POST_UPDATING_APPS_FAILED]: + raise wsme.exc.ClientSideError(_( + "Kubernetes upgrade must be in %s or %s state to update applications" % + (kubernetes.KUBE_UPGRADE_COMPLETE, + kubernetes.KUBE_POST_UPDATING_APPS_FAILED))) + + # Update the upgrade state + kube_upgrade_obj.state = kubernetes.KUBE_POST_UPDATING_APPS + kube_upgrade_obj.save() + + # Update apps that contain 'k8s_upgrade.timing = post' metadata + pecan.request.rpcapi.kube_post_application_update(pecan.request.context, + kube_upgrade_obj.to_version) + + LOG.info("Updating applications to match current Kubernetes version %s" % + kube_upgrade_obj.to_version) + return KubeUpgrade.convert_with_links(kube_upgrade_obj) else: raise wsme.exc.ClientSideError(_( "Invalid state %s supplied" % updates['state'])) @@ -656,12 +789,16 @@ class KubeUpgradeController(rest.RestController): raise wsme.exc.ClientSideError(_( "A kubernetes upgrade is not in progress")) if kube_upgrade_obj.state not in [kubernetes.KUBE_UPGRADE_COMPLETE, - kubernetes.KUBE_UPGRADE_ABORTED]: + kubernetes.KUBE_POST_UPDATING_APPS_FAILED, + kubernetes.KUBE_POST_UPDATED_APPS, + kubernetes.KUBE_UPGRADE_ABORTED]: # The upgrade must be in complete or abort state to delete raise wsme.exc.ClientSideError(_( - "Kubernetes upgrade must be in %s or %s state to delete" % + "Kubernetes upgrade must be in %s, %s, %s or %s state to delete" % (kubernetes.KUBE_UPGRADE_COMPLETE, - kubernetes.KUBE_UPGRADE_ABORTED))) + kubernetes.KUBE_POST_UPDATING_APPS_FAILED, + kubernetes.KUBE_POST_UPDATED_APPS, + kubernetes.KUBE_UPGRADE_ABORTED))) # Clean up k8s control-plane backup pecan.request.rpcapi.remove_kube_control_plane_backup( diff --git a/sysinv/sysinv/sysinv/sysinv/common/kubernetes.py b/sysinv/sysinv/sysinv/sysinv/common/kubernetes.py index 07df57f608..ac7d722038 100644 --- a/sysinv/sysinv/sysinv/sysinv/common/kubernetes.py +++ b/sysinv/sysinv/sysinv/sysinv/common/kubernetes.py @@ -78,12 +78,16 @@ KUBE_CONTROLLER_MANAGER = 'kube-controller-manager' KUBE_SCHEDULER = 'kube-scheduler' # Kubernetes upgrade states -KUBE_UPGRADE_STARTING = 'upgrade-starting' -KUBE_UPGRADE_STARTING_FAILED = 'upgrade-starting-failed' KUBE_UPGRADE_STARTED = 'upgrade-started' KUBE_UPGRADE_DOWNLOADING_IMAGES = 'downloading-images' KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED = 'downloading-images-failed' KUBE_UPGRADE_DOWNLOADED_IMAGES = 'downloaded-images' +KUBE_PRE_UPDATING_APPS = 'pre-updating-apps' +KUBE_PRE_UPDATING_APPS_FAILED = 'pre-updating-apps-failed' +KUBE_PRE_UPDATED_APPS = 'pre-updated-apps' +KUBE_POST_UPDATING_APPS = 'post-updating-apps' +KUBE_POST_UPDATING_APPS_FAILED = 'post-updating-apps-failed' +KUBE_POST_UPDATED_APPS = 'post-updated-apps' KUBE_UPGRADING_NETWORKING = 'upgrading-networking' KUBE_UPGRADING_NETWORKING_FAILED = 'upgrading-networking-failed' KUBE_UPGRADED_NETWORKING = 'upgraded-networking' diff --git a/sysinv/sysinv/sysinv/sysinv/conductor/manager.py b/sysinv/sysinv/sysinv/sysinv/conductor/manager.py index 2f5622637f..7f9511153a 100644 --- a/sysinv/sysinv/sysinv/sysinv/conductor/manager.py +++ b/sysinv/sysinv/sysinv/sysinv/conductor/manager.py @@ -7618,9 +7618,45 @@ class ConductorManager(service.PeriodicService): self._update_image_conversion_alarm(fm_constants.FM_ALARM_STATE_CLEAR, constants.FILESYSTEM_NAME_IMAGE_CONVERSION) - def _auto_upload_managed_app(self, context, app_name): + def _auto_upload_managed_app(self, + context, + app_name, + k8s_version=None, + k8s_upgrade_timing=None, + async_upload=True): + """ Automatically upload managed applications. + + :param context: Context of the request. + :param app_name: Name of the application to be uploaded. + :param k8s_version: Kubernetes target version. + :param k8s_upgrade_timing: When applications should be uploaded. + :param async_upload: Upload asynchronously if True. Upload synchronously if False. + :return: True if the upload successfully started when running asynchronously. + True if the app was successfully uploaded when running synchronously. + False if an error has occurred. + None if there is not an upload version available for the given app. + """ + if self._patching_operation_is_occurring(): - return + return False + + # Delete current uploaded version if a newer one is available + try: + existing_app = kubeapp_obj.get_by_name(context, app_name) + app_bundle = self._get_app_bundle_for_update(existing_app, k8s_version, k8s_upgrade_timing) + if app_bundle: + hook_info_delete = LifecycleHookInfo() + hook_info_delete.mode = constants.APP_LIFECYCLE_MODE_AUTO + self.perform_app_delete(context, existing_app, hook_info_delete) + else: + LOG.debug("No bundle found for uploading a new version of %s" % app_name) + return + except exception.KubeAppNotFound: + pass + except Exception as e: + LOG.exception("Failed to delete app {} during automatic upload: {}" + .format(app_name, e)) + return False LOG.info("Platform managed application %s: Creating..." % app_name) app_data = {'name': app_name, @@ -7628,46 +7664,65 @@ class ConductorManager(service.PeriodicService): 'manifest_name': constants.APP_MANIFEST_NAME_PLACEHOLDER, 'manifest_file': constants.APP_TARFILE_NAME_PLACEHOLDER, 'status': constants.APP_UPLOAD_IN_PROGRESS} + try: self.dbapi.kube_app_create(app_data) app = kubeapp_obj.get_by_name(context, app_name) + app_bundle = self._get_app_bundle_for_update(app, k8s_version, k8s_upgrade_timing) + + if app_bundle is None: + # Skip if no bundles are found + LOG.debug("No bundle found for uploading %s" % app_name) + return + + tarball = self._check_tarfile(app_name, app_bundle.file_path) + if ((tarball.manifest_name is None) or + (tarball.manifest_file is None)): + app.status = constants.APP_UPLOAD_FAILURE + app.save() + return False + + app.name = tarball.app_name + app.app_version = tarball.app_version + app.manifest_name = tarball.manifest_name + app.manifest_file = os.path.basename(tarball.manifest_file) + app.save() + + # Action: Upload. + # Do not block this audit task or any other periodic task. This + # could be long running. The next audit cycle will pick up the + # latest status. + LOG.info("Platform managed application %s: " + "Uploading..." % app_name) + + hook_info = LifecycleHookInfo() + hook_info.mode = constants.APP_LIFECYCLE_MODE_AUTO + + if async_upload: + greenthread.spawn(self.perform_app_upload, + context, + app, + tarball.tarball_name, + hook_info) + else: + self.perform_app_upload(context, + app, + tarball.tarball_name, + hook_info) except exception.KubeAppAlreadyExists as e: LOG.exception(e) - return + return False except exception.KubeAppNotFound as e: LOG.exception(e) - return - - tarfile = self._search_tarfile(app_name, managed_app=True) - if tarfile is None: - # Skip if no tarball or multiple tarballs found - return - - tarball = self._check_tarfile(app_name, tarfile) - if ((tarball.manifest_name is None) or - (tarball.manifest_file is None)): - app.status = constants.APP_UPLOAD_FAILURE - app.save() - return - - app.name = tarball.app_name - app.app_version = tarball.app_version - app.manifest_name = tarball.manifest_name - app.manifest_file = os.path.basename(tarball.manifest_file) - app.save() - - # Action: Upload. - # Do not block this audit task or any other periodic task. This - # could be long running. The next audit cycle will pick up the - # latest status. - LOG.info("Platform managed application %s: " - "Uploading..." % app_name) - - hook_info = LifecycleHookInfo() - hook_info.mode = constants.APP_LIFECYCLE_MODE_AUTO - - greenthread.spawn(self.perform_app_upload, context, - app, tarball.tarball_name, hook_info) + return False + except Exception as e: + if k8s_version: + LOG.exception("App {} automatic upload to match Kubernetes version {} " + "failed with: {}".format(app.name, k8s_version, e)) + else: + LOG.exception("App {} automatic upload {} failed with: {}" + .format(app.name, k8s_version, e)) + return False def _auto_apply_managed_app(self, context, app_name): try: @@ -7702,7 +7757,7 @@ class ConductorManager(service.PeriodicService): self._inner_sync_auto_apply(context, app_name) - def update_apps_based_on_k8s_version_sync(self, context, k8s_version, k8s_upgrade_timing): + def update_apps_based_on_k8s_version(self, context, k8s_version, k8s_upgrade_timing): """ Update applications based on a given Kubernetes version (blocking). :param context: Context of the request @@ -7730,10 +7785,12 @@ class ConductorManager(service.PeriodicService): except exception.KubeAppNotFound: continue - # Apps should be either in 'applied' or 'apply-failure' state. + # Apps should be either in 'applied' or 'apply-failure' state to be updated. # Applied apps are selected to be updated since they are currently in use. # If the app is in 'apply-failure' state we give it a chance to be # successfully applied via the update process. + # If a newer compatible version of an app in 'uploaded' or 'uploaded-failed' state + # is available then the current version is removed and the new one is uploaded. if (app.status == constants.APP_APPLY_SUCCESS or app.status == constants.APP_APPLY_FAILURE): threads[app.name] = threadpool.spawn(self._auto_update_app, @@ -7742,6 +7799,14 @@ class ConductorManager(service.PeriodicService): k8s_version, k8s_upgrade_timing, async_update=False) + elif (app.status == constants.APP_UPLOAD_SUCCESS or + app.status == constants.APP_UPLOAD_FAILURE): + threads[app.name] = threadpool.spawn(self._auto_upload_managed_app, + context, + app_name, + k8s_version, + k8s_upgrade_timing, + async_upload=False) # Wait for all updates to finish threadpool.waitall() @@ -7755,38 +7820,6 @@ class ConductorManager(service.PeriodicService): return result - def update_apps_based_on_k8s_version_async(self, context, k8s_version, k8s_upgrade_timing): - """ Update applications based on a given Kubernetes version (non-blocking). - - :param context: Context of the request - :param k8s_version: Kubernetes target version. - :param k8s_upgrade_timing: When applications should be updated. - """ - update_candidates = [app_name for app_name in - self.apps_metadata[constants.APP_METADATA_APPS].keys()] - - LOG.info("Checking available application updates for Kubernetes version {}." - .format(k8s_version)) - - for app_name in update_candidates: - try: - app = kubeapp_obj.get_by_name(context, app_name) - except exception.KubeAppNotFound: - continue - - # Apps should be either in 'applied' or 'apply-failure' state. - # Applied apps are selected to be updated since they are currently in use. - # If the app is in 'apply-failure' state we give it a chance to be - # successfully applied via the update process. - if (app.status == constants.APP_APPLY_SUCCESS or - app.status == constants.APP_APPLY_FAILURE): - if self._auto_update_app(context, - app_name, - k8s_version, - k8s_upgrade_timing) is False: - LOG.error("Failed to update {} to match Kubernetes version {}" - .format(app_name, k8s_version)) - def _get_app_bundle_for_update(self, app, k8s_version=None, k8s_upgrade_timing=None): """ Retrieve metadata from the most updated application bundle that can be used to update the given app. @@ -7833,11 +7866,13 @@ class ConductorManager(service.PeriodicService): .format(k8s_version, bundle_metadata.k8s_maximum_version, bundle_metadata.file_path)) - elif LooseVersion(bundle_metadata.version) == LooseVersion(app.app_version): + elif (app.app_version != constants.APP_VERSION_PLACEHOLDER and + LooseVersion(bundle_metadata.version) == LooseVersion(app.app_version)): LOG.debug("Bundle {} version and installed app version are the same ({})" .format(bundle_metadata.file_path, app.app_version)) - elif LooseVersion(bundle_metadata.version) < LooseVersion(app.app_version): + elif (app.app_version != constants.APP_VERSION_PLACEHOLDER and + LooseVersion(bundle_metadata.version) < LooseVersion(app.app_version)): LOG.debug("Bundle {} version {} is lower than installed app version ({})" .format(bundle_metadata.file_path, bundle_metadata.version, @@ -7859,7 +7894,8 @@ class ConductorManager(service.PeriodicService): # bundle is available instead. if (auto_downgrade and app.app_version not in available_versions and - latest_downgrade_bundle is not None): + latest_downgrade_bundle is not None and + k8s_upgrade_timing is None): LOG.info("Application {} will be downgraded from version {} to {}" .format(app.name, app.app_version, latest_downgrade_bundle.version)) return latest_downgrade_bundle @@ -8462,11 +8498,12 @@ class ConductorManager(service.PeriodicService): # Skip kubernetes labels audit when K8S upgrade is in progress. # The kube-apiserver will not be available during kube-upgrade-abort operation. + # Kubernetes upgrade may be completed but apps still need to be post updated. try: self.verify_k8s_upgrade_not_in_progress() - except Exception: - LOG.info("k8s Upgrade in progress - _k8s_application_audit skip " - "activity") + self.verify_k8s_app_upgrade_is_completed() + except Exception as e: + LOG.info("_k8s_application_audit skip activity: {}".format(str(e))) return if self._verify_restore_in_progress(): @@ -10621,6 +10658,22 @@ class ConductorManager(service.PeriodicService): raise exception.SysinvException(_( "Kubernetes upgrade is in progress and not completed.")) + def verify_k8s_app_upgrade_is_completed(self): + """ Check if application update steps have finished during a k8s upgrade + + Raise an exception if the final update step (post-updated-apps) hasn't + been reached. + """ + try: + kube_upgrade = self.dbapi.kube_upgrade_get_one() + if kube_upgrade.state == kubernetes.KUBE_POST_UPDATED_APPS: + return + except exception.NotFound: + pass + else: + raise exception.SysinvException(_( + "Application post update not completed for the existing k8s upgrade")) + def verify_upgrade_not_in_progress(self): """ Check if there is an upgrade in progress. @@ -16234,8 +16287,10 @@ class ConductorManager(service.PeriodicService): """ # Defer apps reapply evaluation if Kubernetes upgrades are in progress + # or if apps are still post updating. try: self.verify_k8s_upgrade_not_in_progress() + self.verify_k8s_app_upgrade_is_completed() except Exception as e: LOG.info("Deferring apps reapply evaluation. {}".format(str(e))) return @@ -16615,6 +16670,7 @@ class ConductorManager(service.PeriodicService): lifecycle_hook_info_app_upload) except Exception as e: LOG.error("Error performing app_lifecycle_actions %s" % str(e)) + return False def perform_app_apply(self, context, rpc_app, mode, lifecycle_hook_info_app_apply): """Handling of application install request (via AppOperator) @@ -16939,6 +16995,18 @@ class ConductorManager(service.PeriodicService): LOG.info("Successfully completed k8s control plane backup.") + def _check_app_kube_compatibility(self, app, kube_version): + """Checks if an application is compatible with a kubernetes version + + :param app: Application object + :param kube_version: Kubernetes version + """ + kube_min_version, kube_max_version = \ + cutils.get_app_supported_kube_version(app.name, app.app_version) + + return kubernetes.is_kube_version_supported( + kube_version, kube_min_version, kube_max_version) + def _check_installed_apps_compatibility(self, kube_version): """Checks whether all installed applications are compatible with the new k8s version @@ -16956,11 +17024,7 @@ class ConductorManager(service.PeriodicService): if app.status != constants.APP_APPLY_SUCCESS: continue - kube_min_version, kube_max_version = \ - cutils.get_app_supported_kube_version(app.name, app.app_version) - - if not kubernetes.is_kube_version_supported( - kube_version, kube_min_version, kube_max_version): + if not self._check_app_kube_compatibility(app, kube_version): LOG.error("The installed Application {} ({}) is incompatible with the " "new Kubernetes version {}.".format(app.name, app.app_version, @@ -16969,30 +17033,6 @@ class ConductorManager(service.PeriodicService): return success - def kube_upgrade_start(self, context, k8s_version): - """ Start a Kubernetes upgrade by updating all required apps. - - :param context: Context of the request. - :param k8s_version: Kubernetes target version. - :param k8s_upgrade_timing: When apps should be updated. - """ - - kube_upgrade_obj = objects.kube_upgrade.get_one(context) - - if (self.update_apps_based_on_k8s_version_sync(context, - k8s_version, - constants.APP_METADATA_TIMING_PRE) and - self._check_installed_apps_compatibility(k8s_version)): - kube_upgrade_obj.state = kubernetes.KUBE_UPGRADE_STARTED - LOG.info("Started kubernetes upgrade from version: %s to version: %s" - % (kube_upgrade_obj.from_version, kube_upgrade_obj.to_version)) - else: - kube_upgrade_obj.state = kubernetes.KUBE_UPGRADE_STARTING_FAILED - LOG.info("Failed to start kubernetes upgrade from version: %s to version: %s" - % (kube_upgrade_obj.from_version, kube_upgrade_obj.to_version)) - - kube_upgrade_obj.save() - def kube_download_images(self, context, kube_version): """Download the kubernetes images for this version""" @@ -17082,6 +17122,42 @@ class ConductorManager(service.PeriodicService): kube_upgrade_obj.state = kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES kube_upgrade_obj.save() + def kube_application_update(self, context, timing, success_state, failure_state): + """ Generic method to update applications during Kubernetes upgrade + + :param context: Context of the request. + """ + + kube_upgrade_obj = objects.kube_upgrade.get_one(context) + + # Update all apps that are compatible with the target k8s version. + # Check for compatibility after updating since an app update may fail + # and be reverted to a previous incompatible version. + if (self.update_apps_based_on_k8s_version(context, + kube_upgrade_obj.to_version, + timing) and + self._check_installed_apps_compatibility(kube_upgrade_obj.to_version)): + kube_upgrade_obj.state = success_state + LOG.info("Applications updated to match Kubernetes version %s." + % (kube_upgrade_obj.to_version)) + else: + kube_upgrade_obj.state = failure_state + LOG.info("Failed to update applications to match Kubernetes version %s." + % (kube_upgrade_obj.to_version)) + + kube_upgrade_obj.save() + + def kube_pre_application_update(self, context): + """ Update applications before Kubernetes is upgraded. + + :param context: Context of the request. + """ + + self.kube_application_update(context, + constants.APP_METADATA_TIMING_PRE, + kubernetes.KUBE_PRE_UPDATED_APPS, + kubernetes.KUBE_PRE_UPDATING_APPS_FAILED) + def kube_host_cordon(self, context, host_name): """Cordon the pods to evict on this host""" @@ -17400,6 +17476,31 @@ class ConductorManager(service.PeriodicService): kube_upgrade_obj.state = kubernetes.KUBE_UPGRADED_STORAGE kube_upgrade_obj.save() + def kube_post_application_update(self, context, k8s_version): + """ Update applications after Kubernetes is upgraded. + + :param context: Context of the request. + :param k8s_version: Target Kubernetes version + """ + + self.kube_application_update(context, + constants.APP_METADATA_TIMING_POST, + kubernetes.KUBE_POST_UPDATED_APPS, + kubernetes.KUBE_POST_UPDATING_APPS_FAILED) + + # Remove remaining uploaded apps that are not compatible with the new + # Kubernetes version + apps = self.dbapi.kube_app_get_all() + for app in apps: + if app.status != constants.APP_UPLOAD_SUCCESS and \ + app.status != constants.APP_UPLOAD_FAILURE: + continue + + if not self._check_app_kube_compatibility(app, k8s_version): + hook_info_delete = LifecycleHookInfo() + hook_info_delete.mode = constants.APP_LIFECYCLE_MODE_AUTO + self.perform_app_delete(context, app, hook_info_delete) + def kube_upgrade_abort(self, context, kube_state): """ This is an abort procedure we call via 'system kube-upgrade-abort' diff --git a/sysinv/sysinv/sysinv/sysinv/conductor/rpcapi.py b/sysinv/sysinv/sysinv/sysinv/conductor/rpcapi.py index 27f85f66f1..3c979e12ce 100644 --- a/sysinv/sysinv/sysinv/sysinv/conductor/rpcapi.py +++ b/sysinv/sysinv/sysinv/sysinv/conductor/rpcapi.py @@ -1707,32 +1707,24 @@ class ConductorAPI(sysinv.openstack.common.rpc.proxy.RpcProxy): return self.call(context, self.make_msg('get_fernet_keys', key_id=key_id)) - def kube_upgrade_start(self, context, k8s_version): - """Asynchronously, start a Kubernetes upgrade to the given version. + def kube_pre_application_update(self, context): + """Asynchronously, update applications before Kubernetes is upgraded. - :param context: Context of the request - :param k8s_version: Kubernetes target version - :param k8s_upgrade_timing: When applications should be updated + :param context: Context of the request. """ - return self.cast(context, self.make_msg('kube_upgrade_start', + return self.cast(context, self.make_msg('kube_pre_application_update')) + + def kube_post_application_update(self, context, k8s_version): + """Asynchronously, update applications after Kubernetes is upgraded. + + :param context: Context of the request. + :param k8s_version: Target Kubernetes version. + """ + + return self.cast(context, self.make_msg('kube_post_application_update', k8s_version=k8s_version)) - def update_apps_based_on_k8s_version_async(self, - context, - k8s_version, - k8s_upgrade_timing): - """Asynchronously, update all applications based on a given Kubernetes version. - - :param context: Context of the request - :param k8s_version: Kubernetes target version - :param k8s_upgrade_timing: When applications should be updated - """ - - return self.cast(context, self.make_msg('update_apps_based_on_k8s_version_async', - k8s_version=k8s_version, - k8s_upgrade_timing=k8s_upgrade_timing)) - def evaluate_apps_reapply(self, context, trigger): """Synchronously, determine whether an application re-apply is needed, and if so, raise the re-apply flag. diff --git a/sysinv/sysinv/sysinv/sysinv/db/api.py b/sysinv/sysinv/sysinv/sysinv/db/api.py index 1259ee091c..c907a706d0 100644 --- a/sysinv/sysinv/sysinv/sysinv/db/api.py +++ b/sysinv/sysinv/sysinv/sysinv/db/api.py @@ -5134,9 +5134,9 @@ class Connection(object): :param name: Application name. :param k8s_auto_update: Whether automatically updating the application is enabled when upgrading Kubernetes. - :param timing: Application update timing during Kubernetes upgrade - "pre": during kube-upgrade-start. - "post": during kube-upgrade-complete. + :param timing: Application update timing during Kubernetes upgrade. + "pre": before upgrading Kubernetes. + "post": after upgrading Kubernetes. :param limit: Maximum number of entries to return. :param marker: The last item of the previous page; we return the next result set. @@ -5155,6 +5155,35 @@ class Connection(object): def kube_app_bundle_destroy_by_file_path(self, file_path): """Delete records from kube_app_bundle that match a file path""" + @abc.abstractmethod + def kube_app_bundle_is_k8s_compatible(self, + name, k8s_timing, + target_k8s_version, current_k8s_version=None): + """Check if a given application has bundles compatible with current + and target Kubernetes versions. + + :param name: Application name. + :param timing: Application update timing during Kubernetes upgrade + "pre": before upgrading Kubernetes. + "post": after upgrading Kubernetes. + :param target_k8s_version: Kubernetes version that is going to be installed. + :param current_k8s_version: Kubernetes version that is currently running (optional). + :returns: True if app is compatible. False otherwise. + """ + + @abc.abstractmethod + def kube_app_bundle_max_k8s_compatible_by_name(self, + name, + current_k8s_version, + target_k8s_version): + """ Get the maximum compatible Kubernetes version for a given app + + :param name: Application name. + :param current_k8s_version: Kubernetes version that is currently running. + :param target_k8s_version: Kubernetes version that is going to be installed. + :returns: maximum compatible Kubernetes version + """ + @abc.abstractmethod def address_get_by_name_and_family(self, name, family): """ Search database address using name and family diff --git a/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/api.py b/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/api.py index 17778c354e..af60d322fc 100644 --- a/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/api.py +++ b/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/api.py @@ -27,6 +27,7 @@ from oslo_db import exception as db_exc from oslo_db.sqlalchemy import enginefacade from oslo_db.sqlalchemy import utils as db_utils +from sqlalchemy import func from sqlalchemy import insert from sqlalchemy import inspect from sqlalchemy import or_ @@ -9620,3 +9621,49 @@ class Connection(api.Connection): def kube_app_bundle_destroy_by_file_path(self, file_path): self.kube_app_bundle_destroy_all(file_path) + + def kube_app_bundle_is_k8s_compatible(self, + name, k8s_timing, + target_k8s_version, current_k8s_version=None): + + # Stacking filters for better readability + target_k8s_version = target_k8s_version.strip().lstrip('v') + query = model_query(models.KubeAppBundle) + query = query.filter_by(name=name, + auto_update=True, + k8s_auto_update=True, + k8s_timing=k8s_timing) + query = query.filter(models.KubeAppBundle.k8s_minimum_version <= target_k8s_version) + query = query.filter(or_(models.KubeAppBundle.k8s_maximum_version >= target_k8s_version, + models.KubeAppBundle.k8s_maximum_version.is_(None))) + if current_k8s_version: + current_k8s_version = current_k8s_version.strip().lstrip('v') + query = query.filter(models.KubeAppBundle.k8s_minimum_version <= current_k8s_version) + query = query.filter(or_(models.KubeAppBundle.k8s_maximum_version >= current_k8s_version, + models.KubeAppBundle.k8s_maximum_version.is_(None))) + + return query.count() > 0 + + def kube_app_bundle_max_k8s_compatible_by_name(self, + name, + current_k8s_version, + target_k8s_version): + + # Stacking filters for better readability + current_k8s_version = current_k8s_version.strip().lstrip('v') + target_k8s_version = target_k8s_version.strip().lstrip('v') + query = model_query(func.max(models.KubeAppBundle.k8s_maximum_version)) + query = query.filter_by(name=name, auto_update=True, k8s_auto_update=True) + query = query.filter(or_( + models.KubeAppBundle.k8s_timing == constants.APP_METADATA_TIMING_POST, + models.KubeAppBundle.k8s_minimum_version == current_k8s_version)) + query = query.filter(models.KubeAppBundle.k8s_maximum_version <= target_k8s_version) + + try: + result = query.one()[0] + except NoResultFound: + result = None + except MultipleResultsFound: + raise exception.MultipleResults() + + return result diff --git a/sysinv/sysinv/sysinv/sysinv/tests/api/test_kube_upgrade.py b/sysinv/sysinv/sysinv/sysinv/tests/api/test_kube_upgrade.py index 16cb48555c..ac77f91cb2 100644 --- a/sysinv/sysinv/sysinv/sysinv/tests/api/test_kube_upgrade.py +++ b/sysinv/sysinv/sysinv/sysinv/tests/api/test_kube_upgrade.py @@ -72,11 +72,11 @@ class FakeFmClient(object): class FakeConductorAPI(object): def __init__(self): - self.kube_upgrade_start = mock.MagicMock() + self.kube_pre_application_update = mock.MagicMock() self.kube_download_images = mock.MagicMock() self.kube_upgrade_networking = mock.MagicMock() + self.kube_post_application_update = mock.MagicMock() self.kube_upgrade_abort = mock.MagicMock() - self.update_apps_based_on_k8s_version_async = mock.MagicMock() self.evaluate_apps_reapply = mock.MagicMock() self.remove_kube_control_plane_backup = mock.MagicMock() self.kube_delete_container_images = mock.MagicMock() @@ -274,15 +274,11 @@ class TestPostKubeUpgradeSimplex(TestKubeUpgrade, result = self.post_json('/kube_upgrade', create_dict, headers={'User-Agent': 'sysinv-test'}) - # Verify that the upgrade was started - self.fake_conductor_api.kube_upgrade_start.\ - assert_called_with(mock.ANY, 'v1.43.3') - # Verify that the upgrade has the expected attributes self.assertEqual(result.json['from_version'], 'v1.42.1') self.assertEqual(result.json['to_version'], 'v1.43.3') self.assertEqual(result.json['state'], - kubernetes.KUBE_UPGRADE_STARTING) + kubernetes.KUBE_UPGRADE_STARTED) # see if kubeadm_version was changed in DB kube_cmd_version = self.dbapi.kube_cmd_version_get() @@ -380,15 +376,11 @@ class TestPostKubeUpgrade(TestKubeUpgrade, result = self.post_json('/kube_upgrade', create_dict, headers={'User-Agent': 'sysinv-test'}) - # Verify that the upgrade was started - self.fake_conductor_api.kube_upgrade_start.\ - assert_called_with(mock.ANY, 'v1.43.2') - # Verify that the upgrade has the expected attributes self.assertEqual(result.json['from_version'], 'v1.43.1') self.assertEqual(result.json['to_version'], 'v1.43.2') self.assertEqual(result.json['state'], - kubernetes.KUBE_UPGRADE_STARTING) + kubernetes.KUBE_UPGRADE_STARTED) # see if kubeadm_version was changed in DB kube_cmd_version = self.dbapi.kube_cmd_version_get() @@ -512,15 +504,11 @@ class TestPostKubeUpgrade(TestKubeUpgrade, result = self.post_json('/kube_upgrade', create_dict, headers={'User-Agent': 'sysinv-test'}) - # Verify that the upgrade was started - self.fake_conductor_api.kube_upgrade_start.\ - assert_called_with(mock.ANY, 'v1.43.2') - # Verify that the upgrade has the expected attributes self.assertEqual(result.json['from_version'], 'v1.43.1') self.assertEqual(result.json['to_version'], 'v1.43.2') self.assertEqual(result.json['state'], - kubernetes.KUBE_UPGRADE_STARTING) + kubernetes.KUBE_UPGRADE_STARTED) @mock.patch('sysinv.common.health.Health._check_trident_compatibility', lambda x: True) def test_force_create_system_unhealthy_from_mgmt_affecting_alarms(self): @@ -557,15 +545,11 @@ class TestPostKubeUpgrade(TestKubeUpgrade, result = self.post_json('/kube_upgrade', create_dict, headers={'User-Agent': 'sysinv-test'}) - # Verify that the upgrade was started - self.fake_conductor_api.kube_upgrade_start.\ - assert_called_with(mock.ANY, 'v1.43.2') - # Verify that the upgrade has the expected attributes self.assertEqual(result.json['from_version'], 'v1.43.1') self.assertEqual(result.json['to_version'], 'v1.43.2') self.assertEqual(result.json['state'], - kubernetes.KUBE_UPGRADE_STARTING) + kubernetes.KUBE_UPGRADE_STARTED) @mock.patch('sysinv.common.health.Health._check_trident_compatibility', lambda x: True) def test_create_system_unhealthy_from_bad_apps(self): @@ -607,15 +591,11 @@ class TestPostKubeUpgrade(TestKubeUpgrade, result = self.post_json('/kube_upgrade', create_dict, headers={'User-Agent': 'sysinv-test'}) - # Verify that the upgrade was started - self.fake_conductor_api.kube_upgrade_start.\ - assert_called_with(mock.ANY, 'v1.43.3') - # Verify that the upgrade has the expected attributes self.assertEqual(result.json['from_version'], 'v1.43.2') self.assertEqual(result.json['to_version'], 'v1.43.3') self.assertEqual(result.json['state'], - kubernetes.KUBE_UPGRADE_STARTING) + kubernetes.KUBE_UPGRADE_STARTED) def test_create_applied_patch_missing(self): # Test creation of upgrade when applied patch is missing @@ -649,6 +629,99 @@ class TestPostKubeUpgrade(TestKubeUpgrade, class TestPatch(TestKubeUpgrade, dbbase.ProvisionedControllerHostTestCase): + def test_update_state_pre_update_apps(self): + # Test updating the state of an upgrade when + # pre updating applications + + # Create the upgrade + kube_upgrade = dbutils.create_test_kube_upgrade( + from_version='v1.43.1', + to_version='v1.43.2', + state=kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES) + uuid = kube_upgrade.uuid + + # Update state + new_state = kubernetes.KUBE_PRE_UPDATING_APPS + response = self.patch_json('/kube_upgrade', + [{'path': '/state', + 'value': new_state, + 'op': 'replace'}], + headers={'User-Agent': 'sysinv-test'}) + self.assertEqual(response.content_type, 'application/json') + self.assertEqual(response.status_code, http_client.OK) + self.assertEqual(response.json['from_version'], 'v1.43.1') + self.assertEqual(response.json['to_version'], 'v1.43.2') + self.assertEqual(response.json['state'], new_state) + + # Verify that the images were downloaded + self.fake_conductor_api.kube_pre_application_update.\ + assert_called_with(mock.ANY) + + # Verify that the upgrade was updated with the new state + result = self.get_json('/kube_upgrade/%s' % uuid) + self.assertEqual(result['from_version'], 'v1.43.1') + self.assertEqual(result['to_version'], 'v1.43.2') + self.assertEqual(result['state'], new_state) + + def test_update_state_pre_update_apps_after_failure(self): + # Test updating the state of an upgrade when retrying to + # pre update applications after a failure + + # Create the upgrade + kube_upgrade = dbutils.create_test_kube_upgrade( + from_version='v1.43.1', + to_version='v1.43.2', + state=kubernetes.KUBE_PRE_UPDATING_APPS_FAILED) + uuid = kube_upgrade.uuid + + # Update state + new_state = kubernetes.KUBE_PRE_UPDATING_APPS + response = self.patch_json('/kube_upgrade', + [{'path': '/state', + 'value': new_state, + 'op': 'replace'}], + headers={'User-Agent': 'sysinv-test'}) + self.assertEqual(response.content_type, 'application/json') + self.assertEqual(response.status_code, http_client.OK) + self.assertEqual(response.json['from_version'], 'v1.43.1') + self.assertEqual(response.json['to_version'], 'v1.43.2') + self.assertEqual(response.json['state'], new_state) + + # Verify that the images were downloaded + self.fake_conductor_api.kube_pre_application_update.\ + assert_called_with(mock.ANY) + + # Verify that the upgrade was updated with the new state + result = self.get_json('/kube_upgrade/%s' % uuid) + self.assertEqual(result['from_version'], 'v1.43.1') + self.assertEqual(result['to_version'], 'v1.43.2') + self.assertEqual(result['state'], new_state) + + def test_update_state_pre_update_apps_invalid_state(self): + # Test updating the state of an upgrade when attempting to + # pre update apps in an invalid state + + # Create the upgrade + dbutils.create_test_kube_upgrade( + from_version='v1.43.1', + to_version='v1.43.2', + state=kubernetes.KUBE_UPGRADE_STARTED) + + # Update state + new_state = kubernetes.KUBE_PRE_UPDATING_APPS + result = self.patch_json('/kube_upgrade', + [{'path': '/state', + 'value': new_state, + 'op': 'replace'}], + headers={'User-Agent': 'sysinv-test'}, + expect_errors=True) + + # Verify the failure + self.assertEqual(result.content_type, 'application/json') + self.assertEqual(http_client.BAD_REQUEST, result.status_int) + self.assertIn("Kubernetes upgrade must be in", + result.json['error_message']) + def test_update_state_download_images(self): # Test updating the state of an upgrade to download images @@ -748,7 +821,7 @@ class TestPatch(TestKubeUpgrade, kube_upgrade = dbutils.create_test_kube_upgrade( from_version='v1.43.1', to_version='v1.43.2', - state=kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES) + state=kubernetes.KUBE_PRE_UPDATED_APPS) uuid = kube_upgrade.uuid # Update state @@ -833,6 +906,99 @@ class TestPatch(TestKubeUpgrade, self.assertIn("Kubernetes upgrade must be in", result.json['error_message']) + def test_update_state_post_update_apps(self): + # Test updating the state of an upgrade when + # post updating applications + + # Create the upgrade + kube_upgrade = dbutils.create_test_kube_upgrade( + from_version='v1.43.1', + to_version='v1.43.2', + state=kubernetes.KUBE_UPGRADE_COMPLETE) + uuid = kube_upgrade.uuid + + # Update state + new_state = kubernetes.KUBE_POST_UPDATING_APPS + response = self.patch_json('/kube_upgrade', + [{'path': '/state', + 'value': new_state, + 'op': 'replace'}], + headers={'User-Agent': 'sysinv-test'}) + self.assertEqual(response.content_type, 'application/json') + self.assertEqual(response.status_code, http_client.OK) + self.assertEqual(response.json['from_version'], 'v1.43.1') + self.assertEqual(response.json['to_version'], 'v1.43.2') + self.assertEqual(response.json['state'], new_state) + + # Run post application update + self.fake_conductor_api.kube_post_application_update.\ + assert_called_with(mock.ANY, kube_upgrade.to_version) + + # Verify that the upgrade was updated with the new state + result = self.get_json('/kube_upgrade/%s' % uuid) + self.assertEqual(result['from_version'], 'v1.43.1') + self.assertEqual(result['to_version'], 'v1.43.2') + self.assertEqual(result['state'], new_state) + + def test_update_state_post_update_apps_after_failure(self): + # Test updating the state of an upgrade when retrying to + # post update applications after a failure + + # Create the upgrade + kube_upgrade = dbutils.create_test_kube_upgrade( + from_version='v1.43.1', + to_version='v1.43.2', + state=kubernetes.KUBE_POST_UPDATING_APPS_FAILED) + uuid = kube_upgrade.uuid + + # Update state + new_state = kubernetes.KUBE_POST_UPDATING_APPS + response = self.patch_json('/kube_upgrade', + [{'path': '/state', + 'value': new_state, + 'op': 'replace'}], + headers={'User-Agent': 'sysinv-test'}) + self.assertEqual(response.content_type, 'application/json') + self.assertEqual(response.status_code, http_client.OK) + self.assertEqual(response.json['from_version'], 'v1.43.1') + self.assertEqual(response.json['to_version'], 'v1.43.2') + self.assertEqual(response.json['state'], new_state) + + # Verify that the images were downloaded + self.fake_conductor_api.kube_post_application_update.\ + assert_called_with(mock.ANY, kube_upgrade.to_version) + + # Verify that the upgrade was updated with the new state + result = self.get_json('/kube_upgrade/%s' % uuid) + self.assertEqual(result['from_version'], 'v1.43.1') + self.assertEqual(result['to_version'], 'v1.43.2') + self.assertEqual(result['state'], new_state) + + def test_update_state_post_update_apps_invalid_state(self): + # Test updating the state of an upgrade to post update apps in an + # invalid state + + # Create the upgrade + dbutils.create_test_kube_upgrade( + from_version='v1.43.1', + to_version='v1.43.2', + state=kubernetes.KUBE_UPGRADED_NETWORKING) + + # Update state + new_state = kubernetes.KUBE_POST_UPDATING_APPS + result = self.patch_json('/kube_upgrade', + [{'path': '/state', + 'value': new_state, + 'op': 'replace'}], + headers={'User-Agent': 'sysinv-test'}, + expect_errors=True) + + # Verify the failure + self.assertEqual(result.content_type, 'application/json') + self.assertEqual(http_client.BAD_REQUEST, result.status_int) + self.assertIn("Kubernetes upgrade must be in", + result.json['error_message']) + def test_update_state_complete(self): # Test updating the state of an upgrade to complete self.kube_get_version_states_result = {'v1.42.1': 'available', diff --git a/sysinv/sysinv/sysinv/sysinv/tests/conductor/test_manager.py b/sysinv/sysinv/sysinv/sysinv/tests/conductor/test_manager.py index 693146a92c..acbffe7632 100644 --- a/sysinv/sysinv/sysinv/sysinv/tests/conductor/test_manager.py +++ b/sysinv/sysinv/sysinv/sysinv/tests/conductor/test_manager.py @@ -1050,7 +1050,7 @@ class ManagerTestCase(base.DbTestCase): ret = self.service.platform_interfaces(self.context, ihost['id'] + 1) self.assertEqual(ret, []) - def test_kube_upgrade_start(self): + def test_kube_pre_application_update(self): # Create application dbutils.create_test_app( name='stx-openstack', @@ -1066,19 +1066,19 @@ class ManagerTestCase(base.DbTestCase): utils.create_test_kube_upgrade( from_version=from_version, to_version=to_version, - state=kubernetes.KUBE_UPGRADE_STARTING, + state=kubernetes.KUBE_UPGRADE_STARTED, ) - # Start upgrade - self.service.kube_upgrade_start(self.context, to_version) + # Run update + self.service.kube_pre_application_update(self.context) # Verify that the upgrade state was updated updated_upgrade = self.dbapi.kube_upgrade_get_one() self.assertEqual(updated_upgrade.state, - kubernetes.KUBE_UPGRADE_STARTED) + kubernetes.KUBE_PRE_UPDATED_APPS) - def test_kube_upgrade_start_app_not_compatible(self): - # Test creation of upgrade when the installed application isn't + def test_kube_pre_application_update_app_not_compatible(self): + # Test pre app update step when the installed application isn't # compatible with the new kubernetes version # Create application @@ -1096,16 +1096,16 @@ class ManagerTestCase(base.DbTestCase): utils.create_test_kube_upgrade( from_version=from_version, to_version=to_version, - state=kubernetes.KUBE_UPGRADE_STARTING, + state=kubernetes.KUBE_UPGRADE_STARTED, ) - # Start upgrade - self.service.kube_upgrade_start(self.context, to_version) + # Run update + self.service.kube_pre_application_update(self.context) # Verify that the upgrade state was updated updated_upgrade = self.dbapi.kube_upgrade_get_one() self.assertEqual(updated_upgrade.state, - kubernetes.KUBE_UPGRADE_STARTING_FAILED) + kubernetes.KUBE_PRE_UPDATING_APPS_FAILED) def test_kube_download_images(self): # Create controller-0 @@ -2271,6 +2271,63 @@ class ManagerTestCase(base.DbTestCase): mock_backup_kube_control_plane.assert_called() mock_remove_kube_control_plane_backup.assert_called() + def test_kube_post_application_update(self): + # Create application + dbutils.create_test_app( + name='stx-openstack', + app_version='1.0-19', + manifest_name='manifest', + manifest_file='stx-openstack.yaml', + status='applied', + active=True) + + # Create an upgrade + from_version = 'v1.42.1' + to_version = 'v1.43.1' + utils.create_test_kube_upgrade( + from_version=from_version, + to_version=to_version, + state=kubernetes.KUBE_UPGRADING_KUBELETS, + ) + + # Run post update + self.service.kube_post_application_update(self.context, to_version) + + # Verify that the upgrade state was updated + updated_upgrade = self.dbapi.kube_upgrade_get_one() + self.assertEqual(updated_upgrade.state, + kubernetes.KUBE_POST_UPDATED_APPS) + + def test_kube_post_application_update_app_not_compatible(self): + # Test post app update step when the installed application isn't + # compatible with the new kubernetes version + + # Create application + dbutils.create_test_app( + name='stx-openstack', + app_version='1.0-19', + manifest_name='manifest', + manifest_file='stx-openstack.yaml', + status='applied', + active=True) + + # Create an upgrade + from_version = 'v1.42.1' + to_version = 'v1.43.2' + utils.create_test_kube_upgrade( + from_version=from_version, + to_version=to_version, + state=kubernetes.KUBE_UPGRADING_KUBELETS, + ) + + # Run update + self.service.kube_post_application_update(self.context, to_version) + + # Verify that the upgrade state was updated + updated_upgrade = self.dbapi.kube_upgrade_get_one() + self.assertEqual(updated_upgrade.state, + kubernetes.KUBE_POST_UPDATING_APPS_FAILED) + @mock.patch('sysinv.conductor.manager.utils.HostHelper.get_active_controller') @mock.patch('sysinv.conductor.manager.' 'ConductorManager._config_apply_runtime_manifest')