diff --git a/sysinv/sysinv/sysinv/sysinv/common/health.py b/sysinv/sysinv/sysinv/sysinv/common/health.py index 4522e217e8..f669504ba8 100755 --- a/sysinv/sysinv/sysinv/sysinv/common/health.py +++ b/sysinv/sysinv/sysinv/sysinv/common/health.py @@ -4,6 +4,7 @@ # SPDX-License-Identifier: Apache-2.0 # from eventlet.green import subprocess +import json import os from oslo_log import log @@ -237,6 +238,31 @@ class Health(object): success = not fail_pod_list return success, fail_pod_list + def _check_trident_compatibility(self): + """Checks that the running Trident service has been + upgraded and is compatible with all possible k8s + upgrade versions.""" + + incompatible_version = '20.04' + + try: + output = subprocess.check_output( # pylint: disable=not-callable + 'export KUBECONFIG=/etc/kubernetes/admin.conf && \ + tridentctl -n trident version -o json', + shell=True, stderr=subprocess.STDOUT).decode('utf-8') + if output: + json_output = json.loads(output) + if 'server' in json_output.keys(): + if incompatible_version in json_output['server']['version']: + return False + else: + return True + except Exception as e: + LOG.info("Exception %s occured when trying to get trident version" % e) + return True + + return True + def _check_kube_applications(self): """Checks that each kubernetes application is in a valid state""" @@ -495,6 +521,12 @@ class Health(object): alarm_ignore_list=alarm_ignore_list) success, apps_not_valid = self._check_kube_applications() + + # can be removed in the release after stx6 + if not self._check_trident_compatibility(): + apps_not_valid.append("NetApp Trident Driver") + success = False + output += _( 'All kubernetes applications are in a valid state: [%s]\n') \ % (Health.SUCCESS_MSG if success else Health.FAIL_MSG) diff --git a/sysinv/sysinv/sysinv/sysinv/tests/api/test_kube_rootca_update.py b/sysinv/sysinv/sysinv/sysinv/tests/api/test_kube_rootca_update.py index 7b7ff0819b..1caffb1555 100644 --- a/sysinv/sysinv/sysinv/sysinv/tests/api/test_kube_rootca_update.py +++ b/sysinv/sysinv/sysinv/sysinv/tests/api/test_kube_rootca_update.py @@ -178,6 +178,7 @@ class TestKubeRootCAUpdate(base.FunctionalTest): class TestPostKubeRootCAUpdate(TestKubeRootCAUpdate, dbbase.ProvisionedControllerHostTestCase): + @mock.patch('sysinv.common.health.Health._check_trident_compatibility', lambda x: True) def test_create(self): # Test creation of kubernetes rootca update create_dict = dbutils.get_test_kube_rootca_update() @@ -197,6 +198,7 @@ class TestPostKubeRootCAUpdate(TestKubeRootCAUpdate, self.assertEqual(host_updates[0]['target_rootca_cert'], None) self.assertEqual(host_updates[0]['effective_rootca_cert'], 'current_cert_serial') + @mock.patch('sysinv.common.health.Health._check_trident_compatibility', lambda x: True) def test_create_rootca_update_unhealthy_from_alarms(self): """ Test creation of kube rootca update while there are alarms""" # Test creation of kubernetes rootca update when system health check fails @@ -360,6 +362,7 @@ class TestKubeRootCAUpdateComplete(TestKubeRootCAUpdate, super(TestKubeRootCAUpdateComplete, self).setUp() self.url = '/kube_rootca_update' + @mock.patch('sysinv.common.health.Health._check_trident_compatibility', lambda x: True) def test_update_complete_update_exists(self): dbutils.create_test_kube_rootca_update(state=kubernetes.KUBE_ROOTCA_UPDATED_PODS_TRUSTNEWCA) dbutils.create_test_kube_rootca_host_update(host_id=self.host.id, diff --git a/sysinv/sysinv/sysinv/sysinv/tests/api/test_kube_upgrade.py b/sysinv/sysinv/sysinv/sysinv/tests/api/test_kube_upgrade.py index 47fc36d8c9..eedfd790e6 100644 --- a/sysinv/sysinv/sysinv/sysinv/tests/api/test_kube_upgrade.py +++ b/sysinv/sysinv/sysinv/sysinv/tests/api/test_kube_upgrade.py @@ -252,6 +252,7 @@ class TestListKubeUpgrade(TestKubeUpgrade): class TestPostKubeUpgrade(TestKubeUpgrade, dbbase.ProvisionedControllerHostTestCase): + @mock.patch('sysinv.common.health.Health._check_trident_compatibility', lambda x: True) def test_create(self): # Test creation of upgrade create_dict = dbutils.post_get_test_kube_upgrade(to_version='v1.43.2') @@ -377,6 +378,7 @@ class TestPostKubeUpgrade(TestKubeUpgrade, self.assertIn("incompatible with the new Kubernetes version v1.43.2", result.json['error_message']) + @mock.patch('sysinv.common.health.Health._check_trident_compatibility', lambda x: True) def test_create_system_unhealthy_from_alarms(self): """Test creation of a kube upgrade while there are alarms""" # Test creation of upgrade when system health check fails @@ -395,6 +397,7 @@ class TestPostKubeUpgrade(TestKubeUpgrade, self.assertIn("System is not in a valid state", result.json['error_message']) + @mock.patch('sysinv.common.health.Health._check_trident_compatibility', lambda x: True) def test_force_create_system_unhealthy_from_alarms(self): # Test creation of upgrade when system health check fails but # overridden with force @@ -414,6 +417,7 @@ class TestPostKubeUpgrade(TestKubeUpgrade, self.assertEqual(result.json['state'], kubernetes.KUBE_UPGRADE_STARTED) + @mock.patch('sysinv.common.health.Health._check_trident_compatibility', lambda x: True) def test_force_create_system_unhealthy_from_mgmt_affecting_alarms(self): """ Test kube upgrade create fails when mgmt affecting alarms found""" @@ -433,6 +437,7 @@ class TestPostKubeUpgrade(TestKubeUpgrade, self.assertIn("System is not in a valid state", result.json['error_message']) + @mock.patch('sysinv.common.health.Health._check_trident_compatibility', lambda x: True) def test_create_system_can_ignore_alarms(self): # Test creation of upgrade when system health check fails but # overridden with force @@ -453,6 +458,7 @@ class TestPostKubeUpgrade(TestKubeUpgrade, self.assertEqual(result.json['state'], kubernetes.KUBE_UPGRADE_STARTED) + @mock.patch('sysinv.common.health.Health._check_trident_compatibility', lambda x: True) def test_create_system_unhealthy_from_bad_apps(self): """ Test kube upgrade create fails when invalid kube app found""" @@ -477,6 +483,7 @@ class TestPostKubeUpgrade(TestKubeUpgrade, self.assertIn("Run system health-query-kube-upgrade for more details.", result.json['error_message']) + @mock.patch('sysinv.common.health.Health._check_trident_compatibility', lambda x: True) def test_create_no_patches_required(self): # Test creation of upgrade when no applied patches are required self.mock_patch_is_applied_result = False diff --git a/sysinv/sysinv/sysinv/sysinv/tests/common/test_health.py b/sysinv/sysinv/sysinv/sysinv/tests/common/test_health.py index 10f478fada..4fce6a0618 100644 --- a/sysinv/sysinv/sysinv/sysinv/tests/common/test_health.py +++ b/sysinv/sysinv/sysinv/sysinv/tests/common/test_health.py @@ -286,6 +286,7 @@ class TestHealth(dbbase.BaseHostTestCase): assert "not ready: kube-controller-manager-controller-1" in output, \ "output: %s" % output + @mock.patch('sysinv.common.health.Health._check_trident_compatibility', lambda x: True) def test_get_system_health_kube_upgrade(self): # Create controller-0 config_uuid = str(uuid.uuid4()) @@ -329,6 +330,7 @@ class TestHealth(dbbase.BaseHostTestCase): self.context) assert health_ok is True, "output: %s" % output + @mock.patch('sysinv.common.health.Health._check_trident_compatibility', lambda x: True) def test_get_system_health_kube_upgrade_k8s_app_invalid_state(self): # Create controller-0 config_uuid = str(uuid.uuid4())