starlingx
/
config
Code Issues Proposed changes

Check trident compatibility pre k8s upgrade 

There is an incompatibility between
k8s 1.19 and NetApp Trident driver
so we should disallow an upgrade
to k8s 1.19 if trident is version
20.04.

Story: 2008972
Task: 44071
Signed-off-by: Daniel Safta <daniel.safta@windriver.com>
Change-Id: I5af677c10312bae7c10101bf425e1db79c716670
This commit is contained in:
Daniel Safta 2021-11-16 13:27:27 +02:00
parent dd82f2b3ac
commit 6437316312
4 changed files with 44 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
sysinv/sysinv/sysinv/sysinv/common/health.py
View File

@ -4,6 +4,7 @@
# SPDX-License-Identifier: Apache-2.0
#
from eventlet.green import subprocess
import json
import os
from oslo_log import log
@ -237,6 +238,31 @@ class Health(object):
success = not fail_pod_list
return success, fail_pod_list
def _check_trident_compatibility(self):
"""Checks that the running Trident service has been
upgraded and is compatible with all possible k8s
upgrade versions."""
incompatible_version = '20.04'
try:
output = subprocess.check_output( # pylint: disable=not-callable
'export KUBECONFIG=/etc/kubernetes/admin.conf && \
tridentctl -n trident version -o json',
shell=True, stderr=subprocess.STDOUT).decode('utf-8')
if output:
json_output = json.loads(output)
if 'server' in json_output.keys():
if incompatible_version in json_output['server']['version']:
return False
else:
return True
except Exception as e:
LOG.info("Exception %s occured when trying to get trident version" % e)
return True
return True
def _check_kube_applications(self):
"""Checks that each kubernetes application is in a valid state"""
@ -495,6 +521,12 @@ class Health(object):
alarm_ignore_list=alarm_ignore_list)
success, apps_not_valid = self._check_kube_applications()
# can be removed in the release after stx6
if not self._check_trident_compatibility():
apps_not_valid.append("NetApp Trident Driver")
success = False
output += _(
'All kubernetes applications are in a valid state: [%s]\n') \
% (Health.SUCCESS_MSG if success else Health.FAIL_MSG)

3
sysinv/sysinv/sysinv/sysinv/tests/api/test_kube_rootca_update.py
View File

@ -178,6 +178,7 @@ class TestKubeRootCAUpdate(base.FunctionalTest):
class TestPostKubeRootCAUpdate(TestKubeRootCAUpdate,
dbbase.ProvisionedControllerHostTestCase):
@mock.patch('sysinv.common.health.Health._check_trident_compatibility', lambda x: True)
def test_create(self):
# Test creation of kubernetes rootca update
create_dict = dbutils.get_test_kube_rootca_update()
@ -197,6 +198,7 @@ class TestPostKubeRootCAUpdate(TestKubeRootCAUpdate,
self.assertEqual(host_updates[0]['target_rootca_cert'], None)
self.assertEqual(host_updates[0]['effective_rootca_cert'], 'current_cert_serial')
@mock.patch('sysinv.common.health.Health._check_trident_compatibility', lambda x: True)
def test_create_rootca_update_unhealthy_from_alarms(self):
""" Test creation of kube rootca update while there are alarms"""
# Test creation of kubernetes rootca update when system health check fails
@ -360,6 +362,7 @@ class TestKubeRootCAUpdateComplete(TestKubeRootCAUpdate,
super(TestKubeRootCAUpdateComplete, self).setUp()
self.url = '/kube_rootca_update'
@mock.patch('sysinv.common.health.Health._check_trident_compatibility', lambda x: True)
def test_update_complete_update_exists(self):
dbutils.create_test_kube_rootca_update(state=kubernetes.KUBE_ROOTCA_UPDATED_PODS_TRUSTNEWCA)
dbutils.create_test_kube_rootca_host_update(host_id=self.host.id,

7
sysinv/sysinv/sysinv/sysinv/tests/api/test_kube_upgrade.py
View File

@ -252,6 +252,7 @@ class TestListKubeUpgrade(TestKubeUpgrade):
class TestPostKubeUpgrade(TestKubeUpgrade,
dbbase.ProvisionedControllerHostTestCase):
@mock.patch('sysinv.common.health.Health._check_trident_compatibility', lambda x: True)
def test_create(self):
# Test creation of upgrade
create_dict = dbutils.post_get_test_kube_upgrade(to_version='v1.43.2')
@ -377,6 +378,7 @@ class TestPostKubeUpgrade(TestKubeUpgrade,
self.assertIn("incompatible with the new Kubernetes version v1.43.2",
result.json['error_message'])
@mock.patch('sysinv.common.health.Health._check_trident_compatibility', lambda x: True)
def test_create_system_unhealthy_from_alarms(self):
"""Test creation of a kube upgrade while there are alarms"""
# Test creation of upgrade when system health check fails
@ -395,6 +397,7 @@ class TestPostKubeUpgrade(TestKubeUpgrade,
self.assertIn("System is not in a valid state",
result.json['error_message'])
@mock.patch('sysinv.common.health.Health._check_trident_compatibility', lambda x: True)
def test_force_create_system_unhealthy_from_alarms(self):
# Test creation of upgrade when system health check fails but
# overridden with force
@ -414,6 +417,7 @@ class TestPostKubeUpgrade(TestKubeUpgrade,
self.assertEqual(result.json['state'],
kubernetes.KUBE_UPGRADE_STARTED)
@mock.patch('sysinv.common.health.Health._check_trident_compatibility', lambda x: True)
def test_force_create_system_unhealthy_from_mgmt_affecting_alarms(self):
""" Test kube upgrade create fails when mgmt affecting alarms found"""
@ -433,6 +437,7 @@ class TestPostKubeUpgrade(TestKubeUpgrade,
self.assertIn("System is not in a valid state",
result.json['error_message'])
@mock.patch('sysinv.common.health.Health._check_trident_compatibility', lambda x: True)
def test_create_system_can_ignore_alarms(self):
# Test creation of upgrade when system health check fails but
# overridden with force
@ -453,6 +458,7 @@ class TestPostKubeUpgrade(TestKubeUpgrade,
self.assertEqual(result.json['state'],
kubernetes.KUBE_UPGRADE_STARTED)
@mock.patch('sysinv.common.health.Health._check_trident_compatibility', lambda x: True)
def test_create_system_unhealthy_from_bad_apps(self):
""" Test kube upgrade create fails when invalid kube app found"""
@ -477,6 +483,7 @@ class TestPostKubeUpgrade(TestKubeUpgrade,
self.assertIn("Run system health-query-kube-upgrade for more details.",
result.json['error_message'])
@mock.patch('sysinv.common.health.Health._check_trident_compatibility', lambda x: True)
def test_create_no_patches_required(self):
# Test creation of upgrade when no applied patches are required
self.mock_patch_is_applied_result = False

2
sysinv/sysinv/sysinv/sysinv/tests/common/test_health.py
View File

@ -286,6 +286,7 @@ class TestHealth(dbbase.BaseHostTestCase):
assert "not ready: kube-controller-manager-controller-1" in output, \
"output: %s" % output
@mock.patch('sysinv.common.health.Health._check_trident_compatibility', lambda x: True)
def test_get_system_health_kube_upgrade(self):
# Create controller-0
config_uuid = str(uuid.uuid4())
@ -329,6 +330,7 @@ class TestHealth(dbbase.BaseHostTestCase):
self.context)
assert health_ok is True, "output: %s" % output
@mock.patch('sysinv.common.health.Health._check_trident_compatibility', lambda x: True)
def test_get_system_health_kube_upgrade_k8s_app_invalid_state(self):
# Create controller-0
config_uuid = str(uuid.uuid4())