starlingx
/
config
Code Issues Proposed changes

Merge "Kube rootca update abort - API"

This commit is contained in:
Zuul 2021-08-30 19:19:32 +00:00 committed by Gerrit Code Review
parent 0273db6417 f9d3c9112c
commit 1c3292da14
3 changed files with 152 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

121
sysinv/sysinv/sysinv/sysinv/api/controllers/v1/kube_rootca_update.py
View File

@ -38,6 +38,13 @@ LOG = log.getLogger(__name__)
LOCK_KUBE_ROOTCA_CONTROLLER = 'KubeRootCAController'
class KubeRootCAUpdatePatchType(types.JsonPatchType):
@staticmethod
def mandatory_attrs():
return ['/state']
class KubeRootCAGenerateController(rest.RestController):
""" API representation of a Kubernetes Generate Root CA Certificate"""
@ -372,9 +379,9 @@ class KubeRootCAHostUpdateListController(rest.RestController):
class KubeRootCAUpdateController(rest.RestController):
"""REST controller for kubernetes rootCA updates."""
# Controller for /kube_rootca_update/upload, upload new root CA
# Controller for /kube_rootca_update/upload_cert, upload new root CA
# certificate.
upload = KubeRootCAUploadController()
upload_cert = KubeRootCAUploadController()
# Controller for /kube_rootca_update/generate_cert, generates a new root CA
generate_cert = KubeRootCAGenerateController()
# Controller for /kube_rootca_update/pods, update pods certificates.
@ -387,6 +394,14 @@ class KubeRootCAUpdateController(rest.RestController):
self.alarm_instance_id = "%s=%s" % (fm_constants.FM_ENTITY_TYPE_HOST,
constants.CONTROLLER_HOSTNAME)
def _get_updates(self, patch):
"""Retrieve the updated attributes from the patch request."""
updates = {}
for p in patch:
attribute = p['path'] if p['path'][0] != '/' else p['path'][1:]
updates[attribute] = p['value']
return updates
def _check_cluster_health(self, command_name, alarm_ignore_list=None, force=False):
healthy, output = pecan.request.rpcapi.get_system_health(
pecan.request.context,
@ -434,20 +449,24 @@ class KubeRootCAUpdateController(rest.RestController):
secret_list=secret_list)
@cutils.synchronized(LOCK_KUBE_ROOTCA_CONTROLLER)
@wsme_pecan.wsexpose(KubeRootCAUpdate, body=six.text_type)
def post(self, body):
@wsme_pecan.wsexpose(KubeRootCAUpdate, wtypes.text, body=six.text_type)
def post(self, force, body):
"""Create a new Kubernetes RootCA Update and start update."""
force = body.get('force', False) is True
alarm_ignore_list = body.get('alarm_ignore_list')
force = force == 'True'
alarm_ignore_list = body.get('alarm_ignore_list', [])
alarm_ignore_list.append(fm_constants.FM_ALARM_ID_KUBE_ROOTCA_UPDATE_ABORTED)
try:
pecan.request.dbapi.kube_rootca_update_get_one()
update = pecan.request.dbapi.kube_rootca_update_get_one()
except exception.NotFound:
pass
else:
raise wsme.exc.ClientSideError(_(
"A kubernetes rootca update is already in progress"))
if update.state == kubernetes.KUBE_ROOTCA_UPDATE_ABORTED:
pecan.request.dbapi.kube_rootca_update_destroy(update.id)
else:
raise wsme.exc.ClientSideError((
"A kubernetes rootca update is already in progress"))
# There must not be a platform upgrade in progress
try:
@ -497,6 +516,13 @@ class KubeRootCAUpdateController(rest.RestController):
pecan.request.dbapi.kube_rootca_host_update_create(host.id,
{'effective_rootca_cert': from_rootca_cert})
# clear update aborted alarm if there is one
if self.fm_api.get_fault(fm_constants.FM_ALARM_ID_KUBE_ROOTCA_UPDATE_ABORTED,
self.alarm_instance_id):
self.fm_api.clear_fault(fm_constants.FM_ALARM_ID_KUBE_ROOTCA_UPDATE_ABORTED,
self.alarm_instance_id)
# raise update in progess alarm
fault = fm_api.Fault(
alarm_id=fm_constants.FM_ALARM_ID_KUBE_ROOTCA_UPDATE_IN_PROGRESS,
alarm_state=fm_constants.FM_ALARM_STATE_SET,
@ -511,6 +537,7 @@ class KubeRootCAUpdateController(rest.RestController):
proposed_repair_action="Wait for kubernetes rootca procedure to complete",
service_affecting=False)
self.fm_api.set_fault(fault)
LOG.info("Started kubernetes rootca update")
return KubeRootCAUpdate.convert_with_links(new_update)
@ -528,20 +555,34 @@ class KubeRootCAUpdateController(rest.RestController):
rpc_kube_rootca_update = pecan.request.dbapi.kube_rootca_update_get_list()
return KubeRootCAUpdateCollection.convert_with_links(rpc_kube_rootca_update)
@wsme_pecan.wsexpose(KubeRootCAUpdate, wtypes.text)
def patch(self, force=None):
@cutils.synchronized(LOCK_KUBE_ROOTCA_CONTROLLER)
@wsme.validate(wtypes.text, [KubeRootCAUpdatePatchType])
@wsme_pecan.wsexpose(KubeRootCAUpdate, wtypes.text, body=[KubeRootCAUpdatePatchType])
def patch(self, force, patch):
"""Completes the kubernetes rootca, clearing both tables and alarm"""
force = force == 'True'
updates = self._get_updates(patch)
update_state = updates['state']
if update_state not in [kubernetes.KUBE_ROOTCA_UPDATE_COMPLETED,
kubernetes.KUBE_ROOTCA_UPDATE_ABORTED]:
raise wsme.exc.ClientSideError(_(
"Invalid state %s supplied" % update_state))
# Check if there is an update in progress and the current state
try:
rpc_kube_rootca_update = pecan.request.dbapi.kube_rootca_update_get_one()
if rpc_kube_rootca_update.state != kubernetes.KUBE_ROOTCA_UPDATED_PODS_TRUSTNEWCA:
if update_state == kubernetes.KUBE_ROOTCA_UPDATE_COMPLETED \
and rpc_kube_rootca_update.state != kubernetes.KUBE_ROOTCA_UPDATED_PODS_TRUSTNEWCA:
raise wsme.exc.ClientSideError(_(
"kube-rootca-update-complete rejected: Expect to find cluster update state %s, "
"not allowed when cluster update state is %s."
% (kubernetes.KUBE_ROOTCA_UPDATED_PODS_TRUSTNEWCA, rpc_kube_rootca_update.state)))
elif update_state == kubernetes.KUBE_ROOTCA_UPDATE_ABORTED \
and rpc_kube_rootca_update.state == kubernetes.KUBE_ROOTCA_UPDATE_ABORTED:
raise wsme.exc.ClientSideError(_(
"kube-rootca-update-complete rejected: The update has already been aborted."))
except exception.NotFound:
raise wsme.exc.ClientSideError(_(
"kube-rootca-update-complete rejected: No kubernetes root CA update in progress."))
@ -558,22 +599,54 @@ class KubeRootCAUpdateController(rest.RestController):
hostnames = [host.hostname for host in rpc_host_update_list]
self._clear_kubernetes_resources(hostnames)
pecan.request.dbapi.kube_rootca_update_destroy(rpc_kube_rootca_update.id)
# cleanup kube_rootca_host_update table
pecan.request.dbapi.kube_rootca_host_update_destroy_all()
app_alarms = self.fm_api.get_faults(self.alarm_instance_id)
self.fm_api.clear_fault(app_alarms[0].alarm_id, app_alarms[0].entity_instance_id)
# if update is in the list of states, abort will be the same as complete
if update_state == kubernetes.KUBE_ROOTCA_UPDATE_ABORTED \
and rpc_kube_rootca_update.state in [kubernetes.KUBE_ROOTCA_UPDATED_PODS_TRUSTNEWCA]:
update_state = kubernetes.KUBE_ROOTCA_UPDATE_COMPLETED
rpc_kube_rootca_update.state = kubernetes.KUBE_ROOTCA_UPDATE_COMPLETED
rpc_kube_rootca_update.updated_at = datetime.datetime.utcnow().isoformat()
values = dict()
values['state'] = update_state
update = \
pecan.request.dbapi.kube_rootca_update_update(rpc_kube_rootca_update.id, values)
# If applicable, notify dcmanager that the update is completed
system = pecan.request.dbapi.isystem_get_one()
role = system.get('distributed_cloud_role')
if role == constants.DISTRIBUTED_CLOUD_ROLE_SYSTEMCONTROLLER:
dc_api.notify_dcmanager_kube_rootca_update_completed()
# cleanup kube_rootca_update table for completion
if update_state == kubernetes.KUBE_ROOTCA_UPDATE_COMPLETED:
pecan.request.dbapi.kube_rootca_update_destroy(rpc_kube_rootca_update.id)
return KubeRootCAUpdate.convert_with_links(rpc_kube_rootca_update)
# If applicable, notify dcmanager that the update is completed
system = pecan.request.dbapi.isystem_get_one()
role = system.get('distributed_cloud_role')
if role == constants.DISTRIBUTED_CLOUD_ROLE_SYSTEMCONTROLLER:
dc_api.notify_dcmanager_kube_rootca_update_completed()
# clear update in progess alarm
if self.fm_api.get_fault(fm_constants.FM_ALARM_ID_KUBE_ROOTCA_UPDATE_IN_PROGRESS,
self.alarm_instance_id):
self.fm_api.clear_fault(fm_constants.FM_ALARM_ID_KUBE_ROOTCA_UPDATE_IN_PROGRESS,
self.alarm_instance_id)
# raise update aborted alarm if this is an abort
if update_state == kubernetes.KUBE_ROOTCA_UPDATE_ABORTED:
fault = fm_api.Fault(
alarm_id=fm_constants.FM_ALARM_ID_KUBE_ROOTCA_UPDATE_ABORTED,
alarm_state=fm_constants.FM_ALARM_STATE_SET,
entity_type_id=fm_constants.FM_ENTITY_TYPE_HOST,
entity_instance_id=self.alarm_instance_id,
severity=fm_constants.FM_ALARM_SEVERITY_MINOR,
reason_text="Kubernetes root CA update aborted, certificates may not be fully updated.",
# environmental
alarm_type=fm_constants.FM_ALARM_TYPE_5,
# unspecified-reason
probable_cause=fm_constants.ALARM_PROBABLE_CAUSE_65,
proposed_repair_action="Fully update certificates by a new root CA update.",
service_affecting=False)
self.fm_api.set_fault(fault)
LOG.info("Kubernetes rootca update aborted")
return KubeRootCAUpdate.convert_with_links(update)
class KubeRootCAHostUpdateController(rest.RestController):

1
sysinv/sysinv/sysinv/sysinv/common/kubernetes.py
View File

@ -100,6 +100,7 @@ KUBE_ROOTCA_UPDATING_PODS_TRUSTNEWCA = 'updating-pods-trustNewCA'
KUBE_ROOTCA_UPDATED_PODS_TRUSTNEWCA = 'updated-pods-trustNewCA'
KUBE_ROOTCA_UPDATING_PODS_TRUSTNEWCA_FAILED = 'updating-pods-trustNewCA-failed'
KUBE_ROOTCA_UPDATE_COMPLETED = 'update-completed'
KUBE_ROOTCA_UPDATE_ABORTED = 'update-aborted'
# Kubernetes rootca host update states
KUBE_ROOTCA_UPDATING_HOST_TRUSTBOTHCAS = 'updating-host-trustBothCAs'

67
sysinv/sysinv/sysinv/sysinv/tests/api/test_kube_rootca_update.py
View File

@ -181,7 +181,7 @@ class TestPostKubeRootCAUpdate(TestKubeRootCAUpdate,
def test_create(self):
# Test creation of kubernetes rootca update
create_dict = dbutils.get_test_kube_rootca_update()
result = self.post_json('/kube_rootca_update', create_dict,
result = self.post_json('/kube_rootca_update?force=False', create_dict,
headers=self.headers)
# Verify that the kubernetes rootca update has the expected attributes
@ -206,7 +206,7 @@ class TestPostKubeRootCAUpdate(TestKubeRootCAUpdate,
# Test creation of kubernetes rootca update
create_dict = dbutils.get_test_kube_rootca_update()
result = self.post_json('/kube_rootca_update', create_dict,
result = self.post_json('/kube_rootca_update?force=False', create_dict,
headers=self.headers,
expect_errors=True)
@ -220,7 +220,7 @@ class TestPostKubeRootCAUpdate(TestKubeRootCAUpdate,
# Test creation of rootca update when a kubernetes rootca update already exists
dbutils.create_test_kube_rootca_update()
create_dict = dbutils.post_get_test_kube_rootca_update(state=kubernetes.KUBE_ROOTCA_UPDATE_STARTED)
result = self.post_json('/kube_rootca_update', create_dict,
result = self.post_json('/kube_rootca_update?force=False', create_dict,
headers=self.headers,
expect_errors=True)
@ -238,7 +238,7 @@ class TestPostKubeRootCAUpdate(TestKubeRootCAUpdate,
state=kubernetes.KUBE_UPGRADING_FIRST_MASTER,
)
create_dict = dbutils.post_get_test_kube_rootca_update()
result = self.post_json('/kube_rootca_update', create_dict,
result = self.post_json('/kube_rootca_update?force=False', create_dict,
headers=self.headers,
expect_errors=True)
@ -257,7 +257,7 @@ class TestPostKubeRootCAUpdate(TestKubeRootCAUpdate,
dbutils.create_test_upgrade()
create_dict = dbutils.post_get_test_kube_rootca_update()
result = self.post_json('/kube_rootca_update', create_dict,
result = self.post_json('/kube_rootca_update?force=False', create_dict,
headers=self.headers,
expect_errors=True)
@ -368,7 +368,10 @@ class TestKubeRootCAUpdateComplete(TestKubeRootCAUpdate,
state=kubernetes.KUBE_ROOTCA_UPDATED_PODS_TRUSTNEWCA)
self.fake_fm_client.alarm.list.return_value = [FAKE_ROOTCA_UPDATE_ALARM]
result = self.patch_json(self.url, {})
patch = [{'op': 'replace',
'path': '/state',
'value': 'update-completed'}]
result = self.patch_json(self.url + '?force=False', patch)
result = result.json
self.assertEqual(result['state'], kubernetes.KUBE_ROOTCA_UPDATE_COMPLETED)
@ -385,15 +388,17 @@ class TestKubeRootCAUpdateComplete(TestKubeRootCAUpdate,
host_updates = self.dbapi.kube_rootca_host_update_get_list()
self.assertEqual(len(host_updates), 0)
def test_update_complete_force_update_exists(self):
def test_update_complete_force_pods_trustnewca_update_exists(self):
dbutils.create_test_kube_rootca_update(state=kubernetes.KUBE_ROOTCA_UPDATED_PODS_TRUSTNEWCA)
dbutils.create_test_kube_rootca_host_update(host_id=self.host.id,
state=kubernetes.KUBE_ROOTCA_UPDATED_PODS_TRUSTNEWCA)
dbutils.create_test_kube_rootca_host_update(host_id=self.host2.id,
state=kubernetes.KUBE_ROOTCA_UPDATED_PODS_TRUSTNEWCA)
self.fake_fm_client.alarm.list.return_value = [FAKE_ROOTCA_UPDATE_ALARM, FakeAlarm('900.401', "False")]
result = self.patch_json(self.url + '?force=True', {})
patch = [{'op': 'replace',
'path': '/state',
'value': 'update-aborted'}]
result = self.patch_json(self.url + '?force=False', patch)
result = result.json
self.assertEqual(result['state'], kubernetes.KUBE_ROOTCA_UPDATE_COMPLETED)
@ -410,6 +415,32 @@ class TestKubeRootCAUpdateComplete(TestKubeRootCAUpdate,
host_updates = self.dbapi.kube_rootca_host_update_get_list()
self.assertEqual(len(host_updates), 0)
def test_update_complete_force_hosts_updatecerts_trustnewca_update_exists(self):
dbutils.create_test_kube_rootca_update(state=kubernetes.KUBE_ROOTCA_UPDATING_HOST_TRUSTNEWCA)
dbutils.create_test_kube_rootca_host_update(host_id=self.host.id,
state=kubernetes.KUBE_ROOTCA_UPDATED_HOST_UPDATECERTS)
dbutils.create_test_kube_rootca_host_update(host_id=self.host2.id,
state=kubernetes.KUBE_ROOTCA_UPDATED_HOST_TRUSTNEWCA)
patch = [{'op': 'replace',
'path': '/state',
'value': 'update-aborted'}]
result = self.patch_json(self.url + '?force=False', patch)
result = result.json
self.assertEqual(result['state'], kubernetes.KUBE_ROOTCA_UPDATE_ABORTED)
self.assertEqual(result['from_rootca_cert'], 'oldCertSerial')
self.assertEqual(result['to_rootca_cert'], 'newCertSerial')
# Verify that the kube_rootca_update table is update with "update-aborted"
update_entry = self.dbapi.kube_rootca_update_get_one()
self.assertNotEqual(update_entry, None)
self.assertEqual(update_entry.state, kubernetes.KUBE_ROOTCA_UPDATE_ABORTED)
# Verify that the kube_rootca_host_update table is cleaned
host_list = self.dbapi.kube_rootca_host_update_get_list()
self.assertEqual(len(host_list), 0)
def test_update_complete_invalid_health(self):
dbutils.create_test_kube_rootca_update(state=kubernetes.KUBE_ROOTCA_UPDATED_PODS_TRUSTNEWCA)
dbutils.create_test_kube_rootca_host_update(host_id=self.host.id,
@ -418,7 +449,11 @@ class TestKubeRootCAUpdateComplete(TestKubeRootCAUpdate,
state=kubernetes.KUBE_ROOTCA_UPDATED_PODS_TRUSTNEWCA)
self.fake_fm_client.alarm.list.return_value = [FAKE_MGMT_ALARM]
result = self.patch_json(self.url, {}, expect_errors=True)
patch = [{'op': 'replace',
'path': '/state',
'value': 'update-completed'}]
result = self.patch_json(self.url + '?force=False', patch, expect_errors=True)
self.assertEqual(result.status_int, http_client.BAD_REQUEST)
self.assertIn("System is not healthy. Run system health-query for more details.",
@ -430,7 +465,10 @@ class TestKubeRootCAUpdateComplete(TestKubeRootCAUpdate,
self.assertEqual(len(host_update_list), 2)
def test_update_complete_no_update(self):
result = self.patch_json(self.url, {}, expect_errors=True)
patch = [{'op': 'replace',
'path': '/state',
'value': 'update-completed'}]
result = self.patch_json(self.url + '?force=False', patch, expect_errors=True)
self.assertEqual(result.status_int, http_client.BAD_REQUEST)
self.assertIn("kube-rootca-update-complete rejected: No kubernetes root CA update in progress.",
@ -439,7 +477,10 @@ class TestKubeRootCAUpdateComplete(TestKubeRootCAUpdate,
def test_update_complete_invalid_phase(self):
dbutils.create_test_kube_rootca_update()
result = self.patch_json(self.url, {}, expect_errors=True)
patch = [{'op': 'replace',
'path': '/state',
'value': 'update-completed'}]
result = self.patch_json(self.url + '?force=False', patch, expect_errors=True)
self.assertEqual(result.status_int, http_client.BAD_REQUEST)
self.assertIn("kube-rootca-update-complete rejected: Expect to find cluster update"
@ -470,7 +511,7 @@ class TestKubeRootCAUpload(TestKubeRootCAUpdate,
setup_config_certificate(fake_save_rootca_return)
files = [('file', certfile)]
response = self.post_with_files('%s/%s' % ('/kube_rootca_update', 'upload'),
response = self.post_with_files('%s/%s' % ('/kube_rootca_update', 'upload_cert'),
{},
upload_files=files,
headers=self.headers,