From 26bcccd07deb5fca39a98b0b4d0e501598060a18 Mon Sep 17 00:00:00 2001 From: Andy Ning Date: Wed, 2 Jun 2021 14:32:59 -0400 Subject: [PATCH] kubernetes rootca host update trustNewCA Added host update phase trustNewCA API implementation, including - changes to conductor manager - precheck for validation of the procedure phase - tox unit tests for the API. - call to puppet manifest application via puppet plugin Story: 2008675 Task: 42699 Depends-on: https://review.opendev.org/c/starlingx/stx-puppet/+/798308 Change-Id: I914828454f3601de6607e5ace8b415d124c95111 Signed-off-by: Andy Ning --- .../api/controllers/v1/kube_rootca_update.py | 62 +++++++ .../sysinv/sysinv/sysinv/conductor/manager.py | 11 +- .../tests/api/test_kube_rootca_update.py | 170 ++++++++++++++++++ 3 files changed, 240 insertions(+), 3 deletions(-) diff --git a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/kube_rootca_update.py b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/kube_rootca_update.py index 5cc87c42c8..011f46b738 100644 --- a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/kube_rootca_update.py +++ b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/kube_rootca_update.py @@ -621,6 +621,64 @@ class KubeRootCAHostUpdateController(rest.RestController): % (cluster_update.state, kubernetes.KUBE_ROOTCA_UPDATING_HOST_TRUSTBOTHCAS))) + def _precheck_trustnewca(self, cluster_update, ihost): + + # Get all the host update state + host_updates = pecan.request.dbapi.kube_rootca_host_update_get_list() + if len(host_updates) == 0: + raise wsme.exc.ClientSideError(_( + "kube-rootca-host-update rejected: host update " + "not started yet")) + + if cluster_update.state in [ + kubernetes.KUBE_ROOTCA_UPDATED_HOST_TRUSTNEWCA, + kubernetes.KUBE_ROOTCA_UPDATING_HOST_TRUSTNEWCA, + kubernetes.KUBE_ROOTCA_UPDATING_HOST_TRUSTNEWCA_FAILED]: + if cluster_update.state == \ + kubernetes.KUBE_ROOTCA_UPDATED_HOST_TRUSTNEWCA: + # trustNewCA phase completed + raise wsme.exc.ClientSideError(_( + "kube-rootca-host-update rejected: update already " + "completed on cluster")) + for host_update in host_updates: + if host_update.state == kubernetes.\ + KUBE_ROOTCA_UPDATING_HOST_TRUSTNEWCA_FAILED: + # other host is on FAILED state + if host_update.host_id != ihost.id: + host_name = pecan.request.dbapi.ihost_get( + host_update.host_id).hostname + raise wsme.exc.ClientSideError(_( + "kube-rootca-host-update rejected: update " + "failed on host %s" % host_name)) + elif host_update.state == \ + kubernetes.KUBE_ROOTCA_UPDATING_HOST_TRUSTNEWCA: + # procedure already in progress in some host + host_name = pecan.request.dbapi.ihost_get( + host_update.host_id).hostname + raise wsme.exc.ClientSideError(_( + "kube-rootca-host-update rejected: update in " + "progress on host %s" % host_name)) + elif host_update.state == \ + kubernetes.KUBE_ROOTCA_UPDATED_HOST_TRUSTNEWCA: + if host_update.host_id == ihost.id: + raise wsme.exc.ClientSideError(_( + "kube-rootca-host-update rejected: update " + "already completed on host %s" + % ihost.hostname)) + + # If this is the first host to be update on this phase we need to ensure + # that cluster_update state is what we expect, which is updateCerts + # phase has completed successfully on all hosts. + else: + if cluster_update.state != \ + kubernetes.KUBE_ROOTCA_UPDATED_HOST_UPDATECERTS: + raise wsme.exc.ClientSideError(_( + "kube-rootca-host-update rejected: not " + "allowed when cluster update is in state: %s. " + "(only allowed when in state: %s)" + % (cluster_update.state, + kubernetes.KUBE_ROOTCA_UPDATED_HOST_UPDATECERTS))) + @cutils.synchronized(LOCK_KUBE_ROOTCA_CONTROLLER) @wsme_pecan.wsexpose(KubeRootCAHostUpdate, types.uuid, body=six.text_type) def post(self, host_uuid, body): @@ -671,6 +729,10 @@ class KubeRootCAHostUpdateController(rest.RestController): # kube root CA update on host phase updateCerts self._precheck_updatecerts(update, ihost) update_state = kubernetes.KUBE_ROOTCA_UPDATING_HOST_UPDATECERTS + elif phase == constants.KUBE_CERT_UPDATE_TRUSTNEWCA: + # kube root CA update on host phase trustNewCA + self._precheck_trustnewca(update, ihost) + update_state = kubernetes.KUBE_ROOTCA_UPDATING_HOST_TRUSTNEWCA else: raise wsme.exc.ClientSideError(_( "kube-rootca-host-update rejected: not supported phase.")) diff --git a/sysinv/sysinv/sysinv/sysinv/conductor/manager.py b/sysinv/sysinv/sysinv/sysinv/conductor/manager.py index 1bc948d8fd..a8562999f8 100644 --- a/sysinv/sysinv/sysinv/sysinv/conductor/manager.py +++ b/sysinv/sysinv/sysinv/sysinv/conductor/manager.py @@ -8345,21 +8345,26 @@ class ConductorManager(service.PeriodicService): LOG.info("Kube root CA update phase '%s' succeeded on host: %s" % (reported_cfg, host_uuid)) + values = {} + h_update = self.dbapi.kube_rootca_host_update_get_by_host(host_uuid) + if reported_cfg == puppet_common.REPORT_KUBE_CERT_UPDATE_TRUSTBOTHCAS: state = kubernetes.KUBE_ROOTCA_UPDATED_HOST_TRUSTBOTHCAS + values.update({'state': state}) elif reported_cfg == puppet_common.REPORT_KUBE_CERT_UPDATE_UPDATECERTS: state = kubernetes.KUBE_ROOTCA_UPDATED_HOST_UPDATECERTS + values.update({'state': state}) elif reported_cfg == puppet_common.REPORT_KUBE_CERT_UPDATE_TRUSTNEWCA: state = kubernetes.KUBE_ROOTCA_UPDATED_HOST_TRUSTNEWCA + values.update({'state': state, + 'effective_rootca_cert': h_update.target_rootca_cert}) else: LOG.info("Not supported reported_cfg: %s" % reported_cfg) raise exception.SysinvException(_( "Not supported reported_cfg: %s" % reported_cfg)) # Update host 'update state' - h_update = self.dbapi.kube_rootca_host_update_get_by_host(host_uuid) - self.dbapi.kube_rootca_host_update_update(h_update.id, - {'state': state}) + self.dbapi.kube_rootca_host_update_update(h_update.id, values) # Update cluster 'update state' hosts = self.dbapi.ihost_get_list() diff --git a/sysinv/sysinv/sysinv/sysinv/tests/api/test_kube_rootca_update.py b/sysinv/sysinv/sysinv/sysinv/tests/api/test_kube_rootca_update.py index 75383215c5..0fd60ca35b 100644 --- a/sysinv/sysinv/sysinv/sysinv/tests/api/test_kube_rootca_update.py +++ b/sysinv/sysinv/sysinv/sysinv/tests/api/test_kube_rootca_update.py @@ -965,3 +965,173 @@ class TestKubeRootCAHostUpdateUpdateCerts(TestKubeRootCAHostUpdate, self.assertIn("kube-rootca-host-update rejected: update failed " "on host %s" % self.host2.hostname, result.json['error_message']) + + +class TestKubeRootCAHostUpdateTrustNewCA(TestKubeRootCAHostUpdate, + dbbase.ProvisionedAIODuplexSystemTestCase): + def setUp(self): + super(TestKubeRootCAHostUpdateTrustNewCA, self).setUp() + # Set host root CA update phase + self.set_phase(constants.KUBE_CERT_UPDATE_TRUSTNEWCA) + + def test_trustnewca_host_update(self): + # Test kubernetes rootca host update phase trustNewCA success case + create_dict = {'phase': self.phase} + + # overall update is updateCerts phase completed + dbutils.create_test_kube_rootca_update( + state=kubernetes.KUBE_ROOTCA_UPDATED_HOST_UPDATECERTS) + + # host update is updateCerts phase completed on this host + dbutils.create_test_kube_rootca_host_update(host_id=self.host.id, + state=kubernetes.KUBE_ROOTCA_UPDATED_HOST_UPDATECERTS) + + result = self.post_json(self.post_url, create_dict, + headers=self.headers, + expect_errors=True) + + # Verify that the rootca host update has the expected attributes + self.assertEqual(result.json['state'], + kubernetes.KUBE_ROOTCA_UPDATING_HOST_TRUSTNEWCA) + self.assertEqual(result.json['target_rootca_cert'], 'newCertSerial') + self.assertEqual(result.json['effective_rootca_cert'], 'oldCertSerial') + + # Verify that the overall rootca update has the expected attributes + result = dbutils.get_kube_rootca_update() + self.assertEqual(result.state, + kubernetes.KUBE_ROOTCA_UPDATING_HOST_TRUSTNEWCA) + + def test_trustnewca_host_update_failed_retry(self): + # Test kubernetes rootca host update phase trustNewCA failed and retry + create_dict = {'phase': self.phase} + + # overall update is trustNewCA phase in progress + dbutils.create_test_kube_rootca_update( + state=kubernetes.KUBE_ROOTCA_UPDATING_HOST_TRUSTNEWCA) + + # host update is trustNewCA phase failed on this host + dbutils.create_test_kube_rootca_host_update(host_id=self.host.id, + state=kubernetes.KUBE_ROOTCA_UPDATING_HOST_TRUSTNEWCA_FAILED) + + result = self.post_json(self.post_url, create_dict, + headers=self.headers, + expect_errors=True) + + # Verify that the rootca host update has the expected attributes + self.assertEqual(result.json['state'], + kubernetes.KUBE_ROOTCA_UPDATING_HOST_TRUSTNEWCA) + self.assertEqual(result.json['target_rootca_cert'], 'newCertSerial') + self.assertEqual(result.json['effective_rootca_cert'], 'oldCertSerial') + + # Verify that the overall rootca update has the expected attributes + result = dbutils.get_kube_rootca_update() + self.assertEqual(result.state, + kubernetes.KUBE_ROOTCA_UPDATING_HOST_TRUSTNEWCA) + + def test_trustnewca_host_update_in_progress(self): + create_dict = {'phase': self.phase} + + # overall update is trustNewCA phase in progress + dbutils.create_test_kube_rootca_update( + state=kubernetes.KUBE_ROOTCA_UPDATING_HOST_TRUSTNEWCA) + + # host update is trustNewCA phase in progress on this host + dbutils.create_test_kube_rootca_host_update(host_id=self.host.id, + state=kubernetes.KUBE_ROOTCA_UPDATING_HOST_TRUSTNEWCA) + + result = self.post_json(self.post_url, create_dict, + headers=self.headers, + expect_errors=True) + + self.assertEqual(http_client.BAD_REQUEST, result.status_int) + self.assertIn("kube-rootca-host-update rejected: update in progress " + "on host %s" % self.host.hostname, + result.json['error_message']) + + def test_trustnewca_host_update_failed_cluster_in_advanced_state(self): + create_dict = {'phase': self.phase} + + # overall update is pods trustnewca phase completed + dbutils.create_test_kube_rootca_update( + state=kubernetes.KUBE_ROOTCA_UPDATING_PODS_TRUSTNEWCA) + + dbutils.create_test_kube_rootca_host_update(host_id=self.host.id, + state=kubernetes.KUBE_ROOTCA_UPDATED_HOST_TRUSTNEWCA) + + result = self.post_json(self.post_url, create_dict, + headers=self.headers, + expect_errors=True) + + # but client make a call to perform host update phase trustNewCA + self.assertEqual(http_client.BAD_REQUEST, result.status_int) + self.assertIn("kube-rootca-host-update rejected: not " + "allowed when cluster update is in state: %s. " + "(only allowed when in state: %s)" + % (kubernetes.KUBE_ROOTCA_UPDATING_PODS_TRUSTNEWCA, + kubernetes.KUBE_ROOTCA_UPDATED_HOST_UPDATECERTS), + result.json['error_message']) + + def test_trustnewca_host_update_failed_already_completed(self): + create_dict = {'phase': self.phase} + + # overall update is host update trustnewca phase in progress + dbutils.create_test_kube_rootca_update( + state=kubernetes.KUBE_ROOTCA_UPDATING_HOST_TRUSTNEWCA) + + # host update is trustnewca phase completed on this host + dbutils.create_test_kube_rootca_host_update(host_id=self.host.id, + state=kubernetes.KUBE_ROOTCA_UPDATED_HOST_TRUSTNEWCA) + + result = self.post_json(self.post_url, create_dict, + headers=self.headers, + expect_errors=True) + self.assertEqual(http_client.BAD_REQUEST, result.status_int) + self.assertIn("kube-rootca-host-update rejected: update already " + "completed on host %s" % self.host.hostname, + result.json['error_message']) + + def test_trustnewca_host_update_failed_in_past_state(self): + create_dict = {'phase': self.phase} + + # overall update is host update trustnewca phase in progress + dbutils.create_test_kube_rootca_update( + state=kubernetes.KUBE_ROOTCA_UPDATING_HOST_UPDATECERTS) + + # host update is trustbothcas phase completed + dbutils.create_test_kube_rootca_host_update(host_id=self.host.id, + state=kubernetes.KUBE_ROOTCA_UPDATED_HOST_TRUSTBOTHCAS) + + result = self.post_json(self.post_url, create_dict, + headers=self.headers, + expect_errors=True) + + self.assertEqual(http_client.BAD_REQUEST, result.status_int) + self.assertIn("kube-rootca-host-update rejected: not " + "allowed when cluster update is in state: %s. " + "(only allowed when in state: %s)" + % (kubernetes.KUBE_ROOTCA_UPDATING_HOST_UPDATECERTS, + kubernetes.KUBE_ROOTCA_UPDATED_HOST_UPDATECERTS), + result.json['error_message']) + + def test_trustnewca_host_update_failed_other_host_failed(self): + create_dict = {'phase': self.phase} + + # overall update is host update trustnewca phase failed + dbutils.create_test_kube_rootca_update( + state=kubernetes.KUBE_ROOTCA_UPDATING_HOST_TRUSTNEWCA_FAILED) + + # host update is trustnewca phase failed on host2 + dbutils.create_test_kube_rootca_host_update(host_id=self.host2.id, + state=kubernetes.KUBE_ROOTCA_UPDATING_HOST_TRUSTNEWCA_FAILED) + + dbutils.create_test_kube_rootca_host_update(host_id=self.host.id, + state=kubernetes.KUBE_ROOTCA_UPDATED_HOST_UPDATECERTS) + + result = self.post_json(self.post_url, create_dict, + headers=self.headers, + expect_errors=True) + + self.assertEqual(http_client.BAD_REQUEST, result.status_int) + self.assertIn("kube-rootca-host-update rejected: update failed " + "on host %s" % self.host2.hostname, + result.json['error_message'])