From d57d3a07b8e2897f1cf087c7fd757d67645b0ffe Mon Sep 17 00:00:00 2001 From: Jim Gauld Date: Wed, 1 Jun 2022 11:24:14 -0400 Subject: [PATCH] Add runtime reconfiguration of kubelet This adds the CLI command 'system kube-config-kubelet'. This invokes puppet runtime manifests to reconfigure kubelet-config ConfigMap with new parameters, and to upgrade kubernetes nodes with new parameters, and restart kubelet. This gives the ability to update kubelet parameters with a software patch. The specific kubelet-config parameters are provided within the puppet manifests and its supporting parameters script. The specific settings values and engineering are described in the puppet component. Identical settings are also configured at install time in ansible-playbooks. TESTING: PASS - manually fill /var/lib/docker to exceed imageGC and verify GC operates PASS - AIO-DX fresh install gets updated kubelet config PASS - AIO-DX apply/remove designer patch with updated kubelet config PASS - 'system kube-config-kubelet' updates K8S nodes kubelet config PASS - AIO-DX reinstall controller-1 has updated kubelet config PASS - AIO-DX install new worker node gets updated kubelet config PASS - build and view REST documentation Partial-Bug: 1977754 Depends-On: https://review.opendev.org/c/starlingx/stx-puppet/+/844298 Depends-On: https://review.opendev.org/c/starlingx/ansible-playbooks/+/844305 Signed-off-by: Jim Gauld Change-Id: Iad32a724d3f681bc9854fa663299f8539f70fd2a --- api-ref/source/api-ref-sysinv-v1-config.rst | 50 ++++++++++++ .../cgts-client/cgtsclient/v1/client.py | 3 + .../cgtsclient/v1/kube_config_kubelet.py | 26 +++++++ .../v1/kube_config_kubelet_shell.py | 29 +++++++ .../cgts-client/cgtsclient/v1/shell.py | 4 +- .../sysinv/api/controllers/v1/__init__.py | 13 ++++ .../api/controllers/v1/kube_config_kubelet.py | 36 +++++++++ .../sysinv/sysinv/sysinv/conductor/manager.py | 77 ++++++++++++++++++- .../sysinv/sysinv/sysinv/conductor/rpcapi.py | 7 ++ sysinv/sysinv/sysinv/sysinv/puppet/common.py | 1 + 10 files changed, 244 insertions(+), 2 deletions(-) create mode 100644 sysinv/cgts-client/cgts-client/cgtsclient/v1/kube_config_kubelet.py create mode 100644 sysinv/cgts-client/cgts-client/cgtsclient/v1/kube_config_kubelet_shell.py create mode 100644 sysinv/sysinv/sysinv/sysinv/api/controllers/v1/kube_config_kubelet.py diff --git a/api-ref/source/api-ref-sysinv-v1-config.rst b/api-ref/source/api-ref-sysinv-v1-config.rst index 3516df7109..fad15bb1aa 100644 --- a/api-ref/source/api-ref-sysinv-v1-config.rst +++ b/api-ref/source/api-ref-sysinv-v1-config.rst @@ -655,6 +655,16 @@ itemNotFound (404) "href": "http://10.10.10.3:6385/kube_upgrade/", "rel": "bookmark" } + ], + "kube_config_kubelet": [ + { + "href": "http://10.10.10.3:6385/v1/kube_config_kubelet/", + "rel": "self" + }, + { + "href": "http://10.10.10.3:6385/kube_config_kubelet/", + "rel": "bookmark" + } ] } @@ -12515,3 +12525,43 @@ forbidden (403), badMethod (405), overLimit (413) } This operation does not accept a request body. + +------------------------- +Kubernetes config kubelet +------------------------- + +These APIs allow the reconfiguration of kubelet-config parameters and restart of kubelet on each node. + +****************************************************************************** +Apply kubelet-config parameters reconfiguration and restart kubelet procedure +****************************************************************************** + +.. rest_method:: POST /v1/kube_config_kubelet/apply + +**Normal response codes** + +200 + +**Error response codes** + +computeFault (400, 500, ...), serviceUnavailable (503), badRequest (400), +unauthorized (401), forbidden (403), badMethod (405), overLimit (413) + +**Request parameters** + + +**Response parameters** + +.. csv-table:: + :header: "Parameter", "Style", "Type", "Description" + :widths: 20, 20, 20, 60 + + "success", "plain", "xsd:string", "The success message indicating start of the kube-config-kubelet apply" + "error", "plain", "xsd:string", "The error message in case something wrong happen on the API execution" + +:: + + { + "success": "kube-config-kubelet applied.", + "error": "" + } diff --git a/sysinv/cgts-client/cgts-client/cgtsclient/v1/client.py b/sysinv/cgts-client/cgts-client/cgtsclient/v1/client.py index 7c5fdf3192..61ce8c85ac 100644 --- a/sysinv/cgts-client/cgts-client/cgtsclient/v1/client.py +++ b/sysinv/cgts-client/cgts-client/cgtsclient/v1/client.py @@ -55,6 +55,7 @@ from cgtsclient.v1 import isystem from cgtsclient.v1 import iuser from cgtsclient.v1 import kube_cluster from cgtsclient.v1 import kube_cmd_version +from cgtsclient.v1 import kube_config_kubelet from cgtsclient.v1 import kube_host_upgrade from cgtsclient.v1 import kube_rootca_update from cgtsclient.v1 import kube_upgrade @@ -181,3 +182,5 @@ class Client(object): self.device_label = device_label.DeviceLabelManager(self.http_client) self.restore = restore.RestoreManager(self.http_client) self.kube_rootca_update = kube_rootca_update.KubeRootCAUpdateManager(self.http_client) + self.kube_config_kubelet = \ + kube_config_kubelet.KubeConfigKubeletManager(self.http_client) diff --git a/sysinv/cgts-client/cgts-client/cgtsclient/v1/kube_config_kubelet.py b/sysinv/cgts-client/cgts-client/cgtsclient/v1/kube_config_kubelet.py new file mode 100644 index 0000000000..c89679894b --- /dev/null +++ b/sysinv/cgts-client/cgts-client/cgtsclient/v1/kube_config_kubelet.py @@ -0,0 +1,26 @@ +# +# Copyright (c) 2022 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +from cgtsclient.common import base + + +class KubeConfigKubelet(base.Resource): + def __repr__(self): + return "" % self._info + + +class KubeConfigKubeletManager(base.Manager): + resource_class = KubeConfigKubelet + + @staticmethod + def _path(id=None): + return '/v1/kube_config_kubelet/%s' % id if id \ + else '/v1/kube_config_kubelet' + + def apply(self): + path = self._path("apply") + _, body = self.api.json_request('POST', path) + return body diff --git a/sysinv/cgts-client/cgts-client/cgtsclient/v1/kube_config_kubelet_shell.py b/sysinv/cgts-client/cgts-client/cgtsclient/v1/kube_config_kubelet_shell.py new file mode 100644 index 0000000000..e27c85a50f --- /dev/null +++ b/sysinv/cgts-client/cgts-client/cgtsclient/v1/kube_config_kubelet_shell.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# All Rights Reserved. +# + +from cgtsclient import exc + + +def do_kube_config_kubelet(cc, args): + """Apply the kubelet config.""" + + try: + response = cc.kube_config_kubelet.apply() + except exc.HTTPNotFound: + raise exc.CommandError('Failed to apply kubelet config. No response.') + except Exception as e: + raise exc.CommandError('Failed to apply kubelet config: %s' % (e)) + else: + success = response.get('success') + error = response.get('error') + if success: + print("Success: " + success) + if error: + print("Error: " + error) diff --git a/sysinv/cgts-client/cgts-client/cgtsclient/v1/shell.py b/sysinv/cgts-client/cgts-client/cgtsclient/v1/shell.py index e4c29cf410..527a17536e 100644 --- a/sysinv/cgts-client/cgts-client/cgtsclient/v1/shell.py +++ b/sysinv/cgts-client/cgts-client/cgtsclient/v1/shell.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2013-2021 Wind River Systems, Inc. +# Copyright (c) 2013-2022 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -42,6 +42,7 @@ from cgtsclient.v1 import isystem_shell from cgtsclient.v1 import iuser_shell from cgtsclient.v1 import kube_cluster_shell +from cgtsclient.v1 import kube_config_kubelet_shell from cgtsclient.v1 import kube_rootca_update_shell from cgtsclient.v1 import kube_upgrade_shell from cgtsclient.v1 import kube_version_shell @@ -128,6 +129,7 @@ COMMAND_MODULES = [ app_shell, host_fs_shell, kube_cluster_shell, + kube_config_kubelet_shell, kube_version_shell, kube_upgrade_shell, kube_rootca_update_shell, diff --git a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/__init__.py b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/__init__.py index 28d01742ce..35c753bca1 100644 --- a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/__init__.py +++ b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/__init__.py @@ -47,6 +47,7 @@ from sysinv.api.controllers.v1 import kube_rootca_update from sysinv.api.controllers.v1 import kube_upgrade from sysinv.api.controllers.v1 import kube_version from sysinv.api.controllers.v1 import kube_cmd_version +from sysinv.api.controllers.v1 import kube_config_kubelet from sysinv.api.controllers.v1 import label from sysinv.api.controllers.v1 import interface from sysinv.api.controllers.v1 import interface_network @@ -281,6 +282,9 @@ class V1(base.APIBase): kube_host_upgrades = [link.Link] "Links to the kube_host_upgrade resource" + kube_config_kubelet = [link.Link] + "Links to the kube_config_kubelet resource " + device_images = [link.Link] "Links to the device images resource" @@ -870,6 +874,14 @@ class V1(base.APIBase): 'kube_host_upgrades', '', bookmark=True)] + v1.kube_config_kubelet = [ + link.Link.make_link('self', pecan.request.host_url, + 'kube_config_kubelet', ''), + link.Link.make_link('bookmark', pecan.request.host_url, + 'kube_config_kubelet', '', + bookmark=True) + ] + v1.device_images = [link.Link.make_link('self', pecan.request.host_url, 'device_images', ''), link.Link.make_link('bookmark', @@ -974,6 +986,7 @@ class Controller(rest.RestController): kube_upgrade = kube_upgrade.KubeUpgradeController() kube_rootca_update = kube_rootca_update.KubeRootCAUpdateController() kube_host_upgrades = kube_host_upgrade.KubeHostUpgradeController() + kube_config_kubelet = kube_config_kubelet.KubeConfigKubeletController() device_images = device_image.DeviceImageController() device_image_state = device_image_state.DeviceImageStateController() device_labels = device_label.DeviceLabelController() diff --git a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/kube_config_kubelet.py b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/kube_config_kubelet.py new file mode 100644 index 0000000000..43778eb743 --- /dev/null +++ b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/kube_config_kubelet.py @@ -0,0 +1,36 @@ +######################################################################## +# +# Copyright (c) 2022 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +######################################################################## + +import pecan +from pecan import expose +from pecan import rest + +from sysinv.common import utils as cutils +from sysinv.openstack.common.rpc.common import RemoteError + +LOCK_NAME = 'KubeConfigKubeletController' + + +class KubeConfigKubeletController(rest.RestController): + """REST controller for kube_config_kubelet.""" + + _custom_actions = { + 'apply': ['POST'], + } + + @expose('json') + @cutils.synchronized(LOCK_NAME) + def apply(self): + try: + pecan.request.rpcapi.kube_config_kubelet(pecan.request.context) + except RemoteError as e: + return dict(success="", error=e.value) + except Exception as ex: + return dict(success="", error=str(ex)) + + return dict(success="kube-config-kubelet applied.", error="") diff --git a/sysinv/sysinv/sysinv/sysinv/conductor/manager.py b/sysinv/sysinv/sysinv/sysinv/conductor/manager.py index c32f99bfe0..e97f2fd410 100644 --- a/sysinv/sysinv/sysinv/sysinv/conductor/manager.py +++ b/sysinv/sysinv/sysinv/sysinv/conductor/manager.py @@ -1703,6 +1703,30 @@ class ConductorManager(service.PeriodicService): return False + def kube_config_kubelet(self, context): + """Update kubernetes nodes kubelet configuration ConfigMap. + + This method updates kubelet parameters in configmaps/kubelet-config. + This leverages puppet report status so we can wait for completion + of the runtime manifest and trigger subsequent per-node configuration. + + :param context: request context + """ + active_controller = utils.HostHelper.get_active_controller(self.dbapi) + personalities = [constants.CONTROLLER] + config_uuid = self._config_update_hosts(context, personalities, + [active_controller.uuid]) + config_dict = { + "personalities": personalities, + "host_uuids": [active_controller.uuid], + "classes": [ + 'platform::kubernetes::master::update_kubelet_params::runtime'], + puppet_common.REPORT_STATUS_CFG: + puppet_common.REPORT_KUBE_UPDATE_KUBELET_PARAMS + } + self._config_apply_runtime_manifest( + context, config_uuid=config_uuid, config_dict=config_dict) + def update_keystone_password(self, context): """This method calls a puppet class 'openstack::keystone::password::runtime' @@ -8488,6 +8512,26 @@ class ConductorManager(service.PeriodicService): self.report_sysparam_http_update_success, [], self.report_sysparam_http_update_failure, [error] ) + # Kubernetes kubelet parameters update and per node configuration + elif reported_cfg == puppet_common.REPORT_KUBE_UPDATE_KUBELET_PARAMS: + # The agent is reporting the runtime update_kubelet_params has been applied. + host_uuid = iconfig['host_uuid'] + if status == puppet_common.REPORT_SUCCESS: + # Update action was successful. + # Invoke per-node kubelet upgrade runtime configuration. + success = True + self.handle_kube_update_params_success(context, host_uuid) + elif status == puppet_common.REPORT_FAILURE: + # Update action has failed + args = {'cfg': reported_cfg, 'status': status, 'iconfig': iconfig} + LOG.error("config runtime failure, " + "reported_cfg: %(cfg)s status: %(status)s " + "iconfig: %(iconfig)s" % args) + else: + args = {'cfg': reported_cfg, 'status': status, 'iconfig': iconfig} + LOG.error("No match for sysinv-agent manifest application reported! " + "reported_cfg: %(cfg)s status: %(status)s " + "iconfig: %(iconfig)s" % args) else: LOG.error("Reported configuration '%(cfg)s' is not handled by" " report_config_status! iconfig: %(iconfig)s" % @@ -9266,6 +9310,37 @@ class ConductorManager(service.PeriodicService): upgrade.uuid, {'state': constants.UPGRADE_ACTIVATION_FAILED}) + def handle_kube_update_params_success(self, context, host_uuid): + """ + Callback for Sysinv Agent on kube update params success. + + This is invoked after kubelet-config ConfigMap is updated, + and does per-node kubernetes configuration. + + This will download the current kubelet-config ConfigMap, + regenerate the configuration file /var/lib/kubelet/config.yaml, + and restart kubelet per node. + + :param context: request context + :param host_uuid: host unique id + """ + LOG.info("Kube update params phase succeeded on host: %s" + % (host_uuid)) + + personalities = [constants.CONTROLLER, constants.WORKER] + hosts = self.dbapi.ihost_get_list() + host_uuids = [x.uuid for x in hosts if x.personality in personalities] + config_uuid = self._config_update_hosts(context, personalities, + host_uuids=host_uuids) + config_dict = { + "personalities": personalities, + "host_uuids": host_uuids, + "classes": [ + 'platform::kubernetes::update_kubelet_config::runtime'] + } + self._config_apply_runtime_manifest( + context, config_uuid=config_uuid, config_dict=config_dict) + def report_kube_rootca_update_success(self, host_uuid, reported_cfg): """ Callback for Sysinv Agent on kube root CA update success @@ -11165,7 +11240,7 @@ class ConductorManager(service.PeriodicService): config_dict=config_dict) if filter_classes and config_dict['classes'] in filter_classes: - LOG.info("config runtime filter_clasess add %s %s" % + LOG.info("config runtime filter_classes add %s %s" % (filter_classes, config_dict)) self._add_runtime_class_apply_in_progress(filter_classes, host_uuids=config_dict.get('host_uuids', None)) diff --git a/sysinv/sysinv/sysinv/sysinv/conductor/rpcapi.py b/sysinv/sysinv/sysinv/sysinv/conductor/rpcapi.py index e2873ceff4..1f43ae36d2 100644 --- a/sysinv/sysinv/sysinv/sysinv/conductor/rpcapi.py +++ b/sysinv/sysinv/sysinv/sysinv/conductor/rpcapi.py @@ -2088,6 +2088,13 @@ class ConductorAPI(sysinv.openstack.common.rpc.proxy.RpcProxy): return self.cast(context, self.make_msg('kube_upgrade_networking', kube_version=kube_version)) + def kube_config_kubelet(self, context): + """Sychronously, have the conductor configure kubelet. + + :param context: request context. + """ + return self.call(context, self.make_msg('kube_config_kubelet')) + def store_bitstream_file(self, context, filename): """Asynchronously, have the conductor store the device image on this host. diff --git a/sysinv/sysinv/sysinv/sysinv/puppet/common.py b/sysinv/sysinv/sysinv/sysinv/puppet/common.py index fe5d896391..6510d83680 100644 --- a/sysinv/sysinv/sysinv/sysinv/puppet/common.py +++ b/sysinv/sysinv/sysinv/sysinv/puppet/common.py @@ -51,6 +51,7 @@ REPORT_KUBE_CERT_UPDATE_PODS_TRUSTBOTHCAS = \ 'pods_' + constants.KUBE_CERT_UPDATE_TRUSTBOTHCAS REPORT_KUBE_CERT_UPDATE_PODS_TRUSTNEWCA = \ 'pods_' + constants.KUBE_CERT_UPDATE_TRUSTNEWCA +REPORT_KUBE_UPDATE_KUBELET_PARAMS = 'update_kubelet_params' REPORT_HTTP_CONFIG = 'http_config'