From 653f65106eb2ee831cd63f3184c9e77812174bf0 Mon Sep 17 00:00:00 2001 From: Gleb Aronsky Date: Thu, 13 Apr 2023 14:34:07 -0700 Subject: [PATCH] Remove broken/stale K8s upgrade script upgrade_k8s_config.sh does not support "flow" style of YAML. This results in the configmap becoming corrupted when upgrading from k8s 1.21 to 1.22. Removing the legacy code, the changes to the config map are now handled in sysinv. Test Plan PASS: - Manually update scripts on controllers and worker nodes based on https://review.opendev.org/c/starlingx/stx-puppet/+/880547?tab=comments - Perform manual upgrade from k8s v1.21.8 to v1.22.5 - Verify kubernetes successfully upgraded to v1.22.5 Test was performed in the lab with local changes to verify the code. Patch was not tested. Closes-Bug: 2017696 Depends-On: https://review.opendev.org/c/starlingx/stx-puppet/+/880547 Signed-off-by: Gleb Aronsky Change-Id: Ic47c3e40418898917934309f69b35ad1f401eb06 --- .../centos/files/upgrade_k8s_config.sh | 85 ---------- .../centos/kubernetes-unversioned.spec | 5 - .../deb_folder/kubernetes-unversioned.install | 1 - .../debian/deb_folder/rules | 1 - .../debian/deb_folder/upgrade_k8s_config.sh | 148 ------------------ 5 files changed, 240 deletions(-) delete mode 100755 kubernetes/kubernetes-unversioned/centos/files/upgrade_k8s_config.sh delete mode 100755 kubernetes/kubernetes-unversioned/debian/deb_folder/upgrade_k8s_config.sh diff --git a/kubernetes/kubernetes-unversioned/centos/files/upgrade_k8s_config.sh b/kubernetes/kubernetes-unversioned/centos/files/upgrade_k8s_config.sh deleted file mode 100755 index 3aaa5edd8..000000000 --- a/kubernetes/kubernetes-unversioned/centos/files/upgrade_k8s_config.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/bin/bash -# Copyright (c) 2021 Wind River Systems, Inc. -# -# SPDX-License-Identifier: Apache-2.0 -# - -# This will run for every k8s upgrade as a part of the control-plane upgrade of the first master. -# - updates kubeadm-config configmap to configure kube-apiserver manifest with RemoveSelfLink=false. -# - generates a kubelet config override file to configure cgroupDriver=cgroupfs. -# This is consumed by kubeadm upgrade apply -# -# Background: -# Kubernetes 1.21 changed cgroupDriver default to systemd (was cgroupfs). -# Kubernetes 1.20 changed feature-gates RemoveSelfLink default to true. - - -KUBEADM_CONFIGMAP_TMPFILE='/tmp/kubeadm_cm.yaml' - -function log { - logger -p local1.info "$1" -} - -# Update the configmap for kubeadm -function update_apiserver_configmap { - - log "Retrieving kubeadm configmap to temporary location: ${KUBEADM_CONFIGMAP_TMPFILE}" - counter=0 - RC=0 - RETRIES=10 - until [ $counter -gt $RETRIES ]; do - kubectl --kubeconfig=/etc/kubernetes/admin.conf -n kube-system get \ - configmap kubeadm-config -o yaml > ${KUBEADM_CONFIGMAP_TMPFILE} - RC=$? - if [ "$RC" = "0" ] ; then - log "Kubeadm configmap retrieved." - break - fi - log "Error retrieving kubeadm configmap, retrying..." - sleep 5 - let "counter+=1" - done - - if [ $counter -gt $RETRIES ]; then - log "Failed to retrieve kubeadm configmap with error code [$RC]". - exit $RC - fi - - if ! grep -q 'RemoveSelfLink=false' ${KUBEADM_CONFIGMAP_TMPFILE}; then - - log "Updating kube-apiserver feature-gates in retrieved kubeadm-config" - - if sed -i \ -'/^\s*feature-gates:\s*.*HugePageStorageMediumSize='\ -'true/ s/$/,RemoveSelfLink=false/' ${KUBEADM_CONFIGMAP_TMPFILE}; then - - if grep -q 'RemoveSelfLink=false' ${KUBEADM_CONFIGMAP_TMPFILE}; - then - log "Successfully updated retrieved kubeadm-config" - if kubectl --kubeconfig=/etc/kubernetes/admin.conf replace -f \ - ${KUBEADM_CONFIGMAP_TMPFILE}; then - log 'Successfully replaced updated kubeadm configmap.' - else - RC=$? - log "Failed to replace updated kubeadm configmap with error code: [$RC]" - exit $RC - fi - else - log 'Failed to update kube-apiserver feature-gates with an unknown error' - exit -1 - fi - else - RC=$? - log "Failed to update ${KUBEADM_CONFIGMAP_TMPFILE} with error code: [$RC]" - exit $RC - fi - else - log "Kubeadm configmap was already updated with RemoveSelfLink=false. Nothing to do." - fi - - rm -f ${KUBEADM_CONFIGMAP_TMPFILE} - -} - -update_apiserver_configmap -exit 0 diff --git a/kubernetes/kubernetes-unversioned/centos/kubernetes-unversioned.spec b/kubernetes/kubernetes-unversioned/centos/kubernetes-unversioned.spec index f9944f10c..836f4c4bb 100644 --- a/kubernetes/kubernetes-unversioned/centos/kubernetes-unversioned.spec +++ b/kubernetes/kubernetes-unversioned/centos/kubernetes-unversioned.spec @@ -39,8 +39,6 @@ Source2: kubernetes-accounting.conf # kubelet config overrides parameters Source3: kubelet_override.yaml -Source4: upgrade_k8s_config.sh - Source5: sanitize_kubelet_reserved_cpus.sh Patch1: kubelet-service-remove-docker-dependency.patch @@ -101,8 +99,6 @@ install -p -D -m 644 %{SOURCE3} %{buildroot}%{_sysconfdir}/%{_k8s_name}/kubelet_ install -d %{buildroot}%{local_sbindir} # install execution scripts -install -m 700 %{SOURCE4} %{buildroot}/%{local_sbindir}/upgrade_k8s_config.sh - install -m 700 %{SOURCE5} %{buildroot}/%{local_sbindir}/sanitize_kubelet_reserved_cpus.sh # install service files @@ -123,7 +119,6 @@ install -v -p -m 0644 -t %{buildroot}/%{_sysconfdir}/systemd/system.conf.d %{SOU %dir %{_curr_stage2} # the following are execution scripts -%{local_sbindir}/upgrade_k8s_config.sh %{local_sbindir}/sanitize_kubelet_reserved_cpus.sh # the following are symlinks diff --git a/kubernetes/kubernetes-unversioned/debian/deb_folder/kubernetes-unversioned.install b/kubernetes/kubernetes-unversioned/debian/deb_folder/kubernetes-unversioned.install index 0095bb10a..7975fbb5f 100644 --- a/kubernetes/kubernetes-unversioned/debian/deb_folder/kubernetes-unversioned.install +++ b/kubernetes/kubernetes-unversioned/debian/deb_folder/kubernetes-unversioned.install @@ -7,4 +7,3 @@ etc/kubernetes/proxy etc/systemd/system.conf.d/kubernetes-accounting.conf usr/lib/tmpfiles.d/kubernetes.conf usr/local/sbin/sanitize_kubelet_reserved_cpus.sh -usr/local/sbin/upgrade_k8s_config.sh diff --git a/kubernetes/kubernetes-unversioned/debian/deb_folder/rules b/kubernetes/kubernetes-unversioned/debian/deb_folder/rules index 8ab58fb08..957a7ac35 100755 --- a/kubernetes/kubernetes-unversioned/debian/deb_folder/rules +++ b/kubernetes/kubernetes-unversioned/debian/deb_folder/rules @@ -65,7 +65,6 @@ override_dh_install: # install scripts install -v -m 0700 -d ${DEBIAN_DESTDIR}${_local_sbindir} install -v -m 0700 -t ${DEBIAN_DESTDIR}${_local_sbindir} debian/sanitize_kubelet_reserved_cpus.sh - install -v -m 0700 -t ${DEBIAN_DESTDIR}${_local_sbindir} debian/upgrade_k8s_config.sh dh_install diff --git a/kubernetes/kubernetes-unversioned/debian/deb_folder/upgrade_k8s_config.sh b/kubernetes/kubernetes-unversioned/debian/deb_folder/upgrade_k8s_config.sh deleted file mode 100755 index 0b59d3890..000000000 --- a/kubernetes/kubernetes-unversioned/debian/deb_folder/upgrade_k8s_config.sh +++ /dev/null @@ -1,148 +0,0 @@ -#!/bin/bash -# Copyright (c) 2022 Wind River Systems, Inc. -# -# SPDX-License-Identifier: Apache-2.0 -# - -# This will run during k8s upgrades as a part of the control-plane upgrade of -# the first master. It updates the kubeadm-config configmap to edit the -# manifests and updates feature-gates as required for the specific k8s version -# it is upgrading to. -# -# The script updates feature gates for versions 1.22 and 1.24. -# -# It removes below feature gates from kube-apiserver configmap while upgrading -# k8s 1.21.8 to 1.22.5: SCTPSupport=true, HugePageStorageMediumSize=true -# and TTLAfterFinished=true and removes RemoveSelfLink=false while upgrading -# 1.23.1 to 1.24.4. -# -# Background: -# HugePageStorageMediumSize is deprecated in Kubernetes 1.22 -# SCTPSupport blocks kube-apiserver pod to spawn after control-plane upgrade. -# TTLAfterFinished value defaults to true from k8s 1.21 -# Kubernetes 1.24 no longer allows setting kube-apsierver feature-gate -# RemoveSelfLink=false. All the other feature gates we were using now default -# to true so we don't want to specify them anymore. - -# Temporary configuration file -KUBEADM_CONFIGMAP_TMPFILE=$(mktemp /tmp/kubeadm_cm.yaml.XXXXXX 2>/dev/null) - -# Log info message to /var/log/daemon.log -function LOG { - logger -p daemon.info "$0($$): " "${@}" -} - -# Log error message to /var/log/daemon.log -function ERROR { - logger -s -p daemon.error "$0($$): " "${@}" -} - -# Cleanup and exit -function cleanup_and_exit { - rm -v -f "${KUBEADM_CONFIGMAP_TMPFILE}" 2>/dev/null - exit "${1:-0}" -} - -function get_kubeadm_configmap { - LOG "Retrieving kubeadm configmap: ${KUBEADM_CONFIGMAP_TMPFILE}" - local counter=0 - local RETRIES=10 - RC=0 - until [ ${counter} -gt ${RETRIES} ]; do - kubectl --kubeconfig=/etc/kubernetes/admin.conf -n kube-system get \ - configmap kubeadm-config -o yaml > "${KUBEADM_CONFIGMAP_TMPFILE}" - RC=$? - if [ "${RC}" == "0" ] ; then - LOG "Kubeadm configmap retrieved." - break - fi - ERROR "Error retrieving kubeadm configmap, retrying..." - sleep 5 - counter=$(( counter+1 )) - done - - if [ ${counter} -gt ${RETRIES} ]; then - ERROR "Failed to retrieve kubeadm configmap with error code [${RC}]". - cleanup_and_exit ${RC} - fi -} - -# Update feature gates for version 1.22 -function update_feature_gates_v1_22 { - LOG "Updating kube-apiserver feature-gates in retrieved kubeadm-config" - - # Update api-server feature-gates - sed -i \ - 's/^\( *\)feature-gates:\s.*RemoveSelfLink=false/\1feature-gates: RemoveSelfLink=false/g' \ - "${KUBEADM_CONFIGMAP_TMPFILE}" - RC=$? - if [ "${RC}" == "0" ]; then - LOG "Successfully updated kube-apiserver feature-gates in retrieved kubeadm-config" - else - ERROR "Failed to update kube-apiserver feature-gates in retrieved kubeadm-config with error code: [${RC}]" - cleanup_and_exit ${RC} - fi - - # update controller-manager feature-gates - sed -i \ - '/feature-gates: TTLAfterFinished=true/d' "${KUBEADM_CONFIGMAP_TMPFILE}" - RC=$? - if [ "${RC}" == "0" ]; then - LOG "Successfully updated controller-manager feature-gates in retrieved kubeadm-config" - else - # we need not gracefully exit here as failing to update this does not - # make any difference to the k8s cluster functions as default value of - # TTLAfterFinished is true - ERROR "Failed to update controller-manager feature-gates in retrieved kubeadm-config with error code: [${RC}]" - fi -} - -function replace_configmap { - - output=$(kubectl --kubeconfig=/etc/kubernetes/admin.conf replace -f "${KUBEADM_CONFIGMAP_TMPFILE}" 2>&1) - RC=$? - if [ "${RC}" == "0" ]; then - LOG 'Successfully replaced updated kubeadm configmap.' - else - # LP-1996546. kubectl replace expects replacing object to be latest version. - # Although there is low chance that kubeadm-configmap is modified by other - # process in between calls get_kubeadm_configmap and replace_configmap, - # we should still check for that error. If it is the case, then we retry - # modifying and replacing the newer version. - if [[ ${output} == *"the object has been modified; please apply your changes to the latest version and try again" ]]; then - LOG "kubeadm configmap is not the newest version." - else - ERROR "Failed to replace updated kubeadm configmap with error code: [${RC}]" - cleanup_and_exit ${RC} - fi - fi - return ${RC} -} - -K8S_VERSION=$(kubectl version --output=yaml| grep -m1 -oP 'gitVersion: \K(\S+)') -LOG "k8s version: ${K8S_VERSION}" -counter=0 -RETRIES=3 -# Most errors during script execution result in exiting the script except one error. -# If kubeadm-configmap is modified by external process after it is -# retrieved in function get_kubeadm_configmap and before it is modified -# and replaced in the function replace_configmap, we should retry modifying -# and replacing the latest version. -until [ ${counter} -gt ${RETRIES} ]; do - get_kubeadm_configmap - if [[ "${K8S_VERSION}" == "v1.21.8" ]]; then - update_feature_gates_v1_22 - else - LOG "No update required for kubeadm configmap" - break - fi - replace_configmap - if [ "$?" == "0" ]; then - break - else - LOG "Retrying to update the configmap..." - counter=$(( counter+1 )) - fi -done - -cleanup_and_exit 0