From 73632416b3fc5ddaa8e2b4babb93ba00fd6c58ca Mon Sep 17 00:00:00 2001 From: Kaustubh Dhokte Date: Wed, 17 Aug 2022 15:04:36 -0400 Subject: [PATCH] Preserve kube-apiserver manifest params during upgrade-activate This change https://opendev.org/starlingx/integ/commit/a6a5349d025487672fe818aae36a2020a9f9f08c (k8s-1.22.5: remove feature-gates) adds a script that is run during upgrade activate. The script modifies kubeadm cluster config and eventually updates kube-apiserver manifest to remove deprecated features-gates in k8s 1.22. As 'kubeadm init phase' is rerun in the script, it updates the kube-apiserver manifest to be in sync with the kubeadm cluster config. In that process, it nullifies the effect of these two commits, https://opendev.org/starlingx/stx-puppet/commit/04a1c1b0809f66488bd54e3f31d323430e7d9913 (Rework advertise address in apiserver-change-param) and https://opendev.org/starlingx/stx-puppet/commit/52ace69c837acc7e3aff8a2d584968297afd70fe (Amend kube-apiserver 1.23 configuration to use PSP) This change adds a function to the script that preserves the effect of above listed commits. Test Plan: On CentOS AIO-SX PASS: Upgrade Successful. Check if advertise address in kube-apiserver manifest before and after running upgrade-activate is same. Ensure that the seccomp profile configuration is removed after upgrade-activate. Kube-apiserver is running and cluster is accessible after the upgrade. PASS: No Shellcheck errors Closes-Bug: 1986854 Signed-off-by: Kaustubh Dhokte Change-Id: Ib97e14bc5b4ed208e65e16888e1380a3bd9fdb8f --- .../centos/files/update-k8s-feature-gates.sh | 68 +++++++++++++++++-- 1 file changed, 61 insertions(+), 7 deletions(-) diff --git a/kubernetes/kubernetes-1.22.5/centos/files/update-k8s-feature-gates.sh b/kubernetes/kubernetes-1.22.5/centos/files/update-k8s-feature-gates.sh index 590797612..df2b9007f 100644 --- a/kubernetes/kubernetes-1.22.5/centos/files/update-k8s-feature-gates.sh +++ b/kubernetes/kubernetes-1.22.5/centos/files/update-k8s-feature-gates.sh @@ -16,8 +16,17 @@ # SCTPSupport blocks kube-apiserver pod to spawn after control-plane upgrade # TTLAfterFinished value defaults to true from k8s 1.21 # +# The script also preserves the advertise-address in kube-apiserver +# manifest that gets overwritten as kubeadm init is run again in this script. +# In other words, it maintains the effect of this commit +# https://opendev.org/starlingx/stx-puppet/commit/04a1c1b0809f66488bd54e3f31d323430e7d9913 +# +# Similarly, it removes the seccomp profiles configuration from the +# kube-apiserver manifest file to maintain the effect of this commit, +# https://opendev.org/starlingx/stx-puppet/commit/52ace69c837acc7e3aff8a2d584968297afd70fe KUBEADM_CONFIGMAP_TMPFILE='/tmp/kubeadm_cm' +API_SERVER_MANIFEST='/etc/kubernetes/manifests/kube-apiserver.yaml' rc_controller_manager=0 rc_apiserver=0 @@ -36,7 +45,7 @@ function get_kubeadm_configmap { kubectl --kubeconfig=/etc/kubernetes/admin.conf -n kube-system get \ configmap kubeadm-config -o "$1" > ${KUBEADM_CONFIGMAP_TMPFILE} RC=$? - if [ 0 == ${RC} ] ; then + if [ ${RC} == 0 ] ; then log "Kubeadm configmap retrieved." break fi @@ -63,7 +72,7 @@ function update_kubeadm_configmap { 's/^\( *\)feature-gates:\s.*RemoveSelfLink=false/\1feature-gates: RemoveSelfLink=false/g' \ ${KUBEADM_CONFIGMAP_TMPFILE} rc_apiserver=$? - if [ 0 == ${rc_apiserver} ]; then + if [ ${rc_apiserver} == 0 ]; then log "Successfully updated kube-apiserver feature-gates in retrieved kubeadm-config" else log "Failed to update kube-apiserver feature-gates in retrieved kubeadm-config with error code: [${rc_apiserver}]" @@ -73,7 +82,7 @@ function update_kubeadm_configmap { sed -i \ '/feature-gates: TTLAfterFinished=true/d' ${KUBEADM_CONFIGMAP_TMPFILE} rc_controller_manager=$? - if [ 0 == ${rc_controller_manager} ]; then + if [ ${rc_controller_manager} == 0 ]; then log "Successfully updated controller-manager feature-gates in retrieved kubeadm-config" else # we need not gracefully exit here as failing to update this does not @@ -105,11 +114,11 @@ function update_manifests { get_kubeadm_configmap jsonpath='{.data.ClusterConfiguration}' # Rewrite apiserver manifest only if it is updated in the configmap - if [ 0 == ${rc_apiserver} ]; then + if [ ${rc_apiserver} == 0 ]; then kubeadm init phase control-plane apiserver \ --config ${KUBEADM_CONFIGMAP_TMPFILE} RC=$? - if [ 0 == ${RC} ]; then + if [ ${RC} == 0 ]; then log "Success executing kubeadm init phase control-plane for kube-api-server" else log "Failed to update kube-api-server manifest with error code: [${RC}]" @@ -119,11 +128,11 @@ function update_manifests { fi # Rewrite controller-manager manifest only if it is updated in the configmap - if [ 0 == ${rc_controller_manager} ]; then + if [ ${rc_controller_manager} == 0 ]; then kubeadm init phase control-plane controller-manager \ --config ${KUBEADM_CONFIGMAP_TMPFILE} RC=$? - if [ 0 == ${RC} ]; then + if [ ${RC} == 0 ]; then log "Success executing kubeadm init phase control-plane for kube-controller-manager" else log "Failed to update kube-controller-manager manifest with error code: [${RC}]" @@ -134,8 +143,53 @@ function update_manifests { } +function preserve_apiserver_manifest_params { + + # The following code preserves the kube-apiserver advertise address that gets overwitten + # after kubeadm init phase is run in order to preserve the effect of: + # https://opendev.org/starlingx/stx-puppet/commit/04a1c1b0809f66488bd54e3f31d323430e7d9913 + DEFAULT_NETWORK_INTERFACE=$(grep 'advertise-address=' ${API_SERVER_MANIFEST} | cut -d "=" -f2) + RC=$? + if [ ${RC} == 0 ]; then + log "advertise-address: ${DEFAULT_NETWORK_INTERFACE}" + else + log "Failed to get advertise address from kube-apiserver manifest. Error code: [${RC}]" + fi + + if [ "${DEFAULT_NETWORK_INTERFACE}" ] && [ "${APISERVER_ADVERTISE_ADDRESS}" ]; then + sed -i "/oidc-issuer-url/! s/${DEFAULT_NETWORK_INTERFACE}/${APISERVER_ADVERTISE_ADDRESS}/g" ${API_SERVER_MANIFEST} + RC=$? + if [ ${RC} == 0 ]; then + log "Advertise address [${DEFAULT_NETWORK_INTERFACE}] is replaced by [${APISERVER_ADVERTISE_ADDRESS}] in kube-apiserver manifest." + else + log "Failed to preserve advertise address in kube-apiserver manifest. Error code: [${RC}]" + fi + fi + + # The following code removes seccomp profiles configuration from the kube-apiserver manifest + # to preserve the effect of: + # https://opendev.org/starlingx/stx-puppet/commit/52ace69c837acc7e3aff8a2d584968297afd70fe + sed -i '/securityContext:/,/type: RuntimeDefault/d' ${API_SERVER_MANIFEST} + RC=$? + if [ ${RC} == 0 ]; then + log "Seccomp Profile configuration removed from the kube-apiserver manifest if existed." + else + log "Failed to remove Seccomp Profile configuration from the kube-apiserver manifest. Error code: [${RC}]" + fi + +} + +APISERVER_ADVERTISE_ADDRESS=$(grep 'advertise-address=' ${API_SERVER_MANIFEST} | cut -d "=" -f2) +RC=$? +if [ ${RC} == 0 ]; then + log "advertise-address: ${APISERVER_ADVERTISE_ADDRESS}" +else + log "Failed to get advertise address from kube-apiserver manifest. Error code: [${RC}]" +fi + update_kubeadm_configmap update_manifests +preserve_apiserver_manifest_params rm -f ${KUBEADM_CONFIGMAP_TMPFILE}