From 05bbc77057b84975cdae095c0173edb223363c0e Mon Sep 17 00:00:00 2001 From: Jim Gauld Date: Thu, 9 Nov 2023 04:17:58 -0500 Subject: [PATCH] Improve shutdown of containerd This update is to prevent nodes from crashing while powering off during graceful shutdown (or reboot). This improves timing and shutdown of containerd.service. The containerd shutdown script stops all containers via 'crictl stop' with 5 second timeout, followed by stop all pods via 'crictl stopp'. This cleans up lingering /pause sandbox containers. This modifies the arguments to xargs and crictl to let xargs deal with parallelism instead of batching to crictl. crictl appears to do the stop operations serially. The number stop in parallel is engineered to 10. Engineering the number of stop in parallel in relation to shutdown timings under stress load will be addressed in a subsequent update. The engineering TC should align with customer requirements. When testing containerd shutdown under the stress of multiple pods writing to a shared PersistentVolume, even the new parallel shutdown code is not sufficient to complete the shutdown within the default 90-second timeout. Additional changes will be needed to enable clean shutdown under those circumstances. Partial-Bug: 2043069 Test plan: - PASS - build-image, install and boot up on AIO-SX - PASS - perform reboot and verify /var/log/daemon.log has new k8s-container-cleanup.sh logs for 'Stopping all pods' and 'Stopping all containers', and that drbd stops after containerd. - FAIL - verify containerd shutdown works under stress with the new parallel stop pods parameter NPAR=10. The stress load uses ReadWriteMany PVC, and multiple pods, each writing to the shared PVC. Change-Id: Ibfc0a474a40344a629b3f0780449906a9c6b03ba Signed-off-by: Jim Gauld --- .../debian/files/k8s-container-cleanup.sh | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/kubernetes/containerd/debian/files/k8s-container-cleanup.sh b/kubernetes/containerd/debian/files/k8s-container-cleanup.sh index aeb884174..bbdeb8aaa 100755 --- a/kubernetes/containerd/debian/files/k8s-container-cleanup.sh +++ b/kubernetes/containerd/debian/files/k8s-container-cleanup.sh @@ -27,13 +27,26 @@ function ERROR { } function do_force_clean { - LOG "Stopping all containers." # Use crictl to gracefully stop each container. If specified timeout is # reached, it forcibly kills the container. There is no need to check # return code since there is nothing more we can do, and crictl already # logs to daemon.log. - crictl ps -q | xargs -n 10 -r crictl stop --timeout 5 + + # Number to stop in parallel + NPAR=10 + + # Set timeout to 5 seconds in case stop doesn't complete + TIMEOUT="--timeout 5" + + # Stop all containers. + LOG "Stopping all containers." + crictl ps -q | xargs -P ${NPAR} -n 1 -r crictl stop ${TIMEOUT} LOG "Stopping all containers completed." + + # Stop all pods, this will cleanup /pause containers. + LOG "Stopping all pods." + crictl pods -q | xargs -P ${NPAR} -n 1 -r crictl stopp + LOG "Stopping all pods completed." } case "$1" in @@ -52,7 +65,7 @@ case "$1" in 1) # 1 - initializing, starting, degraded, maintenance, stopping if [ "${state}" = "stopping" ]; then - do_force_clean + do_force_clean fi ;; esac