From d807f868d6eb7378d3c7a7a9dbfad2bd9a572627 Mon Sep 17 00:00:00 2001 From: Victor Romano Date: Wed, 13 Mar 2024 09:06:49 -0300 Subject: [PATCH] Fix failed pods not being detected by rootca health check On the health check prior to rootca update, there was a bug that prevented CrashLoopBackoff pods being detected as unhealthy. This is because the pods are in phase "Running", but the status of the container itself is "ready: false". This commit adds an additional check to "Running" pods so if any container inside it is not ready, the pod will be deemed unhealthy. Test plan: - PASS: Attempt to perform a rootca update with a pod in CrashloopBackoff state. Verify the update is not possible and the health check fails with the pod being show as unhealthy is "system health-query-kube-upgrade --rootca" - PASS: Verify the rootca update is possible if no pods are in CrashloopBackoff state. Closes-Bug: 2057779 Change-Id: I115b6621df11516db2279fe6bc96452d27975c50 Signed-off-by: Victor Romano --- sysinv/sysinv/sysinv/sysinv/common/health.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sysinv/sysinv/sysinv/sysinv/common/health.py b/sysinv/sysinv/sysinv/sysinv/common/health.py index 70800ba29c..921ecf10b7 100755 --- a/sysinv/sysinv/sysinv/sysinv/common/health.py +++ b/sysinv/sysinv/sysinv/sysinv/common/health.py @@ -264,6 +264,13 @@ class Health(object): if pod.status.phase not in ['Pending', 'Running', 'Succeeded']: # Add it to the failed list as it's not ready/completed/pending fail_pod_list.append((pod.metadata.name, pod.metadata.namespace)) + elif pod.status.phase == 'Running': + for container_status in pod.status.container_statuses: + if container_status.ready is not True: + # Pod has running status but it's not ready + fail_pod_list.append((pod.metadata.name, + pod.metadata.namespace)) + break success = not fail_pod_list return success, fail_pod_list