From ace57cab1310608f80373e1f6059e2cfd09747b9 Mon Sep 17 00:00:00 2001 From: Boovan Rajendran Date: Sun, 5 Nov 2023 23:33:15 -0500 Subject: [PATCH] Set the affinity of guaranteed pod to non-isolated CPUs This corrects kubelet cpumanager static cpuset tracking for isolcpus for versions 1.26.1 and 1.27.5. This ensures that pods specified with isolcpus + guaranteed QoS + integer cpu requests, are affined to exclusive cpuset and tracked as non-isolated cpus. Test Plan: Set the "kube-cpu-mgr-policy=static" label and allocate isolcpus. The following tests are performed for both k8s 1.26.1 and 1.27.5: PASS: Guaranteed pod without isolcpus specified (Pod is affined to exclusive cpu) PASS: BestEffort pod with isolcpus specified (Pod is affined to isolcpus) PASS: Burstable pod with isolcpus specified (Pod is affined to isolcpus) PASS: Guaranteed pod with isolcpus specified (integer CPU) (Pod is affined to exclusive cpu) PASS: Guaranteed pod with isolcpus specified (fractional CPU) (Pod is affined to defaultCpuSet) Closes-Bug: 2042795 Change-Id: I54ac47d1ec697a53999299cfa84093cb5fd6477a Signed-off-by: Boovan Rajendran --- ...-guaranteed-pod-to-non-isolated-CPUs.patch | 34 +++++++++++++++++++ .../debian/deb_folder/patches/series | 1 + ...-guaranteed-pod-to-non-isolated-CPUs.patch | 34 +++++++++++++++++++ .../debian/deb_folder/patches/series | 1 + 4 files changed, 70 insertions(+) create mode 100644 kubernetes/kubernetes-1.26.1/debian/deb_folder/patches/Affinity-of-guaranteed-pod-to-non-isolated-CPUs.patch create mode 100644 kubernetes/kubernetes-1.27.5/debian/deb_folder/patches/Affinity-of-guaranteed-pod-to-non-isolated-CPUs.patch diff --git a/kubernetes/kubernetes-1.26.1/debian/deb_folder/patches/Affinity-of-guaranteed-pod-to-non-isolated-CPUs.patch b/kubernetes/kubernetes-1.26.1/debian/deb_folder/patches/Affinity-of-guaranteed-pod-to-non-isolated-CPUs.patch new file mode 100644 index 000000000..d13c73d17 --- /dev/null +++ b/kubernetes/kubernetes-1.26.1/debian/deb_folder/patches/Affinity-of-guaranteed-pod-to-non-isolated-CPUs.patch @@ -0,0 +1,34 @@ +From d51af14bfd2653f80d31e5cfdb60335367a4a137 Mon Sep 17 00:00:00 2001 +From: Boovan Rajendran +Date: Mon, 6 Nov 2023 02:22:16 -0500 +Subject: [PATCH] Affinity of guaranteed pod to non-isolated CPUs + +This corrects kubelet cpumanager static cpuset tracking for isolcpus +for versions 1.26.1 and 1.27.5. This ensures that pods specified with +isolcpus + guaranteed QoS + integer cpu requests, are affined to +exclusive cpuset and tracked as non-isolated cpus. + +Signed-off-by: Boovan Rajendran +--- + pkg/kubelet/cm/cpumanager/policy_static.go | 5 ++++- + 1 file changed, 4 insertions(+), 1 deletion(-) + +diff --git a/pkg/kubelet/cm/cpumanager/policy_static.go b/pkg/kubelet/cm/cpumanager/policy_static.go +index 98b8d1d882d..125429035a5 100644 +--- a/pkg/kubelet/cm/cpumanager/policy_static.go ++++ b/pkg/kubelet/cm/cpumanager/policy_static.go +@@ -317,7 +317,10 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai + return nil + } + +- if isolcpus := p.podIsolCPUs(pod, container); isolcpus.Size() > 0 { ++ cpuQuantity := container.Resources.Requests[v1.ResourceCPU] ++ fractionalCpuQuantity := cpuQuantity.MilliValue() % 1000 ++ if isolcpus := p.podIsolCPUs(pod, container); isolcpus.Size() > 0 && ++ v1qos.GetPodQOS(pod) != v1.PodQOSGuaranteed && fractionalCpuQuantity == 0{ + // container has requested isolated CPUs + if set, ok := s.GetCPUSet(string(pod.UID), container.Name); ok { + if set.Equals(isolcpus) { +-- +2.25.1 + diff --git a/kubernetes/kubernetes-1.26.1/debian/deb_folder/patches/series b/kubernetes/kubernetes-1.26.1/debian/deb_folder/patches/series index 1aa14b3be..e3a685ade 100644 --- a/kubernetes/kubernetes-1.26.1/debian/deb_folder/patches/series +++ b/kubernetes/kubernetes-1.26.1/debian/deb_folder/patches/series @@ -6,5 +6,6 @@ kubelet-cpumanager-disable-CFS-quota-throttling.patch kubelet-cpumanager-keep-normal-containers-off-reserv.patch kubelet-cpumanager-infra-pods-use-system-reserved-CP.patch kubelet-cpumanager-introduce-concept-of-isolated-CPU.patch +Affinity-of-guaranteed-pod-to-non-isolated-CPUs.patch enable-support-for-kubernetes-to-ignore-isolcpus.patch kubelet-CFS-quota-throttling-for-non-integer-cpulimit.patch diff --git a/kubernetes/kubernetes-1.27.5/debian/deb_folder/patches/Affinity-of-guaranteed-pod-to-non-isolated-CPUs.patch b/kubernetes/kubernetes-1.27.5/debian/deb_folder/patches/Affinity-of-guaranteed-pod-to-non-isolated-CPUs.patch new file mode 100644 index 000000000..d13c73d17 --- /dev/null +++ b/kubernetes/kubernetes-1.27.5/debian/deb_folder/patches/Affinity-of-guaranteed-pod-to-non-isolated-CPUs.patch @@ -0,0 +1,34 @@ +From d51af14bfd2653f80d31e5cfdb60335367a4a137 Mon Sep 17 00:00:00 2001 +From: Boovan Rajendran +Date: Mon, 6 Nov 2023 02:22:16 -0500 +Subject: [PATCH] Affinity of guaranteed pod to non-isolated CPUs + +This corrects kubelet cpumanager static cpuset tracking for isolcpus +for versions 1.26.1 and 1.27.5. This ensures that pods specified with +isolcpus + guaranteed QoS + integer cpu requests, are affined to +exclusive cpuset and tracked as non-isolated cpus. + +Signed-off-by: Boovan Rajendran +--- + pkg/kubelet/cm/cpumanager/policy_static.go | 5 ++++- + 1 file changed, 4 insertions(+), 1 deletion(-) + +diff --git a/pkg/kubelet/cm/cpumanager/policy_static.go b/pkg/kubelet/cm/cpumanager/policy_static.go +index 98b8d1d882d..125429035a5 100644 +--- a/pkg/kubelet/cm/cpumanager/policy_static.go ++++ b/pkg/kubelet/cm/cpumanager/policy_static.go +@@ -317,7 +317,10 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai + return nil + } + +- if isolcpus := p.podIsolCPUs(pod, container); isolcpus.Size() > 0 { ++ cpuQuantity := container.Resources.Requests[v1.ResourceCPU] ++ fractionalCpuQuantity := cpuQuantity.MilliValue() % 1000 ++ if isolcpus := p.podIsolCPUs(pod, container); isolcpus.Size() > 0 && ++ v1qos.GetPodQOS(pod) != v1.PodQOSGuaranteed && fractionalCpuQuantity == 0{ + // container has requested isolated CPUs + if set, ok := s.GetCPUSet(string(pod.UID), container.Name); ok { + if set.Equals(isolcpus) { +-- +2.25.1 + diff --git a/kubernetes/kubernetes-1.27.5/debian/deb_folder/patches/series b/kubernetes/kubernetes-1.27.5/debian/deb_folder/patches/series index 1aa14b3be..e3a685ade 100644 --- a/kubernetes/kubernetes-1.27.5/debian/deb_folder/patches/series +++ b/kubernetes/kubernetes-1.27.5/debian/deb_folder/patches/series @@ -6,5 +6,6 @@ kubelet-cpumanager-disable-CFS-quota-throttling.patch kubelet-cpumanager-keep-normal-containers-off-reserv.patch kubelet-cpumanager-infra-pods-use-system-reserved-CP.patch kubelet-cpumanager-introduce-concept-of-isolated-CPU.patch +Affinity-of-guaranteed-pod-to-non-isolated-CPUs.patch enable-support-for-kubernetes-to-ignore-isolcpus.patch kubelet-CFS-quota-throttling-for-non-integer-cpulimit.patch