From 424c00985e87e835522c37a18a24145225c0f371 Mon Sep 17 00:00:00 2001 From: Kaustubh Dhokte Date: Fri, 2 Feb 2024 23:33:54 +0000 Subject: [PATCH] Identify platform pods based on pod/namespace labels This change updates kubernetes patch kubelet-cpumanager-introduce-concept-of-isolated-CPU.patch for supported kubernetes versions from 1.24 to 1.28. Currently, for static CPU allocation, pods are identified as platform pods using a hard-coded list of namespaces. New method identifies a pod as a platform pod using label assigned to it or its namespace. Test Plan: PASS: All affected versions of kubernetes package build successfully. PASS: Create a pod with the platform label. Pod is classified as a platform pod. PASS: Create a pod without the platform label but in a namespace with the platform label. Pod is classified as a platform pod. PASS: Create a pod without the platform label and in a namespace without the platform label. Pod is not classified as a platform pod. Depends-On: https://review.opendev.org/c/starlingx/config/+/907640 Depends-On: https://review.opendev.org/c/starlingx/ansible-playbooks/+/907641 Depends-On: https://review.opendev.org/c/starlingx/integ/+/908340 Depends-On: https://review.opendev.org/c/starlingx/integ/+/908958 Story: 2010612 Task: 47513 Change-Id: I654d466e51522b42a2e1d17a1828288089791b8f Signed-off-by: Kaustubh Dhokte --- ...er-introduce-concept-of-isolated-CPU.patch | 320 ++++++++++++++--- ...er-introduce-concept-of-isolated-CPU.patch | 329 ++++++++++++++--- ...er-introduce-concept-of-isolated-CPU.patch | 334 +++++++++++++++--- ...er-introduce-concept-of-isolated-CPU.patch | 333 ++++++++++++++--- ...er-introduce-concept-of-isolated-CPU.patch | 330 ++++++++++++++--- 5 files changed, 1427 insertions(+), 219 deletions(-) diff --git a/kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/kubelet-cpumanager-introduce-concept-of-isolated-CPU.patch b/kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/kubelet-cpumanager-introduce-concept-of-isolated-CPU.patch index 0c2220545..bb1e65ec2 100644 --- a/kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/kubelet-cpumanager-introduce-concept-of-isolated-CPU.patch +++ b/kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/kubelet-cpumanager-introduce-concept-of-isolated-CPU.patch @@ -1,4 +1,4 @@ -From ae7fc7b39bfde784340068b388a13a28b4e76398 Mon Sep 17 00:00:00 2001 +From 66130d332c561ab95853e1277f5076f6070c3002 Mon Sep 17 00:00:00 2001 From: Gleb Aronsky Date: Tue, 25 Jan 2022 13:27:25 -0500 Subject: [PATCH] kubelet cpumanager introduce concept of isolated CPUs @@ -44,13 +44,14 @@ to the right. Co-authored-by: Jim Gauld Co-authored-by: Chris Friesen Signed-off-by: Gleb Aronsky +Signed-off-by: Kaustubh Dhokte --- - pkg/kubelet/cm/container_manager_linux.go | 1 + - pkg/kubelet/cm/cpumanager/cpu_manager.go | 35 +++++++- - pkg/kubelet/cm/cpumanager/cpu_manager_test.go | 14 +++- - pkg/kubelet/cm/cpumanager/policy_static.go | 83 +++++++++++++++++-- - .../cm/cpumanager/policy_static_test.go | 50 ++++++++--- - 5 files changed, 164 insertions(+), 19 deletions(-) + pkg/kubelet/cm/container_manager_linux.go | 1 + + pkg/kubelet/cm/cpumanager/cpu_manager.go | 35 +++- + pkg/kubelet/cm/cpumanager/cpu_manager_test.go | 14 +- + pkg/kubelet/cm/cpumanager/policy_static.go | 157 ++++++++++++++-- + .../cm/cpumanager/policy_static_test.go | 176 +++++++++++++++++- + 5 files changed, 355 insertions(+), 28 deletions(-) diff --git a/pkg/kubelet/cm/container_manager_linux.go b/pkg/kubelet/cm/container_manager_linux.go index 0f09f3eb331..770922ca55d 100644 @@ -212,18 +213,24 @@ index 2c8349662c4..31e4d0585fb 100644 testCases := []struct { description string diff --git a/pkg/kubelet/cm/cpumanager/policy_static.go b/pkg/kubelet/cm/cpumanager/policy_static.go -index a3c93a896df..d6fe69e7165 100644 +index 216b6ce9bf8..30c0afaca32 100644 --- a/pkg/kubelet/cm/cpumanager/policy_static.go +++ b/pkg/kubelet/cm/cpumanager/policy_static.go -@@ -18,6 +18,7 @@ package cpumanager +@@ -17,14 +17,21 @@ limitations under the License. + package cpumanager import ( ++ "context" "fmt" + "strconv" ++ k8sclient "k8s.io/client-go/kubernetes" ++ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ++ restclient "k8s.io/client-go/rest" v1 "k8s.io/api/core/v1" ++ "k8s.io/client-go/tools/clientcmd" "k8s.io/klog/v2" -@@ -25,6 +26,7 @@ import ( + v1qos "k8s.io/kubernetes/pkg/apis/core/v1/helper/qos" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology" "k8s.io/kubernetes/pkg/kubelet/cm/cpuset" @@ -231,7 +238,32 @@ index a3c93a896df..d6fe69e7165 100644 "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager" "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask" ) -@@ -101,6 +103,10 @@ type staticPolicy struct { +@@ -39,6 +46,12 @@ const ( + ErrorSMTAlignment = "SMTAlignmentError" + ) + ++type getPodNamespace func(string) (*v1.Namespace, error) ++type buildFromConfigFlag func(masterUrl string, kubeconfigPath string) (*restclient.Config, error) ++ ++var varGetNamespaceObject getPodNamespace ++var varBuildConfigFromFlags buildFromConfigFlag ++ + // SMTAlignmentError represents an error due to SMT alignment + type SMTAlignmentError struct { + RequestedCPUs int +@@ -53,11 +66,6 @@ func (e SMTAlignmentError) Type() string { + return ErrorSMTAlignment + } + +-// Define namespaces used by platform infrastructure pods +-var infraNamespaces = [...]string{ +- "kube-system", "armada", "cert-manager", "platform-deployment-manager", "portieris", "vault", "notification", "flux-helm", "metrics-server", "node-feature-discovery", "intel-power", "power-metrics", "sriov-fec-system", +-} +- + // staticPolicy is a CPU manager policy that does not change CPU + // assignments for exclusively pinned guaranteed containers after the main + // container process starts. +@@ -101,6 +109,10 @@ type staticPolicy struct { topology *topology.CPUTopology // set of CPUs that is not available for exclusive assignment reserved cpuset.CPUSet @@ -242,7 +274,7 @@ index a3c93a896df..d6fe69e7165 100644 // If true, default CPUSet should exclude reserved CPUs excludeReserved bool // topology manager reference to get container Topology affinity -@@ -117,7 +123,8 @@ var _ Policy = &staticPolicy{} +@@ -117,7 +129,8 @@ var _ Policy = &staticPolicy{} // NewStaticPolicy returns a CPU manager policy that does not change CPU // assignments for exclusively pinned guaranteed containers after the main // container process starts. @@ -252,7 +284,7 @@ index a3c93a896df..d6fe69e7165 100644 opts, err := NewStaticPolicyOptions(cpuPolicyOptions) if err != nil { return nil, err -@@ -128,6 +135,8 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv +@@ -128,6 +141,8 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv policy := &staticPolicy{ topology: topology, affinity: affinity, @@ -261,7 +293,7 @@ index a3c93a896df..d6fe69e7165 100644 excludeReserved: excludeReserved, cpusToReuse: make(map[string]cpuset.CPUSet), options: opts, -@@ -154,6 +163,12 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv +@@ -154,6 +169,12 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv klog.InfoS("Reserved CPUs not available for exclusive assignment", "reservedSize", reserved.Size(), "reserved", reserved) policy.reserved = reserved @@ -274,7 +306,7 @@ index a3c93a896df..d6fe69e7165 100644 return policy, nil } -@@ -187,8 +202,9 @@ func (p *staticPolicy) validateState(s state.State) error { +@@ -187,8 +208,9 @@ func (p *staticPolicy) validateState(s state.State) error { } else { s.SetDefaultCPUSet(allCPUs) } @@ -286,7 +318,17 @@ index a3c93a896df..d6fe69e7165 100644 return nil } -@@ -278,10 +294,11 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai +@@ -269,6 +291,9 @@ func (p *staticPolicy) updateCPUsToReuse(pod *v1.Pod, container *v1.Container, c + } + + func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Container) error { ++ ++ varGetNamespaceObject = getPodNamespaceObject ++ varBuildConfigFromFlags = clientcmd.BuildConfigFromFlags + // Process infra pods before guaranteed pods + if isKubeInfra(pod) { + // Container belongs in reserved pool. +@@ -278,10 +303,11 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai return nil } @@ -300,7 +342,7 @@ index a3c93a896df..d6fe69e7165 100644 } s.SetCPUSet(string(pod.UID), container.Name, cpuset) klog.Infof("[cpumanager] static policy: reserved: AddContainer (namespace: %s, pod UID: %s, pod: %s, container: %s); cpuset=%v", pod.Namespace, string(pod.UID), pod.Name, container.Name, cpuset) -@@ -325,8 +342,34 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai +@@ -325,8 +351,34 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai } s.SetCPUSet(string(pod.UID), container.Name, cpuset) p.updateCPUsToReuse(pod, container, cpuset) @@ -335,10 +377,70 @@ index a3c93a896df..d6fe69e7165 100644 // container belongs in the shared pool (nothing to do; use default cpuset) return nil } -@@ -634,3 +677,33 @@ func isKubeInfra(pod *v1.Pod) bool { - } - return false +@@ -625,12 +677,89 @@ func (p *staticPolicy) generateCPUTopologyHints(availableCPUs cpuset.CPUSet, reu + return hints } + ++func getPodNamespaceObject(podNamespaceName string) (*v1.Namespace, error) { ++ ++ cfg, err := varBuildConfigFromFlags("", "/etc/kubernetes/kubelet.conf") ++ if err != nil { ++ klog.Error("Failed to build client config from /etc/kubernetes/kubelet.conf: ", err.Error()) ++ return nil, err ++ } ++ ++ clientset, err := k8sclient.NewForConfig(cfg) ++ if err != nil { ++ klog.Error("Failed to get clientset for KUBECONFIG /etc/kubernetes/kubelet.conf: ", err.Error()) ++ return nil, err ++ } ++ ++ namespaceObj, err := clientset.CoreV1().Namespaces().Get(context.TODO(), podNamespaceName, metav1.GetOptions{}) ++ if err != nil { ++ klog.Error("Error getting namespace object:", err.Error()) ++ return nil, err ++ } ++ ++ return namespaceObj, nil ++ ++} ++ + // check if a given pod is in a platform infrastructure namespace + func isKubeInfra(pod *v1.Pod) bool { +- for _, namespace := range infraNamespaces { +- if namespace == pod.Namespace { +- return true +- } ++ ++ podName := pod.GetName() ++ podNamespaceName := pod.GetNamespace() ++ ++ klog.InfoS("Checking pod ", podName , " for label 'app.starlingx.io/component=platform'.") ++ podLabels := pod.GetLabels() ++ val, ok := podLabels["app.starlingx.io/component"] ++ if (ok && val == "platform") { ++ klog.InfoS("Pod ", podName, " has 'app.starlingx.io/component=platform' label. Assigning platform CPUs.") ++ return true ++ } ++ ++ klog.InfoS("Pod ", pod.GetName(), " does not have 'app.starlingx.io/component=platform' label. Checking its namespace information...") ++ ++ namespaceObj, err := varGetNamespaceObject(podNamespaceName) ++ if err != nil { ++ return false ++ } ++ ++ namespaceLabels := namespaceObj.GetLabels() ++ val, ok = namespaceLabels["app.starlingx.io/component"] ++ if ok && val == "platform" { ++ klog.InfoS("For pod: ", podName, ", its Namespace ", podNamespaceName, " has 'app.starlingx.io/component=platform' label. Assigning platform CPUs.") ++ return true + } ++ ++ klog.InfoS("Neither pod ", podName, " nor its namespace ", podNamespaceName, " has 'app.starlingx.io/component=platform' label. Not assigning platform CPUs.") + return false ++ ++} + +// get the isolated CPUs (if any) from the devices associated with a specific container +func (p *staticPolicy) podIsolCPUs(pod *v1.Pod, container *v1.Container) cpuset.CPUSet { @@ -368,12 +470,24 @@ index a3c93a896df..d6fe69e7165 100644 + } + } + return cpuSet -+} + } diff --git a/pkg/kubelet/cm/cpumanager/policy_static_test.go b/pkg/kubelet/cm/cpumanager/policy_static_test.go -index d4b4b790210..ecd3e9598d0 100644 +index 7938f787a57..fa3e99fb2bb 100644 --- a/pkg/kubelet/cm/cpumanager/policy_static_test.go +++ b/pkg/kubelet/cm/cpumanager/policy_static_test.go -@@ -25,6 +25,7 @@ import ( +@@ -17,14 +17,19 @@ limitations under the License. + package cpumanager + + import ( ++ "errors" + "fmt" + "reflect" + "testing" + + v1 "k8s.io/api/core/v1" ++ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ++ restclient "k8s.io/client-go/rest" ++ "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology" "k8s.io/kubernetes/pkg/kubelet/cm/cpuset" @@ -381,7 +495,7 @@ index d4b4b790210..ecd3e9598d0 100644 "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager" "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask" ) -@@ -65,8 +66,9 @@ func (spt staticPolicyTest) PseudoClone() staticPolicyTest { +@@ -65,8 +70,9 @@ func (spt staticPolicyTest) PseudoClone() staticPolicyTest { } func TestStaticPolicyName(t *testing.T) { @@ -392,7 +506,7 @@ index d4b4b790210..ecd3e9598d0 100644 policyName := policy.Name() if policyName != "static" { -@@ -76,6 +78,7 @@ func TestStaticPolicyName(t *testing.T) { +@@ -76,6 +82,7 @@ func TestStaticPolicyName(t *testing.T) { } func TestStaticPolicyStart(t *testing.T) { @@ -400,7 +514,7 @@ index d4b4b790210..ecd3e9598d0 100644 testCases := []staticPolicyTest{ { description: "non-corrupted state", -@@ -151,7 +154,7 @@ func TestStaticPolicyStart(t *testing.T) { +@@ -151,7 +158,7 @@ func TestStaticPolicyStart(t *testing.T) { } for _, testCase := range testCases { t.Run(testCase.description, func(t *testing.T) { @@ -409,7 +523,7 @@ index d4b4b790210..ecd3e9598d0 100644 policy := p.(*staticPolicy) st := &mockState{ -@@ -199,7 +202,7 @@ func TestStaticPolicyAdd(t *testing.T) { +@@ -199,7 +206,7 @@ func TestStaticPolicyAdd(t *testing.T) { largeTopoCPUSet := largeTopoBuilder.Result() largeTopoSock0CPUSet := largeTopoSock0Builder.Result() largeTopoSock1CPUSet := largeTopoSock1Builder.Result() @@ -418,7 +532,7 @@ index d4b4b790210..ecd3e9598d0 100644 // these are the cases which must behave the same regardless the policy options. // So we will permutate the options to ensure this holds true. optionsInsensitiveTestCases := []staticPolicyTest{ -@@ -529,8 +532,9 @@ func TestStaticPolicyAdd(t *testing.T) { +@@ -529,8 +536,9 @@ func TestStaticPolicyAdd(t *testing.T) { } func runStaticPolicyTestCase(t *testing.T, testCase staticPolicyTest) { @@ -429,7 +543,7 @@ index d4b4b790210..ecd3e9598d0 100644 st := &mockState{ assignments: testCase.stAssignments, -@@ -596,7 +600,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) { +@@ -596,7 +604,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) { } for _, testCase := range testCases { @@ -438,7 +552,7 @@ index d4b4b790210..ecd3e9598d0 100644 st := &mockState{ assignments: testCase.stAssignments, -@@ -629,6 +633,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) { +@@ -629,6 +637,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) { func TestStaticPolicyRemove(t *testing.T) { excludeReserved := false @@ -446,7 +560,7 @@ index d4b4b790210..ecd3e9598d0 100644 testCases := []staticPolicyTest{ { description: "SingleSocketHT, DeAllocOneContainer", -@@ -710,6 +715,7 @@ func TestStaticPolicyRemove(t *testing.T) { +@@ -710,6 +719,7 @@ func TestStaticPolicyRemove(t *testing.T) { func TestTopologyAwareAllocateCPUs(t *testing.T) { excludeReserved := false @@ -454,7 +568,7 @@ index d4b4b790210..ecd3e9598d0 100644 testCases := []struct { description string topo *topology.CPUTopology -@@ -778,7 +784,8 @@ func TestTopologyAwareAllocateCPUs(t *testing.T) { +@@ -778,7 +788,8 @@ func TestTopologyAwareAllocateCPUs(t *testing.T) { }, } for _, tc := range testCases { @@ -464,7 +578,7 @@ index d4b4b790210..ecd3e9598d0 100644 policy := p.(*staticPolicy) st := &mockState{ assignments: tc.stAssignments, -@@ -811,6 +818,7 @@ type staticPolicyTestWithResvList struct { +@@ -811,6 +822,7 @@ type staticPolicyTestWithResvList struct { topo *topology.CPUTopology numReservedCPUs int reserved cpuset.CPUSet @@ -472,7 +586,7 @@ index d4b4b790210..ecd3e9598d0 100644 stAssignments state.ContainerCPUAssignments stDefaultCPUSet cpuset.CPUSet pod *v1.Pod -@@ -821,6 +829,8 @@ type staticPolicyTestWithResvList struct { +@@ -821,6 +833,8 @@ type staticPolicyTestWithResvList struct { } func TestStaticPolicyStartWithResvList(t *testing.T) { @@ -481,7 +595,7 @@ index d4b4b790210..ecd3e9598d0 100644 testCases := []staticPolicyTestWithResvList{ { description: "empty cpuset", -@@ -850,11 +860,9 @@ func TestStaticPolicyStartWithResvList(t *testing.T) { +@@ -850,11 +864,9 @@ func TestStaticPolicyStartWithResvList(t *testing.T) { expNewErr: fmt.Errorf("[cpumanager] unable to reserve the required amount of CPUs (size of 0-1 did not equal 1)"), }, } @@ -494,7 +608,7 @@ index d4b4b790210..ecd3e9598d0 100644 if !reflect.DeepEqual(err, testCase.expNewErr) { t.Errorf("StaticPolicy Start() error (%v). expected error: %v but got: %v", testCase.description, testCase.expNewErr, err) -@@ -894,6 +902,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { +@@ -894,6 +906,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { topo: topoSingleSocketHT, numReservedCPUs: 1, reserved: cpuset.NewCPUSet(0), @@ -502,7 +616,7 @@ index d4b4b790210..ecd3e9598d0 100644 stAssignments: state.ContainerCPUAssignments{}, stDefaultCPUSet: cpuset.NewCPUSet(1, 2, 3, 4, 5, 6, 7), pod: makePod("fakePod", "fakeContainer2", "8000m", "8000m"), -@@ -906,6 +915,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { +@@ -906,6 +919,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { topo: topoSingleSocketHT, numReservedCPUs: 2, reserved: cpuset.NewCPUSet(0, 1), @@ -510,7 +624,7 @@ index d4b4b790210..ecd3e9598d0 100644 stAssignments: state.ContainerCPUAssignments{}, stDefaultCPUSet: cpuset.NewCPUSet(2, 3, 4, 5, 6, 7), pod: makePod("fakePod", "fakeContainer2", "1000m", "1000m"), -@@ -918,6 +928,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { +@@ -918,6 +932,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { topo: topoSingleSocketHT, numReservedCPUs: 2, reserved: cpuset.NewCPUSet(0, 1), @@ -518,7 +632,7 @@ index d4b4b790210..ecd3e9598d0 100644 stAssignments: state.ContainerCPUAssignments{ "fakePod": map[string]cpuset.CPUSet{ "fakeContainer100": cpuset.NewCPUSet(2, 3, 6, 7), -@@ -934,6 +945,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { +@@ -934,6 +949,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { topo: topoSingleSocketHT, numReservedCPUs: 2, reserved: cpuset.NewCPUSet(0, 1), @@ -526,7 +640,7 @@ index d4b4b790210..ecd3e9598d0 100644 stAssignments: state.ContainerCPUAssignments{ "fakePod": map[string]cpuset.CPUSet{ "fakeContainer100": cpuset.NewCPUSet(2, 3, 6, 7), -@@ -945,11 +957,29 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { +@@ -945,11 +961,29 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { expCPUAlloc: true, expCSet: cpuset.NewCPUSet(0, 1), }, @@ -557,6 +671,132 @@ index d4b4b790210..ecd3e9598d0 100644 st := &mockState{ assignments: testCase.stAssignments, +@@ -1075,3 +1109,125 @@ func TestStaticPolicyOptions(t *testing.T) { + }) + } + } ++ ++func makePodWithLabels(podLabels map[string]string) *v1.Pod { ++ return &v1.Pod{ ++ ObjectMeta: metav1.ObjectMeta{ ++ Name: "test-pod", ++ Namespace: "test-namespace", ++ Labels: podLabels, ++ }, ++ } ++} ++ ++func fakeBuildConfigFromFlags(masterUrl string, kubeconfigPath string) (*restclient.Config, error) { ++ ++ return &restclient.Config{}, nil ++} ++ ++func fakeBuildConfigFromFlagsError(masterUrl string, kubeconfigPath string) (*restclient.Config, error) { ++ ++ errString := fmt.Sprintf("%s file not found", kubeconfigPath) ++ return nil, errors.New(errString) ++ ++} ++ ++func getFakeInfraPodNamespace(_ string) (*v1.Namespace, error) { ++ ++ return &v1.Namespace{ ++ ObjectMeta: metav1.ObjectMeta{ ++ Name: "test-namespace", ++ Labels: map[string]string{ ++ "app.starlingx.io/component": "platform", ++ }, ++ }}, nil ++} ++ ++func getFakeNonInfraPodNamespace(_ string) (*v1.Namespace, error) { ++ ++ return &v1.Namespace{ ++ ObjectMeta: metav1.ObjectMeta{ ++ Name: "test-namespace", ++ Labels: map[string]string{ ++ "fake": "label", ++ }}}, nil ++ ++} ++ ++type kubeInfraPodTestCase struct { ++ description string ++ pod *v1.Pod ++ namespaceFunc getPodNamespace ++ expectedValue bool ++} ++ ++func TestKubeInfraPod(t *testing.T) { ++ testCases := []kubeInfraPodTestCase{ ++ { ++ description: "Pod with platform label and namespace without platform label", ++ pod: makePodWithLabels(map[string]string{ ++ "app.starlingx.io/component": "platform", ++ }), ++ namespaceFunc: getFakeNonInfraPodNamespace, ++ expectedValue: true, ++ ++ }, ++ { ++ description: "Pod without platform label and namespace with platform label", ++ pod: makePodWithLabels(map[string]string{ ++ "test": "label", ++ }), ++ namespaceFunc: getFakeInfraPodNamespace, ++ expectedValue: true, ++ }, ++ { ++ description: "Pod without platform label and namespace without platform label", ++ pod: makePodWithLabels(map[string]string{ ++ "test": "namespace", ++ }), ++ namespaceFunc: getFakeNonInfraPodNamespace, ++ expectedValue: false, ++ }, ++ ++ } ++ ++ for _, testCase := range testCases { ++ t.Run(testCase.description, func(t *testing.T) { ++ ++ varGetNamespaceObject = testCase.namespaceFunc ++ varBuildConfigFromFlags = fakeBuildConfigFromFlags ++ gotValue := isKubeInfra(testCase.pod) ++ ++ if gotValue != testCase.expectedValue { ++ t.Errorf("StaticPolicy isKubeInfraPod() error %v. expected value %v actual value %v", ++ testCase.description, testCase.expectedValue, gotValue) ++ } else { ++ fmt.Printf("StaticPolicy isKubeInfraPod() test successful. : %v ", testCase.description) ++ } ++ ++ }) ++ } ++ ++ test := kubeInfraPodTestCase{ ++ description: "Failure reading kubeconfig file", ++ pod: makePodWithLabels(map[string]string{ ++ "test": "namespace", ++ }), ++ namespaceFunc: getFakeNonInfraPodNamespace, ++ expectedValue: false, ++ } ++ ++ varGetNamespaceObject = getPodNamespaceObject ++ varBuildConfigFromFlags = fakeBuildConfigFromFlagsError ++ ++ gotValue := isKubeInfra(test.pod) ++ ++ if gotValue != test.expectedValue { ++ t.Errorf("StaticPolicy isKubeInfraPod() error %v. expected value %v actual value %v", ++ test.description, test.expectedValue, gotValue) ++ } else { ++ fmt.Printf("StaticPolicy isKubeInfraPod() test successful. : %v ", test.description) ++ } ++ ++} ++ -- 2.25.1 diff --git a/kubernetes/kubernetes-1.25.3/debian/deb_folder/patches/kubelet-cpumanager-introduce-concept-of-isolated-CPU.patch b/kubernetes/kubernetes-1.25.3/debian/deb_folder/patches/kubelet-cpumanager-introduce-concept-of-isolated-CPU.patch index 2860956b3..7767cf620 100644 --- a/kubernetes/kubernetes-1.25.3/debian/deb_folder/patches/kubelet-cpumanager-introduce-concept-of-isolated-CPU.patch +++ b/kubernetes/kubernetes-1.25.3/debian/deb_folder/patches/kubelet-cpumanager-introduce-concept-of-isolated-CPU.patch @@ -1,4 +1,4 @@ -From e8608eb7c300b6e0503885a4848fafc75f20d909 Mon Sep 17 00:00:00 2001 +From 27f0c5ba4cdd5f8f850fa4a5b110a39eaba7cd65 Mon Sep 17 00:00:00 2001 From: Ramesh Kumar Sivanandam Date: Mon, 7 Nov 2022 13:33:03 -0500 Subject: [PATCH] kubelet cpumanager introduce concept of isolated CPUs @@ -45,13 +45,14 @@ Co-authored-by: Jim Gauld Co-authored-by: Chris Friesen Signed-off-by: Gleb Aronsky Signed-off-by: Ramesh Kumar Sivanandam +Signed-off-by: Kaustubh Dhokte --- - pkg/kubelet/cm/container_manager_linux.go | 1 + - pkg/kubelet/cm/cpumanager/cpu_manager.go | 35 +++++++- - pkg/kubelet/cm/cpumanager/cpu_manager_test.go | 20 ++++- - pkg/kubelet/cm/cpumanager/policy_static.go | 83 +++++++++++++++++-- - .../cm/cpumanager/policy_static_test.go | 53 +++++++++--- - 5 files changed, 172 insertions(+), 20 deletions(-) + pkg/kubelet/cm/container_manager_linux.go | 1 + + pkg/kubelet/cm/cpumanager/cpu_manager.go | 35 +++- + pkg/kubelet/cm/cpumanager/cpu_manager_test.go | 20 +- + pkg/kubelet/cm/cpumanager/policy_static.go | 158 ++++++++++++++-- + .../cm/cpumanager/policy_static_test.go | 177 +++++++++++++++++- + 5 files changed, 362 insertions(+), 29 deletions(-) diff --git a/pkg/kubelet/cm/container_manager_linux.go b/pkg/kubelet/cm/container_manager_linux.go index 44c8cda6c40..a3f92b23c69 100644 @@ -242,18 +243,24 @@ index d553b182e0b..57f3f9a1c97 100644 t.Errorf("Expected error, but NewManager succeeded") } diff --git a/pkg/kubelet/cm/cpumanager/policy_static.go b/pkg/kubelet/cm/cpumanager/policy_static.go -index 341e9f3dffe..802e289bfaf 100644 +index 4fc96303622..4631841fe01 100644 --- a/pkg/kubelet/cm/cpumanager/policy_static.go +++ b/pkg/kubelet/cm/cpumanager/policy_static.go -@@ -18,6 +18,7 @@ package cpumanager +@@ -17,14 +17,21 @@ limitations under the License. + package cpumanager import ( ++ "context" "fmt" + "strconv" ++ k8sclient "k8s.io/client-go/kubernetes" ++ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ++ restclient "k8s.io/client-go/rest" v1 "k8s.io/api/core/v1" ++ "k8s.io/client-go/tools/clientcmd" "k8s.io/klog/v2" -@@ -25,6 +26,7 @@ import ( + v1qos "k8s.io/kubernetes/pkg/apis/core/v1/helper/qos" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology" "k8s.io/kubernetes/pkg/kubelet/cm/cpuset" @@ -261,7 +268,33 @@ index 341e9f3dffe..802e289bfaf 100644 "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager" "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask" ) -@@ -101,6 +103,10 @@ type staticPolicy struct { +@@ -39,6 +46,13 @@ const ( + ErrorSMTAlignment = "SMTAlignmentError" + ) + ++type getPodNamespace func(string) (*v1.Namespace, error) ++type buildFromConfigFlag func(masterUrl string, kubeconfigPath string) (*restclient.Config, error) ++ ++var varGetNamespaceObject getPodNamespace ++var varBuildConfigFromFlags buildFromConfigFlag ++ ++ + // SMTAlignmentError represents an error due to SMT alignment + type SMTAlignmentError struct { + RequestedCPUs int +@@ -53,11 +67,6 @@ func (e SMTAlignmentError) Type() string { + return ErrorSMTAlignment + } + +-// Define namespaces used by platform infrastructure pods +-var infraNamespaces = [...]string{ +- "kube-system", "armada", "cert-manager", "platform-deployment-manager", "portieris", "vault", "notification", "flux-helm", "metrics-server", "node-feature-discovery", "intel-power", "power-metrics", "sriov-fec-system", +-} +- + // staticPolicy is a CPU manager policy that does not change CPU + // assignments for exclusively pinned guaranteed containers after the main + // container process starts. +@@ -101,6 +110,10 @@ type staticPolicy struct { topology *topology.CPUTopology // set of CPUs that is not available for exclusive assignment reserved cpuset.CPUSet @@ -272,7 +305,7 @@ index 341e9f3dffe..802e289bfaf 100644 // If true, default CPUSet should exclude reserved CPUs excludeReserved bool // topology manager reference to get container Topology affinity -@@ -117,7 +123,8 @@ var _ Policy = &staticPolicy{} +@@ -117,7 +130,8 @@ var _ Policy = &staticPolicy{} // NewStaticPolicy returns a CPU manager policy that does not change CPU // assignments for exclusively pinned guaranteed containers after the main // container process starts. @@ -282,7 +315,7 @@ index 341e9f3dffe..802e289bfaf 100644 opts, err := NewStaticPolicyOptions(cpuPolicyOptions) if err != nil { return nil, err -@@ -132,6 +139,8 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv +@@ -132,6 +146,8 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv policy := &staticPolicy{ topology: topology, affinity: affinity, @@ -291,7 +324,7 @@ index 341e9f3dffe..802e289bfaf 100644 excludeReserved: excludeReserved, cpusToReuse: make(map[string]cpuset.CPUSet), options: opts, -@@ -158,6 +167,12 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv +@@ -158,6 +174,12 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv klog.InfoS("Reserved CPUs not available for exclusive assignment", "reservedSize", reserved.Size(), "reserved", reserved) policy.reserved = reserved @@ -304,7 +337,7 @@ index 341e9f3dffe..802e289bfaf 100644 return policy, nil } -@@ -191,8 +206,9 @@ func (p *staticPolicy) validateState(s state.State) error { +@@ -191,8 +213,9 @@ func (p *staticPolicy) validateState(s state.State) error { } else { s.SetDefaultCPUSet(allCPUs) } @@ -316,7 +349,17 @@ index 341e9f3dffe..802e289bfaf 100644 return nil } -@@ -282,10 +298,11 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai +@@ -273,6 +296,9 @@ func (p *staticPolicy) updateCPUsToReuse(pod *v1.Pod, container *v1.Container, c + } + + func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Container) error { ++ ++ varGetNamespaceObject = getPodNamespaceObject ++ varBuildConfigFromFlags = clientcmd.BuildConfigFromFlags + // Process infra pods before guaranteed pods + if isKubeInfra(pod) { + // Container belongs in reserved pool. +@@ -282,10 +308,11 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai return nil } @@ -330,7 +373,7 @@ index 341e9f3dffe..802e289bfaf 100644 } s.SetCPUSet(string(pod.UID), container.Name, cpuset) klog.Infof("[cpumanager] static policy: reserved: AddContainer (namespace: %s, pod UID: %s, pod: %s, container: %s); cpuset=%v", pod.Namespace, string(pod.UID), pod.Name, container.Name, cpuset) -@@ -329,8 +346,34 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai +@@ -329,8 +356,34 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai } s.SetCPUSet(string(pod.UID), container.Name, cpuset) p.updateCPUsToReuse(pod, container, cpuset) @@ -365,10 +408,71 @@ index 341e9f3dffe..802e289bfaf 100644 // container belongs in the shared pool (nothing to do; use default cpuset) return nil } -@@ -640,6 +683,36 @@ func isKubeInfra(pod *v1.Pod) bool { - return false +@@ -630,14 +683,91 @@ func (p *staticPolicy) generateCPUTopologyHints(availableCPUs cpuset.CPUSet, reu + return hints } ++func getPodNamespaceObject(podNamespaceName string) (*v1.Namespace, error) { ++ ++ cfg, err := varBuildConfigFromFlags("", "/etc/kubernetes/kubelet.conf") ++ if err != nil { ++ klog.Error("Failed to build client config from /etc/kubernetes/kubelet.conf: ", err.Error()) ++ return nil, err ++ } ++ ++ clientset, err := k8sclient.NewForConfig(cfg) ++ if err != nil { ++ klog.Error("Failed to get clientset for KUBECONFIG /etc/kubernetes/kubelet.conf: ", err.Error()) ++ return nil, err ++ } ++ ++ namespaceObj, err := clientset.CoreV1().Namespaces().Get(context.TODO(), podNamespaceName, metav1.GetOptions{}) ++ if err != nil { ++ klog.Error("Error getting namespace object:", err.Error()) ++ return nil, err ++ } ++ ++ return namespaceObj, nil ++ ++} ++ + // check if a given pod is in a platform infrastructure namespace + func isKubeInfra(pod *v1.Pod) bool { +- for _, namespace := range infraNamespaces { +- if namespace == pod.Namespace { +- return true +- } ++ ++ podName := pod.GetName() ++ podNamespaceName := pod.GetNamespace() ++ ++ klog.InfoS("Checking pod ", podName , " for label 'app.starlingx.io/component=platform'.") ++ podLabels := pod.GetLabels() ++ val, ok := podLabels["app.starlingx.io/component"] ++ if (ok && val == "platform") { ++ klog.InfoS("Pod ", podName, " has 'app.starlingx.io/component=platform' label. Assigning platform CPUs.") ++ return true + } ++ ++ klog.InfoS("Pod ", pod.GetName(), " does not have 'app.starlingx.io/component=platform' label. Checking its namespace information...") ++ ++ namespaceObj, err := varGetNamespaceObject(podNamespaceName) ++ if err != nil { ++ return false ++ } ++ ++ namespaceLabels := namespaceObj.GetLabels() ++ val, ok = namespaceLabels["app.starlingx.io/component"] ++ if ok && val == "platform" { ++ klog.InfoS("For pod: ", podName, ", its Namespace ", podNamespaceName, " has 'app.starlingx.io/component=platform' label. Assigning platform CPUs.") ++ return true ++ } ++ ++ klog.InfoS("Neither pod ", podName, " nor its namespace ", podNamespaceName, " has 'app.starlingx.io/component=platform' label. Not assigning platform CPUs.") + return false ++ ++} ++ +// get the isolated CPUs (if any) from the devices associated with a specific container +func (p *staticPolicy) podIsolCPUs(pod *v1.Pod, container *v1.Container) cpuset.CPUSet { + // NOTE: This is required for TestStaticPolicyAdd() since makePod() does @@ -397,16 +501,28 @@ index 341e9f3dffe..802e289bfaf 100644 + } + } + return cpuSet -+} -+ + } + // isHintSocketAligned function return true if numa nodes in hint are socket aligned. - func (p *staticPolicy) isHintSocketAligned(hint topologymanager.TopologyHint, minAffinitySize int) bool { - numaNodesBitMask := hint.NUMANodeAffinity.GetBits() diff --git a/pkg/kubelet/cm/cpumanager/policy_static_test.go b/pkg/kubelet/cm/cpumanager/policy_static_test.go -index 414e5ce144c..1c43df3b85f 100644 +index 414e5ce144c..b6aad48576f 100644 --- a/pkg/kubelet/cm/cpumanager/policy_static_test.go +++ b/pkg/kubelet/cm/cpumanager/policy_static_test.go -@@ -28,6 +28,7 @@ import ( +@@ -17,10 +17,13 @@ limitations under the License. + package cpumanager + + import ( ++ "errors" + "fmt" + "reflect" + "testing" + ++ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ++ restclient "k8s.io/client-go/rest" + v1 "k8s.io/api/core/v1" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" +@@ -28,6 +31,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology" "k8s.io/kubernetes/pkg/kubelet/cm/cpuset" @@ -414,7 +530,7 @@ index 414e5ce144c..1c43df3b85f 100644 "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager" "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask" ) -@@ -69,8 +70,9 @@ func (spt staticPolicyTest) PseudoClone() staticPolicyTest { +@@ -69,8 +73,9 @@ func (spt staticPolicyTest) PseudoClone() staticPolicyTest { } func TestStaticPolicyName(t *testing.T) { @@ -425,7 +541,7 @@ index 414e5ce144c..1c43df3b85f 100644 policyName := policy.Name() if policyName != "static" { -@@ -80,6 +82,7 @@ func TestStaticPolicyName(t *testing.T) { +@@ -80,6 +85,7 @@ func TestStaticPolicyName(t *testing.T) { } func TestStaticPolicyStart(t *testing.T) { @@ -433,7 +549,7 @@ index 414e5ce144c..1c43df3b85f 100644 testCases := []staticPolicyTest{ { description: "non-corrupted state", -@@ -155,7 +158,7 @@ func TestStaticPolicyStart(t *testing.T) { +@@ -155,7 +161,7 @@ func TestStaticPolicyStart(t *testing.T) { } for _, testCase := range testCases { t.Run(testCase.description, func(t *testing.T) { @@ -442,7 +558,7 @@ index 414e5ce144c..1c43df3b85f 100644 policy := p.(*staticPolicy) st := &mockState{ -@@ -203,7 +206,6 @@ func TestStaticPolicyAdd(t *testing.T) { +@@ -203,7 +209,6 @@ func TestStaticPolicyAdd(t *testing.T) { largeTopoCPUSet := largeTopoBuilder.Result() largeTopoSock0CPUSet := largeTopoSock0Builder.Result() largeTopoSock1CPUSet := largeTopoSock1Builder.Result() @@ -450,7 +566,7 @@ index 414e5ce144c..1c43df3b85f 100644 // these are the cases which must behave the same regardless the policy options. // So we will permutate the options to ensure this holds true. -@@ -577,8 +579,10 @@ func runStaticPolicyTestCase(t *testing.T, testCase staticPolicyTest) { +@@ -577,8 +582,10 @@ func runStaticPolicyTestCase(t *testing.T, testCase staticPolicyTest) { if testCase.topologyHint != nil { tm = topologymanager.NewFakeManagerWithHint(testCase.topologyHint) } @@ -462,7 +578,7 @@ index 414e5ce144c..1c43df3b85f 100644 st := &mockState{ assignments: testCase.stAssignments, -@@ -625,6 +629,8 @@ func runStaticPolicyTestCaseWithFeatureGate(t *testing.T, testCase staticPolicyT +@@ -625,6 +632,8 @@ func runStaticPolicyTestCaseWithFeatureGate(t *testing.T, testCase staticPolicyT } func TestStaticPolicyReuseCPUs(t *testing.T) { @@ -471,7 +587,7 @@ index 414e5ce144c..1c43df3b85f 100644 testCases := []struct { staticPolicyTest expCSetAfterAlloc cpuset.CPUSet -@@ -649,7 +655,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) { +@@ -649,7 +658,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) { } for _, testCase := range testCases { @@ -480,7 +596,7 @@ index 414e5ce144c..1c43df3b85f 100644 st := &mockState{ assignments: testCase.stAssignments, -@@ -682,6 +688,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) { +@@ -682,6 +691,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) { func TestStaticPolicyRemove(t *testing.T) { excludeReserved := false @@ -488,7 +604,7 @@ index 414e5ce144c..1c43df3b85f 100644 testCases := []staticPolicyTest{ { description: "SingleSocketHT, DeAllocOneContainer", -@@ -740,7 +747,7 @@ func TestStaticPolicyRemove(t *testing.T) { +@@ -740,7 +750,7 @@ func TestStaticPolicyRemove(t *testing.T) { } for _, testCase := range testCases { @@ -497,7 +613,7 @@ index 414e5ce144c..1c43df3b85f 100644 st := &mockState{ assignments: testCase.stAssignments, -@@ -763,6 +770,7 @@ func TestStaticPolicyRemove(t *testing.T) { +@@ -763,6 +773,7 @@ func TestStaticPolicyRemove(t *testing.T) { func TestTopologyAwareAllocateCPUs(t *testing.T) { excludeReserved := false @@ -505,7 +621,7 @@ index 414e5ce144c..1c43df3b85f 100644 testCases := []struct { description string topo *topology.CPUTopology -@@ -831,7 +839,8 @@ func TestTopologyAwareAllocateCPUs(t *testing.T) { +@@ -831,7 +842,8 @@ func TestTopologyAwareAllocateCPUs(t *testing.T) { }, } for _, tc := range testCases { @@ -515,7 +631,7 @@ index 414e5ce144c..1c43df3b85f 100644 policy := p.(*staticPolicy) st := &mockState{ assignments: tc.stAssignments, -@@ -864,6 +873,7 @@ type staticPolicyTestWithResvList struct { +@@ -864,6 +876,7 @@ type staticPolicyTestWithResvList struct { topo *topology.CPUTopology numReservedCPUs int reserved cpuset.CPUSet @@ -523,7 +639,7 @@ index 414e5ce144c..1c43df3b85f 100644 stAssignments state.ContainerCPUAssignments stDefaultCPUSet cpuset.CPUSet pod *v1.Pod -@@ -874,6 +884,8 @@ type staticPolicyTestWithResvList struct { +@@ -874,6 +887,8 @@ type staticPolicyTestWithResvList struct { } func TestStaticPolicyStartWithResvList(t *testing.T) { @@ -532,7 +648,7 @@ index 414e5ce144c..1c43df3b85f 100644 testCases := []staticPolicyTestWithResvList{ { description: "empty cpuset", -@@ -903,11 +915,10 @@ func TestStaticPolicyStartWithResvList(t *testing.T) { +@@ -903,11 +918,10 @@ func TestStaticPolicyStartWithResvList(t *testing.T) { expNewErr: fmt.Errorf("[cpumanager] unable to reserve the required amount of CPUs (size of 0-1 did not equal 1)"), }, } @@ -545,7 +661,7 @@ index 414e5ce144c..1c43df3b85f 100644 if !reflect.DeepEqual(err, testCase.expNewErr) { t.Errorf("StaticPolicy Start() error (%v). expected error: %v but got: %v", testCase.description, testCase.expNewErr, err) -@@ -947,6 +958,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { +@@ -947,6 +961,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { topo: topoSingleSocketHT, numReservedCPUs: 1, reserved: cpuset.NewCPUSet(0), @@ -553,7 +669,7 @@ index 414e5ce144c..1c43df3b85f 100644 stAssignments: state.ContainerCPUAssignments{}, stDefaultCPUSet: cpuset.NewCPUSet(1, 2, 3, 4, 5, 6, 7), pod: makePod("fakePod", "fakeContainer2", "8000m", "8000m"), -@@ -959,6 +971,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { +@@ -959,6 +974,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { topo: topoSingleSocketHT, numReservedCPUs: 2, reserved: cpuset.NewCPUSet(0, 1), @@ -561,7 +677,7 @@ index 414e5ce144c..1c43df3b85f 100644 stAssignments: state.ContainerCPUAssignments{}, stDefaultCPUSet: cpuset.NewCPUSet(2, 3, 4, 5, 6, 7), pod: makePod("fakePod", "fakeContainer2", "1000m", "1000m"), -@@ -971,6 +984,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { +@@ -971,6 +987,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { topo: topoSingleSocketHT, numReservedCPUs: 2, reserved: cpuset.NewCPUSet(0, 1), @@ -569,7 +685,7 @@ index 414e5ce144c..1c43df3b85f 100644 stAssignments: state.ContainerCPUAssignments{ "fakePod": map[string]cpuset.CPUSet{ "fakeContainer100": cpuset.NewCPUSet(2, 3, 6, 7), -@@ -987,6 +1001,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { +@@ -987,6 +1004,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { topo: topoSingleSocketHT, numReservedCPUs: 2, reserved: cpuset.NewCPUSet(0, 1), @@ -577,7 +693,7 @@ index 414e5ce144c..1c43df3b85f 100644 stAssignments: state.ContainerCPUAssignments{ "fakePod": map[string]cpuset.CPUSet{ "fakeContainer100": cpuset.NewCPUSet(2, 3, 6, 7), -@@ -998,11 +1013,29 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { +@@ -998,11 +1016,29 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { expCPUAlloc: true, expCSet: cpuset.NewCPUSet(0, 1), }, @@ -608,6 +724,131 @@ index 414e5ce144c..1c43df3b85f 100644 st := &mockState{ assignments: testCase.stAssignments, +@@ -1128,3 +1164,124 @@ func TestStaticPolicyOptions(t *testing.T) { + }) + } + } ++ ++func makePodWithLabels(podLabels map[string]string) *v1.Pod { ++ return &v1.Pod{ ++ ObjectMeta: metav1.ObjectMeta{ ++ Name: "test-pod", ++ Namespace: "test-namespace", ++ Labels: podLabels, ++ }, ++ } ++} ++ ++func fakeBuildConfigFromFlags(masterUrl string, kubeconfigPath string) (*restclient.Config, error) { ++ ++ return &restclient.Config{}, nil ++} ++ ++func fakeBuildConfigFromFlagsError(masterUrl string, kubeconfigPath string) (*restclient.Config, error) { ++ ++ errString := fmt.Sprintf("%s file not found", kubeconfigPath) ++ return nil, errors.New(errString) ++ ++} ++ ++func getFakeInfraPodNamespace(_ string) (*v1.Namespace, error) { ++ ++ return &v1.Namespace{ ++ ObjectMeta: metav1.ObjectMeta{ ++ Name: "test-namespace", ++ Labels: map[string]string{ ++ "app.starlingx.io/component": "platform", ++ }, ++ }}, nil ++} ++ ++func getFakeNonInfraPodNamespace(_ string) (*v1.Namespace, error) { ++ ++ return &v1.Namespace{ ++ ObjectMeta: metav1.ObjectMeta{ ++ Name: "test-namespace", ++ Labels: map[string]string{ ++ "fake": "label", ++ }}}, nil ++ ++} ++ ++type kubeInfraPodTestCase struct { ++ description string ++ pod *v1.Pod ++ namespaceFunc getPodNamespace ++ expectedValue bool ++} ++ ++func TestKubeInfraPod(t *testing.T) { ++ testCases := []kubeInfraPodTestCase{ ++ { ++ description: "Pod with platform label and namespace without platform label", ++ pod: makePodWithLabels(map[string]string{ ++ "app.starlingx.io/component": "platform", ++ }), ++ namespaceFunc: getFakeNonInfraPodNamespace, ++ expectedValue: true, ++ ++ }, ++ { ++ description: "Pod without platform label and namespace with platform label", ++ pod: makePodWithLabels(map[string]string{ ++ "test": "label", ++ }), ++ namespaceFunc: getFakeInfraPodNamespace, ++ expectedValue: true, ++ }, ++ { ++ description: "Pod without platform label and namespace without platform label", ++ pod: makePodWithLabels(map[string]string{ ++ "test": "namespace", ++ }), ++ namespaceFunc: getFakeNonInfraPodNamespace, ++ expectedValue: false, ++ }, ++ ++ } ++ ++ for _, testCase := range testCases { ++ t.Run(testCase.description, func(t *testing.T) { ++ ++ varGetNamespaceObject = testCase.namespaceFunc ++ varBuildConfigFromFlags = fakeBuildConfigFromFlags ++ gotValue := isKubeInfra(testCase.pod) ++ ++ if gotValue != testCase.expectedValue { ++ t.Errorf("StaticPolicy isKubeInfraPod() error %v. expected value %v actual value %v", ++ testCase.description, testCase.expectedValue, gotValue) ++ } else { ++ fmt.Printf("StaticPolicy isKubeInfraPod() test successful. : %v ", testCase.description) ++ } ++ ++ }) ++ } ++ ++ test := kubeInfraPodTestCase{ ++ description: "Failure reading kubeconfig file", ++ pod: makePodWithLabels(map[string]string{ ++ "test": "namespace", ++ }), ++ namespaceFunc: getFakeNonInfraPodNamespace, ++ expectedValue: false, ++ } ++ ++ varGetNamespaceObject = getPodNamespaceObject ++ varBuildConfigFromFlags = fakeBuildConfigFromFlagsError ++ ++ gotValue := isKubeInfra(test.pod) ++ ++ if gotValue != test.expectedValue { ++ t.Errorf("StaticPolicy isKubeInfraPod() error %v. expected value %v actual value %v", ++ test.description, test.expectedValue, gotValue) ++ } else { ++ fmt.Printf("StaticPolicy isKubeInfraPod() test successful. : %v ", test.description) ++ } ++ ++} -- 2.25.1 diff --git a/kubernetes/kubernetes-1.26.1/debian/deb_folder/patches/kubelet-cpumanager-introduce-concept-of-isolated-CPU.patch b/kubernetes/kubernetes-1.26.1/debian/deb_folder/patches/kubelet-cpumanager-introduce-concept-of-isolated-CPU.patch index 6485b4ab3..518005c7b 100644 --- a/kubernetes/kubernetes-1.26.1/debian/deb_folder/patches/kubelet-cpumanager-introduce-concept-of-isolated-CPU.patch +++ b/kubernetes/kubernetes-1.26.1/debian/deb_folder/patches/kubelet-cpumanager-introduce-concept-of-isolated-CPU.patch @@ -1,7 +1,7 @@ -From ed1f8c6a04e7fed096eaae5081c2b5e0c3bc6fed Mon Sep 17 00:00:00 2001 +From 36170864cc9ebb2183e6301cb745e49238d21397 Mon Sep 17 00:00:00 2001 From: Ramesh Kumar Sivanandam Date: Mon, 7 Nov 2022 13:33:03 -0500 -Subject: [PATCH 08/10] kubelet cpumanager introduce concept of isolated CPUs +Subject: [PATCH] kubelet cpumanager introduce concept of isolated CPUs This introduces the concept of "isolated CPUs", which are CPUs that have been isolated at the kernel level via the "isolcpus" kernel boot @@ -46,14 +46,15 @@ Co-authored-by: Chris Friesen Signed-off-by: Gleb Aronsky Signed-off-by: Ramesh Kumar Sivanandam Signed-off-by: Sachin Gopala Krishna +Signed-off-by: Kaustubh Dhokte --- - pkg/kubelet/cm/container_manager_linux.go | 1 + - pkg/kubelet/cm/cpumanager/cpu_manager.go | 35 ++++++- - pkg/kubelet/cm/cpumanager/cpu_manager_test.go | 23 ++++- - pkg/kubelet/cm/cpumanager/policy_static.go | 83 ++++++++++++++-- - .../cm/cpumanager/policy_static_test.go | 53 ++++++++-- - pkg/kubelet/cm/devicemanager/manager_stub.go | 99 +++++++++++++++++++ - 6 files changed, 273 insertions(+), 21 deletions(-) + pkg/kubelet/cm/container_manager_linux.go | 1 + + pkg/kubelet/cm/cpumanager/cpu_manager.go | 35 +++- + pkg/kubelet/cm/cpumanager/cpu_manager_test.go | 23 ++- + pkg/kubelet/cm/cpumanager/policy_static.go | 157 +++++++++++++-- + .../cm/cpumanager/policy_static_test.go | 178 +++++++++++++++++- + pkg/kubelet/cm/devicemanager/manager_stub.go | 99 ++++++++++ + 6 files changed, 463 insertions(+), 30 deletions(-) create mode 100644 pkg/kubelet/cm/devicemanager/manager_stub.go diff --git a/pkg/kubelet/cm/container_manager_linux.go b/pkg/kubelet/cm/container_manager_linux.go @@ -264,18 +265,25 @@ index e7c74453472..78b4ada1a73 100644 testCases := []struct { diff --git a/pkg/kubelet/cm/cpumanager/policy_static.go b/pkg/kubelet/cm/cpumanager/policy_static.go -index 180d018565c..8d18ce65309 100644 +index 26eb400cee6..13bcad5a531 100644 --- a/pkg/kubelet/cm/cpumanager/policy_static.go +++ b/pkg/kubelet/cm/cpumanager/policy_static.go -@@ -18,6 +18,7 @@ package cpumanager +@@ -17,15 +17,22 @@ limitations under the License. + package cpumanager import ( ++ "context" "fmt" + "strconv" ++ k8sclient "k8s.io/client-go/kubernetes" ++ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ++ restclient "k8s.io/client-go/rest" v1 "k8s.io/api/core/v1" ++ "k8s.io/client-go/tools/clientcmd" "k8s.io/klog/v2" -@@ -26,6 +27,7 @@ import ( + + v1qos "k8s.io/kubernetes/pkg/apis/core/v1/helper/qos" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology" "k8s.io/kubernetes/pkg/kubelet/cm/cpuset" @@ -283,7 +291,32 @@ index 180d018565c..8d18ce65309 100644 "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager" "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask" "k8s.io/kubernetes/pkg/kubelet/metrics" -@@ -104,6 +106,10 @@ type staticPolicy struct { +@@ -41,6 +48,12 @@ const ( + ErrorSMTAlignment = "SMTAlignmentError" + ) + ++type getPodNamespace func(string) (*v1.Namespace, error) ++type buildFromConfigFlag func(masterUrl string, kubeconfigPath string) (*restclient.Config, error) ++ ++var varGetNamespaceObject getPodNamespace ++var varBuildConfigFromFlags buildFromConfigFlag ++ + // SMTAlignmentError represents an error due to SMT alignment + type SMTAlignmentError struct { + RequestedCPUs int +@@ -56,11 +69,6 @@ func (e SMTAlignmentError) Type() string { + return ErrorSMTAlignment + } + +-// Define namespaces used by platform infrastructure pods +-var infraNamespaces = [...]string{ +- "kube-system", "armada", "cert-manager", "platform-deployment-manager", "portieris", "vault", "notification", "flux-helm", "metrics-server", "node-feature-discovery", "intel-power", "power-metrics", "sriov-fec-system", +-} +- + // staticPolicy is a CPU manager policy that does not change CPU + // assignments for exclusively pinned guaranteed containers after the main + // container process starts. +@@ -104,6 +112,10 @@ type staticPolicy struct { topology *topology.CPUTopology // set of CPUs that is not available for exclusive assignment reserved cpuset.CPUSet @@ -294,7 +327,7 @@ index 180d018565c..8d18ce65309 100644 // If true, default CPUSet should exclude reserved CPUs excludeReserved bool // topology manager reference to get container Topology affinity -@@ -120,7 +126,8 @@ var _ Policy = &staticPolicy{} +@@ -120,7 +132,8 @@ var _ Policy = &staticPolicy{} // NewStaticPolicy returns a CPU manager policy that does not change CPU // assignments for exclusively pinned guaranteed containers after the main // container process starts. @@ -304,7 +337,7 @@ index 180d018565c..8d18ce65309 100644 opts, err := NewStaticPolicyOptions(cpuPolicyOptions) if err != nil { return nil, err -@@ -135,6 +142,8 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv +@@ -135,6 +148,8 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv policy := &staticPolicy{ topology: topology, affinity: affinity, @@ -313,7 +346,7 @@ index 180d018565c..8d18ce65309 100644 excludeReserved: excludeReserved, cpusToReuse: make(map[string]cpuset.CPUSet), options: opts, -@@ -161,6 +170,12 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv +@@ -161,6 +176,12 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv klog.InfoS("Reserved CPUs not available for exclusive assignment", "reservedSize", reserved.Size(), "reserved", reserved) policy.reserved = reserved @@ -326,7 +359,7 @@ index 180d018565c..8d18ce65309 100644 return policy, nil } -@@ -194,8 +209,9 @@ func (p *staticPolicy) validateState(s state.State) error { +@@ -194,8 +215,9 @@ func (p *staticPolicy) validateState(s state.State) error { } else { s.SetDefaultCPUSet(allCPUs) } @@ -338,7 +371,17 @@ index 180d018565c..8d18ce65309 100644 return nil } -@@ -290,16 +306,39 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai +@@ -281,6 +303,9 @@ func (p *staticPolicy) updateCPUsToReuse(pod *v1.Pod, container *v1.Container, c + } + + func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Container) (rerr error) { ++ ++ varGetNamespaceObject = getPodNamespaceObject ++ varBuildConfigFromFlags = clientcmd.BuildConfigFromFlags + // Process infra pods before guaranteed pods + if isKubeInfra(pod) { + // Container belongs in reserved pool. +@@ -290,16 +315,39 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai return nil } @@ -380,7 +423,7 @@ index 180d018565c..8d18ce65309 100644 numCPUs := p.guaranteedCPUs(pod, container) if numCPUs == 0 { // container belongs in the shared pool (nothing to do; use default cpuset) -@@ -348,7 +387,9 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai +@@ -348,7 +396,9 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai } s.SetCPUSet(string(pod.UID), container.Name, cpuset) p.updateCPUsToReuse(pod, container, cpuset) @@ -391,10 +434,71 @@ index 180d018565c..8d18ce65309 100644 return nil } -@@ -657,6 +698,36 @@ func isKubeInfra(pod *v1.Pod) bool { - return false +@@ -647,14 +697,91 @@ func (p *staticPolicy) generateCPUTopologyHints(availableCPUs cpuset.CPUSet, reu + return hints } ++func getPodNamespaceObject(podNamespaceName string) (*v1.Namespace, error) { ++ ++ cfg, err := varBuildConfigFromFlags("", "/etc/kubernetes/kubelet.conf") ++ if err != nil { ++ klog.Error("Failed to build client config from /etc/kubernetes/kubelet.conf: ", err.Error()) ++ return nil, err ++ } ++ ++ clientset, err := k8sclient.NewForConfig(cfg) ++ if err != nil { ++ klog.Error("Failed to get clientset for KUBECONFIG /etc/kubernetes/kubelet.conf: ", err.Error()) ++ return nil, err ++ } ++ ++ namespaceObj, err := clientset.CoreV1().Namespaces().Get(context.TODO(), podNamespaceName, metav1.GetOptions{}) ++ if err != nil { ++ klog.Error("Error getting namespace object:", err.Error()) ++ return nil, err ++ } ++ ++ return namespaceObj, nil ++ ++} ++ + // check if a given pod is in a platform infrastructure namespace + func isKubeInfra(pod *v1.Pod) bool { +- for _, namespace := range infraNamespaces { +- if namespace == pod.Namespace { +- return true +- } ++ ++ podName := pod.GetName() ++ podNamespaceName := pod.GetNamespace() ++ ++ klog.InfoS("Checking pod ", podName , " for label 'app.starlingx.io/component=platform'.") ++ podLabels := pod.GetLabels() ++ val, ok := podLabels["app.starlingx.io/component"] ++ if (ok && val == "platform") { ++ klog.InfoS("Pod ", podName, " has 'app.starlingx.io/component=platform' label. Assigning platform CPUs.") ++ return true + } ++ ++ klog.InfoS("Pod ", pod.GetName(), " does not have 'app.starlingx.io/component=platform' label. Checking its namespace information...") ++ ++ namespaceObj, err := varGetNamespaceObject(podNamespaceName) ++ if err != nil { ++ return false ++ } ++ ++ namespaceLabels := namespaceObj.GetLabels() ++ val, ok = namespaceLabels["app.starlingx.io/component"] ++ if ok && val == "platform" { ++ klog.InfoS("For pod: ", podName, ", its Namespace ", podNamespaceName, " has 'app.starlingx.io/component=platform' label. Assigning platform CPUs.") ++ return true ++ } ++ ++ klog.InfoS("Neither pod ", podName, " nor its namespace ", podNamespaceName, " has 'app.starlingx.io/component=platform' label. Not assigning platform CPUs.") + return false ++ ++} ++ +// get the isolated CPUs (if any) from the devices associated with a specific container +func (p *staticPolicy) podIsolCPUs(pod *v1.Pod, container *v1.Container) cpuset.CPUSet { + // NOTE: This is required for TestStaticPolicyAdd() since makePod() does @@ -423,16 +527,28 @@ index 180d018565c..8d18ce65309 100644 + } + } + return cpuSet -+} -+ + } + // isHintSocketAligned function return true if numa nodes in hint are socket aligned. - func (p *staticPolicy) isHintSocketAligned(hint topologymanager.TopologyHint, minAffinitySize int) bool { - numaNodesBitMask := hint.NUMANodeAffinity.GetBits() diff --git a/pkg/kubelet/cm/cpumanager/policy_static_test.go b/pkg/kubelet/cm/cpumanager/policy_static_test.go -index 414e5ce144c..1c43df3b85f 100644 +index 414e5ce144c..f79c23accb4 100644 --- a/pkg/kubelet/cm/cpumanager/policy_static_test.go +++ b/pkg/kubelet/cm/cpumanager/policy_static_test.go -@@ -28,6 +28,7 @@ import ( +@@ -17,10 +17,13 @@ limitations under the License. + package cpumanager + + import ( ++ "errors" + "fmt" + "reflect" + "testing" + ++ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ++ restclient "k8s.io/client-go/rest" + v1 "k8s.io/api/core/v1" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" +@@ -28,6 +31,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology" "k8s.io/kubernetes/pkg/kubelet/cm/cpuset" @@ -440,7 +556,7 @@ index 414e5ce144c..1c43df3b85f 100644 "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager" "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask" ) -@@ -69,8 +70,9 @@ func (spt staticPolicyTest) PseudoClone() staticPolicyTest { +@@ -69,8 +73,9 @@ func (spt staticPolicyTest) PseudoClone() staticPolicyTest { } func TestStaticPolicyName(t *testing.T) { @@ -451,7 +567,7 @@ index 414e5ce144c..1c43df3b85f 100644 policyName := policy.Name() if policyName != "static" { -@@ -80,6 +82,7 @@ func TestStaticPolicyName(t *testing.T) { +@@ -80,6 +85,7 @@ func TestStaticPolicyName(t *testing.T) { } func TestStaticPolicyStart(t *testing.T) { @@ -459,7 +575,7 @@ index 414e5ce144c..1c43df3b85f 100644 testCases := []staticPolicyTest{ { description: "non-corrupted state", -@@ -155,7 +158,7 @@ func TestStaticPolicyStart(t *testing.T) { +@@ -155,7 +161,7 @@ func TestStaticPolicyStart(t *testing.T) { } for _, testCase := range testCases { t.Run(testCase.description, func(t *testing.T) { @@ -468,7 +584,7 @@ index 414e5ce144c..1c43df3b85f 100644 policy := p.(*staticPolicy) st := &mockState{ -@@ -203,7 +206,6 @@ func TestStaticPolicyAdd(t *testing.T) { +@@ -203,7 +209,6 @@ func TestStaticPolicyAdd(t *testing.T) { largeTopoCPUSet := largeTopoBuilder.Result() largeTopoSock0CPUSet := largeTopoSock0Builder.Result() largeTopoSock1CPUSet := largeTopoSock1Builder.Result() @@ -476,7 +592,7 @@ index 414e5ce144c..1c43df3b85f 100644 // these are the cases which must behave the same regardless the policy options. // So we will permutate the options to ensure this holds true. -@@ -577,8 +579,10 @@ func runStaticPolicyTestCase(t *testing.T, testCase staticPolicyTest) { +@@ -577,8 +582,10 @@ func runStaticPolicyTestCase(t *testing.T, testCase staticPolicyTest) { if testCase.topologyHint != nil { tm = topologymanager.NewFakeManagerWithHint(testCase.topologyHint) } @@ -488,7 +604,7 @@ index 414e5ce144c..1c43df3b85f 100644 st := &mockState{ assignments: testCase.stAssignments, -@@ -625,6 +629,8 @@ func runStaticPolicyTestCaseWithFeatureGate(t *testing.T, testCase staticPolicyT +@@ -625,6 +632,8 @@ func runStaticPolicyTestCaseWithFeatureGate(t *testing.T, testCase staticPolicyT } func TestStaticPolicyReuseCPUs(t *testing.T) { @@ -497,7 +613,7 @@ index 414e5ce144c..1c43df3b85f 100644 testCases := []struct { staticPolicyTest expCSetAfterAlloc cpuset.CPUSet -@@ -649,7 +655,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) { +@@ -649,7 +658,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) { } for _, testCase := range testCases { @@ -506,7 +622,7 @@ index 414e5ce144c..1c43df3b85f 100644 st := &mockState{ assignments: testCase.stAssignments, -@@ -682,6 +688,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) { +@@ -682,6 +691,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) { func TestStaticPolicyRemove(t *testing.T) { excludeReserved := false @@ -514,7 +630,7 @@ index 414e5ce144c..1c43df3b85f 100644 testCases := []staticPolicyTest{ { description: "SingleSocketHT, DeAllocOneContainer", -@@ -740,7 +747,7 @@ func TestStaticPolicyRemove(t *testing.T) { +@@ -740,7 +750,7 @@ func TestStaticPolicyRemove(t *testing.T) { } for _, testCase := range testCases { @@ -523,7 +639,7 @@ index 414e5ce144c..1c43df3b85f 100644 st := &mockState{ assignments: testCase.stAssignments, -@@ -763,6 +770,7 @@ func TestStaticPolicyRemove(t *testing.T) { +@@ -763,6 +773,7 @@ func TestStaticPolicyRemove(t *testing.T) { func TestTopologyAwareAllocateCPUs(t *testing.T) { excludeReserved := false @@ -531,7 +647,7 @@ index 414e5ce144c..1c43df3b85f 100644 testCases := []struct { description string topo *topology.CPUTopology -@@ -831,7 +839,8 @@ func TestTopologyAwareAllocateCPUs(t *testing.T) { +@@ -831,7 +842,8 @@ func TestTopologyAwareAllocateCPUs(t *testing.T) { }, } for _, tc := range testCases { @@ -541,7 +657,7 @@ index 414e5ce144c..1c43df3b85f 100644 policy := p.(*staticPolicy) st := &mockState{ assignments: tc.stAssignments, -@@ -864,6 +873,7 @@ type staticPolicyTestWithResvList struct { +@@ -864,6 +876,7 @@ type staticPolicyTestWithResvList struct { topo *topology.CPUTopology numReservedCPUs int reserved cpuset.CPUSet @@ -549,7 +665,7 @@ index 414e5ce144c..1c43df3b85f 100644 stAssignments state.ContainerCPUAssignments stDefaultCPUSet cpuset.CPUSet pod *v1.Pod -@@ -874,6 +884,8 @@ type staticPolicyTestWithResvList struct { +@@ -874,6 +887,8 @@ type staticPolicyTestWithResvList struct { } func TestStaticPolicyStartWithResvList(t *testing.T) { @@ -558,7 +674,7 @@ index 414e5ce144c..1c43df3b85f 100644 testCases := []staticPolicyTestWithResvList{ { description: "empty cpuset", -@@ -903,11 +915,10 @@ func TestStaticPolicyStartWithResvList(t *testing.T) { +@@ -903,11 +918,10 @@ func TestStaticPolicyStartWithResvList(t *testing.T) { expNewErr: fmt.Errorf("[cpumanager] unable to reserve the required amount of CPUs (size of 0-1 did not equal 1)"), }, } @@ -571,7 +687,7 @@ index 414e5ce144c..1c43df3b85f 100644 if !reflect.DeepEqual(err, testCase.expNewErr) { t.Errorf("StaticPolicy Start() error (%v). expected error: %v but got: %v", testCase.description, testCase.expNewErr, err) -@@ -947,6 +958,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { +@@ -947,6 +961,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { topo: topoSingleSocketHT, numReservedCPUs: 1, reserved: cpuset.NewCPUSet(0), @@ -579,7 +695,7 @@ index 414e5ce144c..1c43df3b85f 100644 stAssignments: state.ContainerCPUAssignments{}, stDefaultCPUSet: cpuset.NewCPUSet(1, 2, 3, 4, 5, 6, 7), pod: makePod("fakePod", "fakeContainer2", "8000m", "8000m"), -@@ -959,6 +971,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { +@@ -959,6 +974,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { topo: topoSingleSocketHT, numReservedCPUs: 2, reserved: cpuset.NewCPUSet(0, 1), @@ -587,7 +703,7 @@ index 414e5ce144c..1c43df3b85f 100644 stAssignments: state.ContainerCPUAssignments{}, stDefaultCPUSet: cpuset.NewCPUSet(2, 3, 4, 5, 6, 7), pod: makePod("fakePod", "fakeContainer2", "1000m", "1000m"), -@@ -971,6 +984,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { +@@ -971,6 +987,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { topo: topoSingleSocketHT, numReservedCPUs: 2, reserved: cpuset.NewCPUSet(0, 1), @@ -595,7 +711,7 @@ index 414e5ce144c..1c43df3b85f 100644 stAssignments: state.ContainerCPUAssignments{ "fakePod": map[string]cpuset.CPUSet{ "fakeContainer100": cpuset.NewCPUSet(2, 3, 6, 7), -@@ -987,6 +1001,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { +@@ -987,6 +1004,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { topo: topoSingleSocketHT, numReservedCPUs: 2, reserved: cpuset.NewCPUSet(0, 1), @@ -603,7 +719,7 @@ index 414e5ce144c..1c43df3b85f 100644 stAssignments: state.ContainerCPUAssignments{ "fakePod": map[string]cpuset.CPUSet{ "fakeContainer100": cpuset.NewCPUSet(2, 3, 6, 7), -@@ -998,11 +1013,29 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { +@@ -998,11 +1016,29 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { expCPUAlloc: true, expCSet: cpuset.NewCPUSet(0, 1), }, @@ -634,6 +750,132 @@ index 414e5ce144c..1c43df3b85f 100644 st := &mockState{ assignments: testCase.stAssignments, +@@ -1128,3 +1164,125 @@ func TestStaticPolicyOptions(t *testing.T) { + }) + } + } ++ ++func makePodWithLabels(podLabels map[string]string) *v1.Pod { ++ return &v1.Pod{ ++ ObjectMeta: metav1.ObjectMeta{ ++ Name: "test-pod", ++ Namespace: "test-namespace", ++ Labels: podLabels, ++ }, ++ } ++} ++ ++func fakeBuildConfigFromFlags(masterUrl string, kubeconfigPath string) (*restclient.Config, error) { ++ ++ return &restclient.Config{}, nil ++} ++ ++func fakeBuildConfigFromFlagsError(masterUrl string, kubeconfigPath string) (*restclient.Config, error) { ++ ++ errString := fmt.Sprintf("%s file not found", kubeconfigPath) ++ return nil, errors.New(errString) ++ ++} ++ ++func getFakeInfraPodNamespace(_ string) (*v1.Namespace, error) { ++ ++ return &v1.Namespace{ ++ ObjectMeta: metav1.ObjectMeta{ ++ Name: "test-namespace", ++ Labels: map[string]string{ ++ "app.starlingx.io/component": "platform", ++ }, ++ }}, nil ++} ++ ++func getFakeNonInfraPodNamespace(_ string) (*v1.Namespace, error) { ++ ++ return &v1.Namespace{ ++ ObjectMeta: metav1.ObjectMeta{ ++ Name: "test-namespace", ++ Labels: map[string]string{ ++ "fake": "label", ++ }}}, nil ++ ++} ++ ++type kubeInfraPodTestCase struct { ++ description string ++ pod *v1.Pod ++ namespaceFunc getPodNamespace ++ expectedValue bool ++} ++ ++func TestKubeInfraPod(t *testing.T) { ++ testCases := []kubeInfraPodTestCase{ ++ { ++ description: "Pod with platform label and namespace without platform label", ++ pod: makePodWithLabels(map[string]string{ ++ "app.starlingx.io/component": "platform", ++ }), ++ namespaceFunc: getFakeNonInfraPodNamespace, ++ expectedValue: true, ++ ++ }, ++ { ++ description: "Pod without platform label and namespace with platform label", ++ pod: makePodWithLabels(map[string]string{ ++ "test": "label", ++ }), ++ namespaceFunc: getFakeInfraPodNamespace, ++ expectedValue: true, ++ }, ++ { ++ description: "Pod without platform label and namespace without platform label", ++ pod: makePodWithLabels(map[string]string{ ++ "test": "namespace", ++ }), ++ namespaceFunc: getFakeNonInfraPodNamespace, ++ expectedValue: false, ++ }, ++ ++ } ++ ++ for _, testCase := range testCases { ++ t.Run(testCase.description, func(t *testing.T) { ++ ++ varGetNamespaceObject = testCase.namespaceFunc ++ varBuildConfigFromFlags = fakeBuildConfigFromFlags ++ gotValue := isKubeInfra(testCase.pod) ++ ++ if gotValue != testCase.expectedValue { ++ t.Errorf("StaticPolicy isKubeInfraPod() error %v. expected value %v actual value %v", ++ testCase.description, testCase.expectedValue, gotValue) ++ } else { ++ fmt.Printf("StaticPolicy isKubeInfraPod() test successful. : %v ", testCase.description) ++ } ++ ++ }) ++ } ++ ++ test := kubeInfraPodTestCase{ ++ description: "Failure reading kubeconfig file", ++ pod: makePodWithLabels(map[string]string{ ++ "test": "namespace", ++ }), ++ namespaceFunc: getFakeNonInfraPodNamespace, ++ expectedValue: false, ++ } ++ ++ varGetNamespaceObject = getPodNamespaceObject ++ varBuildConfigFromFlags = fakeBuildConfigFromFlagsError ++ ++ gotValue := isKubeInfra(test.pod) ++ ++ if gotValue != test.expectedValue { ++ t.Errorf("StaticPolicy isKubeInfraPod() error %v. expected value %v actual value %v", ++ test.description, test.expectedValue, gotValue) ++ } else { ++ fmt.Printf("StaticPolicy isKubeInfraPod() test successful. : %v ", test.description) ++ } ++ ++} ++ diff --git a/pkg/kubelet/cm/devicemanager/manager_stub.go b/pkg/kubelet/cm/devicemanager/manager_stub.go new file mode 100644 index 00000000000..e6874f88d8a diff --git a/kubernetes/kubernetes-1.27.5/debian/deb_folder/patches/kubelet-cpumanager-introduce-concept-of-isolated-CPU.patch b/kubernetes/kubernetes-1.27.5/debian/deb_folder/patches/kubelet-cpumanager-introduce-concept-of-isolated-CPU.patch index 8a0d87833..90e26fffd 100644 --- a/kubernetes/kubernetes-1.27.5/debian/deb_folder/patches/kubelet-cpumanager-introduce-concept-of-isolated-CPU.patch +++ b/kubernetes/kubernetes-1.27.5/debian/deb_folder/patches/kubelet-cpumanager-introduce-concept-of-isolated-CPU.patch @@ -1,4 +1,4 @@ -From b51d6c0ba6dfd9a34c7f6832d17840820f9985eb Mon Sep 17 00:00:00 2001 +From 8cdc168daa7fa8adc3a47c2e40900e4bf435babe Mon Sep 17 00:00:00 2001 From: Boovan Rajendran Date: Fri, 8 Sep 2023 10:46:07 -0400 Subject: [PATCH] kubelet cpumanager introduce concept of isolated CPUs @@ -47,14 +47,15 @@ Signed-off-by: Gleb Aronsky Signed-off-by: Ramesh Kumar Sivanandam Signed-off-by: Sachin Gopala Krishna Signed-off-by: Boovan Rajendran +Signed-off-by: Kaustubh Dhokte --- - pkg/kubelet/cm/container_manager_linux.go | 1 + - pkg/kubelet/cm/cpumanager/cpu_manager.go | 35 ++++++- - pkg/kubelet/cm/cpumanager/cpu_manager_test.go | 23 ++++- - pkg/kubelet/cm/cpumanager/policy_static.go | 83 ++++++++++++++-- - .../cm/cpumanager/policy_static_test.go | 53 ++++++++-- - pkg/kubelet/cm/devicemanager/manager_stub.go | 99 +++++++++++++++++++ - 6 files changed, 273 insertions(+), 21 deletions(-) + pkg/kubelet/cm/container_manager_linux.go | 1 + + pkg/kubelet/cm/cpumanager/cpu_manager.go | 35 +++- + pkg/kubelet/cm/cpumanager/cpu_manager_test.go | 23 ++- + pkg/kubelet/cm/cpumanager/policy_static.go | 157 ++++++++++++++-- + .../cm/cpumanager/policy_static_test.go | 177 +++++++++++++++++- + pkg/kubelet/cm/devicemanager/manager_stub.go | 99 ++++++++++ + 6 files changed, 462 insertions(+), 30 deletions(-) create mode 100644 pkg/kubelet/cm/devicemanager/manager_stub.go diff --git a/pkg/kubelet/cm/container_manager_linux.go b/pkg/kubelet/cm/container_manager_linux.go @@ -265,18 +266,27 @@ index bb69b0ac084..44a88429a12 100644 testCases := []struct { diff --git a/pkg/kubelet/cm/cpumanager/policy_static.go b/pkg/kubelet/cm/cpumanager/policy_static.go -index 1fdb49b52ad..49f63dd9efd 100644 +index 1fdb49b52ad..99990fb596a 100644 --- a/pkg/kubelet/cm/cpumanager/policy_static.go +++ b/pkg/kubelet/cm/cpumanager/policy_static.go -@@ -18,6 +18,7 @@ package cpumanager +@@ -17,10 +17,16 @@ limitations under the License. + package cpumanager import ( ++ "context" "fmt" + "strconv" ++ k8sclient "k8s.io/client-go/kubernetes" ++ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ++ restclient "k8s.io/client-go/rest" v1 "k8s.io/api/core/v1" utilfeature "k8s.io/apiserver/pkg/util/feature" -@@ -28,6 +29,7 @@ import ( ++ "k8s.io/client-go/tools/clientcmd" + "k8s.io/klog/v2" + podutil "k8s.io/kubernetes/pkg/api/v1/pod" + v1qos "k8s.io/kubernetes/pkg/apis/core/v1/helper/qos" +@@ -28,6 +34,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology" "k8s.io/kubernetes/pkg/kubelet/cm/cpuset" @@ -284,7 +294,32 @@ index 1fdb49b52ad..49f63dd9efd 100644 "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager" "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask" "k8s.io/kubernetes/pkg/kubelet/metrics" -@@ -110,6 +112,10 @@ type staticPolicy struct { +@@ -43,6 +50,12 @@ const ( + ErrorSMTAlignment = "SMTAlignmentError" + ) + ++type getPodNamespace func(string) (*v1.Namespace, error) ++type buildFromConfigFlag func(masterUrl string, kubeconfigPath string) (*restclient.Config, error) ++ ++var varGetNamespaceObject getPodNamespace ++var varBuildConfigFromFlags buildFromConfigFlag ++ + // SMTAlignmentError represents an error due to SMT alignment + type SMTAlignmentError struct { + RequestedCPUs int +@@ -62,11 +75,6 @@ func (e SMTAlignmentError) Type() string { + return ErrorSMTAlignment + } + +-// Define namespaces used by platform infrastructure pods +-var infraNamespaces = [...]string{ +- "kube-system", "armada", "cert-manager", "platform-deployment-manager", "portieris", "vault", "notification", "flux-helm", "metrics-server", +-} +- + // staticPolicy is a CPU manager policy that does not change CPU + // assignments for exclusively pinned guaranteed containers after the main + // container process starts. +@@ -110,6 +118,10 @@ type staticPolicy struct { topology *topology.CPUTopology // set of CPUs that is not available for exclusive assignment reservedCPUs cpuset.CPUSet @@ -295,7 +330,7 @@ index 1fdb49b52ad..49f63dd9efd 100644 // If true, default CPUSet should exclude reserved CPUs excludeReserved bool // Superset of reservedCPUs. It includes not just the reservedCPUs themselves, -@@ -132,7 +138,8 @@ var _ Policy = &staticPolicy{} +@@ -132,7 +144,8 @@ var _ Policy = &staticPolicy{} // NewStaticPolicy returns a CPU manager policy that does not change CPU // assignments for exclusively pinned guaranteed containers after the main // container process starts. @@ -305,7 +340,7 @@ index 1fdb49b52ad..49f63dd9efd 100644 opts, err := NewStaticPolicyOptions(cpuPolicyOptions) if err != nil { return nil, err -@@ -147,6 +154,8 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv +@@ -147,6 +160,8 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv policy := &staticPolicy{ topology: topology, affinity: affinity, @@ -314,7 +349,7 @@ index 1fdb49b52ad..49f63dd9efd 100644 excludeReserved: excludeReserved, cpusToReuse: make(map[string]cpuset.CPUSet), options: opts, -@@ -183,6 +192,12 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv +@@ -183,6 +198,12 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv policy.reservedCPUs = reserved policy.reservedPhysicalCPUs = reservedPhysicalCPUs @@ -327,7 +362,7 @@ index 1fdb49b52ad..49f63dd9efd 100644 return policy, nil } -@@ -216,8 +231,9 @@ func (p *staticPolicy) validateState(s state.State) error { +@@ -216,8 +237,9 @@ func (p *staticPolicy) validateState(s state.State) error { } else { s.SetDefaultCPUSet(allCPUs) } @@ -339,7 +374,17 @@ index 1fdb49b52ad..49f63dd9efd 100644 return nil } -@@ -316,16 +332,39 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai +@@ -307,6 +329,9 @@ func (p *staticPolicy) updateCPUsToReuse(pod *v1.Pod, container *v1.Container, c + } + + func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Container) (rerr error) { ++ ++ varGetNamespaceObject = getPodNamespaceObject ++ varBuildConfigFromFlags = clientcmd.BuildConfigFromFlags + // Process infra pods before guaranteed pods + if isKubeInfra(pod) { + // Container belongs in reserved pool. +@@ -316,16 +341,39 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai return nil } @@ -381,7 +426,7 @@ index 1fdb49b52ad..49f63dd9efd 100644 numCPUs := p.guaranteedCPUs(pod, container) if numCPUs == 0 { // container belongs in the shared pool (nothing to do; use default cpuset) -@@ -391,7 +430,9 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai +@@ -391,7 +439,9 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai } s.SetCPUSet(string(pod.UID), container.Name, cpuset) p.updateCPUsToReuse(pod, container, cpuset) @@ -392,10 +437,71 @@ index 1fdb49b52ad..49f63dd9efd 100644 return nil } -@@ -709,6 +750,36 @@ func isKubeInfra(pod *v1.Pod) bool { - return false +@@ -699,14 +749,91 @@ func (p *staticPolicy) generateCPUTopologyHints(availableCPUs cpuset.CPUSet, reu + return hints } ++func getPodNamespaceObject(podNamespaceName string) (*v1.Namespace, error) { ++ ++ cfg, err := varBuildConfigFromFlags("", "/etc/kubernetes/kubelet.conf") ++ if err != nil { ++ klog.Error("Failed to build client config from /etc/kubernetes/kubelet.conf: ", err.Error()) ++ return nil, err ++ } ++ ++ clientset, err := k8sclient.NewForConfig(cfg) ++ if err != nil { ++ klog.Error("Failed to get clientset for KUBECONFIG /etc/kubernetes/kubelet.conf: ", err.Error()) ++ return nil, err ++ } ++ ++ namespaceObj, err := clientset.CoreV1().Namespaces().Get(context.TODO(), podNamespaceName, metav1.GetOptions{}) ++ if err != nil { ++ klog.Error("Error getting namespace object:", err.Error()) ++ return nil, err ++ } ++ ++ return namespaceObj, nil ++ ++} ++ + // check if a given pod is in a platform infrastructure namespace + func isKubeInfra(pod *v1.Pod) bool { +- for _, namespace := range infraNamespaces { +- if namespace == pod.Namespace { +- return true +- } ++ ++ podName := pod.GetName() ++ podNamespaceName := pod.GetNamespace() ++ ++ klog.InfoS("Checking pod ", podName , " for label 'app.starlingx.io/component=platform'.") ++ podLabels := pod.GetLabels() ++ val, ok := podLabels["app.starlingx.io/component"] ++ if (ok && val == "platform") { ++ klog.InfoS("Pod ", podName, " has 'app.starlingx.io/component=platform' label. Assigning platform CPUs.") ++ return true + } ++ ++ klog.InfoS("Pod ", pod.GetName(), " does not have 'app.starlingx.io/component=platform' label. Checking its namespace information...") ++ ++ namespaceObj, err := varGetNamespaceObject(podNamespaceName) ++ if err != nil { ++ return false ++ } ++ ++ namespaceLabels := namespaceObj.GetLabels() ++ val, ok = namespaceLabels["app.starlingx.io/component"] ++ if ok && val == "platform" { ++ klog.InfoS("For pod: ", podName, ", its Namespace ", podNamespaceName, " has 'app.starlingx.io/component=platform' label. Assigning platform CPUs.") ++ return true ++ } ++ ++ klog.InfoS("Neither pod ", podName, " nor its namespace ", podNamespaceName, " has 'app.starlingx.io/component=platform' label. Not assigning platform CPUs.") + return false ++ ++} ++ +// get the isolated CPUs (if any) from the devices associated with a specific container +func (p *staticPolicy) podIsolCPUs(pod *v1.Pod, container *v1.Container) cpuset.CPUSet { + // NOTE: This is required for TestStaticPolicyAdd() since makePod() does @@ -424,16 +530,28 @@ index 1fdb49b52ad..49f63dd9efd 100644 + } + } + return cpuSet -+} -+ + } + // isHintSocketAligned function return true if numa nodes in hint are socket aligned. - func (p *staticPolicy) isHintSocketAligned(hint topologymanager.TopologyHint, minAffinitySize int) bool { - numaNodesBitMask := hint.NUMANodeAffinity.GetBits() diff --git a/pkg/kubelet/cm/cpumanager/policy_static_test.go b/pkg/kubelet/cm/cpumanager/policy_static_test.go -index 63f31486d19..c25ee484a94 100644 +index 63f31486d19..b0ce9d497d9 100644 --- a/pkg/kubelet/cm/cpumanager/policy_static_test.go +++ b/pkg/kubelet/cm/cpumanager/policy_static_test.go -@@ -28,6 +28,7 @@ import ( +@@ -17,10 +17,13 @@ limitations under the License. + package cpumanager + + import ( ++ "errors" + "fmt" + "reflect" + "testing" + ++ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ++ restclient "k8s.io/client-go/rest" + v1 "k8s.io/api/core/v1" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" +@@ -28,6 +31,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology" "k8s.io/kubernetes/pkg/kubelet/cm/cpuset" @@ -441,7 +559,7 @@ index 63f31486d19..c25ee484a94 100644 "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager" "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask" ) -@@ -70,8 +71,9 @@ func (spt staticPolicyTest) PseudoClone() staticPolicyTest { +@@ -70,8 +74,9 @@ func (spt staticPolicyTest) PseudoClone() staticPolicyTest { } func TestStaticPolicyName(t *testing.T) { @@ -452,7 +570,7 @@ index 63f31486d19..c25ee484a94 100644 policyName := policy.Name() if policyName != "static" { -@@ -81,6 +83,7 @@ func TestStaticPolicyName(t *testing.T) { +@@ -81,6 +86,7 @@ func TestStaticPolicyName(t *testing.T) { } func TestStaticPolicyStart(t *testing.T) { @@ -460,7 +578,7 @@ index 63f31486d19..c25ee484a94 100644 testCases := []staticPolicyTest{ { description: "non-corrupted state", -@@ -156,7 +159,7 @@ func TestStaticPolicyStart(t *testing.T) { +@@ -156,7 +162,7 @@ func TestStaticPolicyStart(t *testing.T) { } for _, testCase := range testCases { t.Run(testCase.description, func(t *testing.T) { @@ -469,7 +587,7 @@ index 63f31486d19..c25ee484a94 100644 policy := p.(*staticPolicy) st := &mockState{ -@@ -204,7 +207,6 @@ func TestStaticPolicyAdd(t *testing.T) { +@@ -204,7 +210,6 @@ func TestStaticPolicyAdd(t *testing.T) { largeTopoCPUSet := cpuset.New(largeTopoCPUids...) largeTopoSock0CPUSet := cpuset.New(largeTopoSock0CPUids...) largeTopoSock1CPUSet := cpuset.New(largeTopoSock1CPUids...) @@ -477,7 +595,7 @@ index 63f31486d19..c25ee484a94 100644 // these are the cases which must behave the same regardless the policy options. // So we will permutate the options to ensure this holds true. -@@ -627,7 +629,9 @@ func runStaticPolicyTestCase(t *testing.T, testCase staticPolicyTest) { +@@ -627,7 +632,9 @@ func runStaticPolicyTestCase(t *testing.T, testCase staticPolicyTest) { cpus = testCase.reservedCPUs.Clone() } testExcl := false @@ -488,7 +606,7 @@ index 63f31486d19..c25ee484a94 100644 st := &mockState{ assignments: testCase.stAssignments, -@@ -674,6 +678,8 @@ func runStaticPolicyTestCaseWithFeatureGate(t *testing.T, testCase staticPolicyT +@@ -674,6 +681,8 @@ func runStaticPolicyTestCaseWithFeatureGate(t *testing.T, testCase staticPolicyT } func TestStaticPolicyReuseCPUs(t *testing.T) { @@ -497,7 +615,7 @@ index 63f31486d19..c25ee484a94 100644 testCases := []struct { staticPolicyTest expCSetAfterAlloc cpuset.CPUSet -@@ -698,7 +704,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) { +@@ -698,7 +707,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) { } for _, testCase := range testCases { @@ -506,7 +624,7 @@ index 63f31486d19..c25ee484a94 100644 st := &mockState{ assignments: testCase.stAssignments, -@@ -731,6 +737,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) { +@@ -731,6 +740,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) { func TestStaticPolicyRemove(t *testing.T) { excludeReserved := false @@ -514,7 +632,7 @@ index 63f31486d19..c25ee484a94 100644 testCases := []staticPolicyTest{ { description: "SingleSocketHT, DeAllocOneContainer", -@@ -789,7 +796,7 @@ func TestStaticPolicyRemove(t *testing.T) { +@@ -789,7 +799,7 @@ func TestStaticPolicyRemove(t *testing.T) { } for _, testCase := range testCases { @@ -523,7 +641,7 @@ index 63f31486d19..c25ee484a94 100644 st := &mockState{ assignments: testCase.stAssignments, -@@ -812,6 +819,7 @@ func TestStaticPolicyRemove(t *testing.T) { +@@ -812,6 +822,7 @@ func TestStaticPolicyRemove(t *testing.T) { func TestTopologyAwareAllocateCPUs(t *testing.T) { excludeReserved := false @@ -531,7 +649,7 @@ index 63f31486d19..c25ee484a94 100644 testCases := []struct { description string topo *topology.CPUTopology -@@ -880,7 +888,8 @@ func TestTopologyAwareAllocateCPUs(t *testing.T) { +@@ -880,7 +891,8 @@ func TestTopologyAwareAllocateCPUs(t *testing.T) { }, } for _, tc := range testCases { @@ -541,7 +659,7 @@ index 63f31486d19..c25ee484a94 100644 policy := p.(*staticPolicy) st := &mockState{ assignments: tc.stAssignments, -@@ -913,6 +922,7 @@ type staticPolicyTestWithResvList struct { +@@ -913,6 +925,7 @@ type staticPolicyTestWithResvList struct { topo *topology.CPUTopology numReservedCPUs int reserved cpuset.CPUSet @@ -549,7 +667,7 @@ index 63f31486d19..c25ee484a94 100644 stAssignments state.ContainerCPUAssignments stDefaultCPUSet cpuset.CPUSet pod *v1.Pod -@@ -923,6 +933,8 @@ type staticPolicyTestWithResvList struct { +@@ -923,6 +936,8 @@ type staticPolicyTestWithResvList struct { } func TestStaticPolicyStartWithResvList(t *testing.T) { @@ -558,7 +676,7 @@ index 63f31486d19..c25ee484a94 100644 testCases := []staticPolicyTestWithResvList{ { description: "empty cpuset", -@@ -952,10 +964,9 @@ func TestStaticPolicyStartWithResvList(t *testing.T) { +@@ -952,10 +967,9 @@ func TestStaticPolicyStartWithResvList(t *testing.T) { expNewErr: fmt.Errorf("[cpumanager] unable to reserve the required amount of CPUs (size of 0-1 did not equal 1)"), }, } @@ -570,7 +688,7 @@ index 63f31486d19..c25ee484a94 100644 if !reflect.DeepEqual(err, testCase.expNewErr) { t.Errorf("StaticPolicy Start() error (%v). expected error: %v but got: %v", -@@ -996,6 +1007,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { +@@ -996,6 +1010,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { topo: topoSingleSocketHT, numReservedCPUs: 1, reserved: cpuset.New(0), @@ -578,7 +696,7 @@ index 63f31486d19..c25ee484a94 100644 stAssignments: state.ContainerCPUAssignments{}, stDefaultCPUSet: cpuset.New(1, 2, 3, 4, 5, 6, 7), pod: makePod("fakePod", "fakeContainer2", "8000m", "8000m"), -@@ -1008,6 +1020,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { +@@ -1008,6 +1023,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { topo: topoSingleSocketHT, numReservedCPUs: 2, reserved: cpuset.New(0, 1), @@ -586,7 +704,7 @@ index 63f31486d19..c25ee484a94 100644 stAssignments: state.ContainerCPUAssignments{}, stDefaultCPUSet: cpuset.New(2, 3, 4, 5, 6, 7), pod: makePod("fakePod", "fakeContainer2", "1000m", "1000m"), -@@ -1020,6 +1033,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { +@@ -1020,6 +1036,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { topo: topoSingleSocketHT, numReservedCPUs: 2, reserved: cpuset.New(0, 1), @@ -594,7 +712,7 @@ index 63f31486d19..c25ee484a94 100644 stAssignments: state.ContainerCPUAssignments{ "fakePod": map[string]cpuset.CPUSet{ "fakeContainer100": cpuset.New(2, 3, 6, 7), -@@ -1036,6 +1050,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { +@@ -1036,6 +1053,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { topo: topoSingleSocketHT, numReservedCPUs: 2, reserved: cpuset.New(0, 1), @@ -602,7 +720,7 @@ index 63f31486d19..c25ee484a94 100644 stAssignments: state.ContainerCPUAssignments{ "fakePod": map[string]cpuset.CPUSet{ "fakeContainer100": cpuset.New(2, 3, 6, 7), -@@ -1047,11 +1062,29 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { +@@ -1047,11 +1065,29 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { expCPUAlloc: true, expCSet: cpuset.New(0, 1), }, @@ -633,6 +751,131 @@ index 63f31486d19..c25ee484a94 100644 st := &mockState{ assignments: testCase.stAssignments, +@@ -1182,3 +1218,124 @@ func newCPUSetPtr(cpus ...int) *cpuset.CPUSet { + ret := cpuset.New(cpus...) + return &ret + } ++ ++func makePodWithLabels(podLabels map[string]string) *v1.Pod { ++ return &v1.Pod{ ++ ObjectMeta: metav1.ObjectMeta{ ++ Name: "test-pod", ++ Namespace: "test-namespace", ++ Labels: podLabels, ++ }, ++ } ++} ++ ++func fakeBuildConfigFromFlags(masterUrl string, kubeconfigPath string) (*restclient.Config, error) { ++ ++ return &restclient.Config{}, nil ++} ++ ++func fakeBuildConfigFromFlagsError(masterUrl string, kubeconfigPath string) (*restclient.Config, error) { ++ ++ errString := fmt.Sprintf("%s file not found", kubeconfigPath) ++ return nil, errors.New(errString) ++ ++} ++ ++func getFakeInfraPodNamespace(_ string) (*v1.Namespace, error) { ++ ++ return &v1.Namespace{ ++ ObjectMeta: metav1.ObjectMeta{ ++ Name: "test-namespace", ++ Labels: map[string]string{ ++ "app.starlingx.io/component": "platform", ++ }, ++ }}, nil ++} ++ ++func getFakeNonInfraPodNamespace(_ string) (*v1.Namespace, error) { ++ ++ return &v1.Namespace{ ++ ObjectMeta: metav1.ObjectMeta{ ++ Name: "test-namespace", ++ Labels: map[string]string{ ++ "fake": "label", ++ }}}, nil ++ ++} ++ ++type kubeInfraPodTestCase struct { ++ description string ++ pod *v1.Pod ++ namespaceFunc getPodNamespace ++ expectedValue bool ++} ++ ++func TestKubeInfraPod(t *testing.T) { ++ testCases := []kubeInfraPodTestCase{ ++ { ++ description: "Pod with platform label and namespace without platform label", ++ pod: makePodWithLabels(map[string]string{ ++ "app.starlingx.io/component": "platform", ++ }), ++ namespaceFunc: getFakeNonInfraPodNamespace, ++ expectedValue: true, ++ ++ }, ++ { ++ description: "Pod without platform label and namespace with platform label", ++ pod: makePodWithLabels(map[string]string{ ++ "test": "label", ++ }), ++ namespaceFunc: getFakeInfraPodNamespace, ++ expectedValue: true, ++ }, ++ { ++ description: "Pod without platform label and namespace without platform label", ++ pod: makePodWithLabels(map[string]string{ ++ "test": "namespace", ++ }), ++ namespaceFunc: getFakeNonInfraPodNamespace, ++ expectedValue: false, ++ }, ++ ++ } ++ ++ for _, testCase := range testCases { ++ t.Run(testCase.description, func(t *testing.T) { ++ ++ varGetNamespaceObject = testCase.namespaceFunc ++ varBuildConfigFromFlags = fakeBuildConfigFromFlags ++ gotValue := isKubeInfra(testCase.pod) ++ ++ if gotValue != testCase.expectedValue { ++ t.Errorf("StaticPolicy isKubeInfraPod() error %v. expected value %v actual value %v", ++ testCase.description, testCase.expectedValue, gotValue) ++ } else { ++ fmt.Printf("StaticPolicy isKubeInfraPod() test successful. : %v ", testCase.description) ++ } ++ ++ }) ++ } ++ ++ test := kubeInfraPodTestCase{ ++ description: "Failure reading kubeconfig file", ++ pod: makePodWithLabels(map[string]string{ ++ "test": "namespace", ++ }), ++ namespaceFunc: getFakeNonInfraPodNamespace, ++ expectedValue: false, ++ } ++ ++ varGetNamespaceObject = getPodNamespaceObject ++ varBuildConfigFromFlags = fakeBuildConfigFromFlagsError ++ ++ gotValue := isKubeInfra(test.pod) ++ ++ if gotValue != test.expectedValue { ++ t.Errorf("StaticPolicy isKubeInfraPod() error %v. expected value %v actual value %v", ++ test.description, test.expectedValue, gotValue) ++ } else { ++ fmt.Printf("StaticPolicy isKubeInfraPod() test successful. : %v ", test.description) ++ } ++ ++} diff --git a/pkg/kubelet/cm/devicemanager/manager_stub.go b/pkg/kubelet/cm/devicemanager/manager_stub.go new file mode 100644 index 00000000000..e6874f88d8a diff --git a/kubernetes/kubernetes-1.28.4/debian/deb_folder/patches/kubelet-cpumanager-introduce-concept-of-isolated-CPU.patch b/kubernetes/kubernetes-1.28.4/debian/deb_folder/patches/kubelet-cpumanager-introduce-concept-of-isolated-CPU.patch index 675d624dd..3cd9b4a9a 100644 --- a/kubernetes/kubernetes-1.28.4/debian/deb_folder/patches/kubelet-cpumanager-introduce-concept-of-isolated-CPU.patch +++ b/kubernetes/kubernetes-1.28.4/debian/deb_folder/patches/kubelet-cpumanager-introduce-concept-of-isolated-CPU.patch @@ -1,4 +1,4 @@ -From 856dfbe0960418618262998ebce65d0ae070c1bb Mon Sep 17 00:00:00 2001 +From 4f74e4f9bbfd2909a3c740cb6a1b5233af277f72 Mon Sep 17 00:00:00 2001 From: Saba Touheed Mujawar Date: Fri, 1 Dec 2023 07:42:14 -0500 Subject: [PATCH] kubelet cpumanager introduce concept of isolated CPUs @@ -48,14 +48,15 @@ Signed-off-by: Ramesh Kumar Sivanandam Signed-off-by: Sachin Gopala Krishna Signed-off-by: Boovan Rajendran Signed-off-by: Saba Touheed Mujawar +Signed-off-by: Kaustubh Dhokte --- pkg/kubelet/cm/container_manager_linux.go | 1 + - pkg/kubelet/cm/cpumanager/cpu_manager.go | 35 +++++- - pkg/kubelet/cm/cpumanager/cpu_manager_test.go | 23 +++- - pkg/kubelet/cm/cpumanager/policy_static.go | 83 ++++++++++++- - .../cm/cpumanager/policy_static_test.go | 53 +++++++-- - pkg/kubelet/cm/devicemanager/manager_stub.go | 110 ++++++++++++++++++ - 6 files changed, 284 insertions(+), 21 deletions(-) + pkg/kubelet/cm/cpumanager/cpu_manager.go | 35 +++- + pkg/kubelet/cm/cpumanager/cpu_manager_test.go | 23 ++- + pkg/kubelet/cm/cpumanager/policy_static.go | 157 +++++++++++++-- + .../cm/cpumanager/policy_static_test.go | 178 +++++++++++++++++- + pkg/kubelet/cm/devicemanager/manager_stub.go | 110 +++++++++++ + 6 files changed, 474 insertions(+), 30 deletions(-) create mode 100644 pkg/kubelet/cm/devicemanager/manager_stub.go diff --git a/pkg/kubelet/cm/container_manager_linux.go b/pkg/kubelet/cm/container_manager_linux.go @@ -266,18 +267,26 @@ index daecd35f67b..2298cc037fe 100644 testCases := []struct { diff --git a/pkg/kubelet/cm/cpumanager/policy_static.go b/pkg/kubelet/cm/cpumanager/policy_static.go -index 9b7545c2207..e9a2defd848 100644 +index 9b7545c2207..e32803306c0 100644 --- a/pkg/kubelet/cm/cpumanager/policy_static.go +++ b/pkg/kubelet/cm/cpumanager/policy_static.go -@@ -18,6 +18,7 @@ package cpumanager +@@ -17,9 +17,15 @@ limitations under the License. + package cpumanager import ( ++ "context" "fmt" + "strconv" ++ k8sclient "k8s.io/client-go/kubernetes" ++ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ++ restclient "k8s.io/client-go/rest" v1 "k8s.io/api/core/v1" ++ "k8s.io/client-go/tools/clientcmd" utilfeature "k8s.io/apiserver/pkg/util/feature" -@@ -27,6 +28,7 @@ import ( + "k8s.io/klog/v2" + podutil "k8s.io/kubernetes/pkg/api/v1/pod" +@@ -27,6 +33,7 @@ import ( "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology" @@ -285,7 +294,32 @@ index 9b7545c2207..e9a2defd848 100644 "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager" "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask" "k8s.io/kubernetes/pkg/kubelet/metrics" -@@ -110,6 +112,10 @@ type staticPolicy struct { +@@ -43,6 +50,12 @@ const ( + ErrorSMTAlignment = "SMTAlignmentError" + ) + ++type getPodNamespace func(string) (*v1.Namespace, error) ++type buildFromConfigFlag func(masterUrl string, kubeconfigPath string) (*restclient.Config, error) ++ ++var varGetNamespaceObject getPodNamespace ++var varBuildConfigFromFlags buildFromConfigFlag ++ + // SMTAlignmentError represents an error due to SMT alignment + type SMTAlignmentError struct { + RequestedCPUs int +@@ -62,11 +75,6 @@ func (e SMTAlignmentError) Type() string { + return ErrorSMTAlignment + } + +-// Define namespaces used by platform infrastructure pods +-var infraNamespaces = [...]string{ +- "kube-system", "armada", "cert-manager", "platform-deployment-manager", "portieris", "vault", "notification", "flux-helm", "metrics-server", +-} +- + // staticPolicy is a CPU manager policy that does not change CPU + // assignments for exclusively pinned guaranteed containers after the main + // container process starts. +@@ -110,6 +118,10 @@ type staticPolicy struct { topology *topology.CPUTopology // set of CPUs that is not available for exclusive assignment reservedCPUs cpuset.CPUSet @@ -296,7 +330,7 @@ index 9b7545c2207..e9a2defd848 100644 // If true, default CPUSet should exclude reserved CPUs excludeReserved bool // Superset of reservedCPUs. It includes not just the reservedCPUs themselves, -@@ -132,7 +138,8 @@ var _ Policy = &staticPolicy{} +@@ -132,7 +144,8 @@ var _ Policy = &staticPolicy{} // NewStaticPolicy returns a CPU manager policy that does not change CPU // assignments for exclusively pinned guaranteed containers after the main // container process starts. @@ -306,7 +340,7 @@ index 9b7545c2207..e9a2defd848 100644 opts, err := NewStaticPolicyOptions(cpuPolicyOptions) if err != nil { return nil, err -@@ -147,6 +154,8 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv +@@ -147,6 +160,8 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv policy := &staticPolicy{ topology: topology, affinity: affinity, @@ -315,7 +349,7 @@ index 9b7545c2207..e9a2defd848 100644 excludeReserved: excludeReserved, cpusToReuse: make(map[string]cpuset.CPUSet), options: opts, -@@ -183,6 +192,12 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv +@@ -183,6 +198,12 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv policy.reservedCPUs = reserved policy.reservedPhysicalCPUs = reservedPhysicalCPUs @@ -328,7 +362,7 @@ index 9b7545c2207..e9a2defd848 100644 return policy, nil } -@@ -216,8 +231,9 @@ func (p *staticPolicy) validateState(s state.State) error { +@@ -216,8 +237,9 @@ func (p *staticPolicy) validateState(s state.State) error { } else { s.SetDefaultCPUSet(allCPUs) } @@ -340,7 +374,17 @@ index 9b7545c2207..e9a2defd848 100644 return nil } -@@ -316,16 +332,39 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai +@@ -307,6 +329,9 @@ func (p *staticPolicy) updateCPUsToReuse(pod *v1.Pod, container *v1.Container, c + } + + func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Container) (rerr error) { ++ ++ varGetNamespaceObject = getPodNamespaceObject ++ varBuildConfigFromFlags = clientcmd.BuildConfigFromFlags + // Process infra pods before guaranteed pods + if isKubeInfra(pod) { + // Container belongs in reserved pool. +@@ -316,16 +341,39 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai return nil } @@ -382,7 +426,7 @@ index 9b7545c2207..e9a2defd848 100644 numCPUs := p.guaranteedCPUs(pod, container) if numCPUs == 0 { // container belongs in the shared pool (nothing to do; use default cpuset) -@@ -391,7 +430,9 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai +@@ -391,7 +439,9 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai } s.SetCPUSet(string(pod.UID), container.Name, cpuset) p.updateCPUsToReuse(pod, container, cpuset) @@ -393,10 +437,71 @@ index 9b7545c2207..e9a2defd848 100644 return nil } -@@ -709,6 +750,36 @@ func isKubeInfra(pod *v1.Pod) bool { - return false +@@ -699,14 +749,91 @@ func (p *staticPolicy) generateCPUTopologyHints(availableCPUs cpuset.CPUSet, reu + return hints } ++func getPodNamespaceObject(podNamespaceName string) (*v1.Namespace, error) { ++ ++ cfg, err := varBuildConfigFromFlags("", "/etc/kubernetes/kubelet.conf") ++ if err != nil { ++ klog.Error("Failed to build client config from /etc/kubernetes/kubelet.conf: ", err.Error()) ++ return nil, err ++ } ++ ++ clientset, err := k8sclient.NewForConfig(cfg) ++ if err != nil { ++ klog.Error("Failed to get clientset for KUBECONFIG /etc/kubernetes/kubelet.conf: ", err.Error()) ++ return nil, err ++ } ++ ++ namespaceObj, err := clientset.CoreV1().Namespaces().Get(context.TODO(), podNamespaceName, metav1.GetOptions{}) ++ if err != nil { ++ klog.Error("Error getting namespace object:", err.Error()) ++ return nil, err ++ } ++ ++ return namespaceObj, nil ++ ++} ++ + // check if a given pod is in a platform infrastructure namespace + func isKubeInfra(pod *v1.Pod) bool { +- for _, namespace := range infraNamespaces { +- if namespace == pod.Namespace { +- return true +- } ++ ++ podName := pod.GetName() ++ podNamespaceName := pod.GetNamespace() ++ ++ klog.InfoS("Checking pod ", podName , " for label 'app.starlingx.io/component=platform'.") ++ podLabels := pod.GetLabels() ++ val, ok := podLabels["app.starlingx.io/component"] ++ if (ok && val == "platform") { ++ klog.InfoS("Pod ", podName, " has 'app.starlingx.io/component=platform' label. Assigning platform CPUs.") ++ return true + } ++ ++ klog.InfoS("Pod ", pod.GetName(), " does not have 'app.starlingx.io/component=platform' label. Checking its namespace information...") ++ ++ namespaceObj, err := varGetNamespaceObject(podNamespaceName) ++ if err != nil { ++ return false ++ } ++ ++ namespaceLabels := namespaceObj.GetLabels() ++ val, ok = namespaceLabels["app.starlingx.io/component"] ++ if ok && val == "platform" { ++ klog.InfoS("For pod: ", podName, ", its Namespace ", podNamespaceName, " has 'app.starlingx.io/component=platform' label. Assigning platform CPUs.") ++ return true ++ } ++ ++ klog.InfoS("Neither pod ", podName, " nor its namespace ", podNamespaceName, " has 'app.starlingx.io/component=platform' label. Not assigning platform CPUs.") + return false ++ ++} ++ +// get the isolated CPUs (if any) from the devices associated with a specific container +func (p *staticPolicy) podIsolCPUs(pod *v1.Pod, container *v1.Container) cpuset.CPUSet { + // NOTE: This is required for TestStaticPolicyAdd() since makePod() does @@ -425,16 +530,27 @@ index 9b7545c2207..e9a2defd848 100644 + } + } + return cpuSet -+} -+ + } + // isHintSocketAligned function return true if numa nodes in hint are socket aligned. - func (p *staticPolicy) isHintSocketAligned(hint topologymanager.TopologyHint, minAffinitySize int) bool { - numaNodesBitMask := hint.NUMANodeAffinity.GetBits() diff --git a/pkg/kubelet/cm/cpumanager/policy_static_test.go b/pkg/kubelet/cm/cpumanager/policy_static_test.go -index b864c6c57c6..cb363bb29ab 100644 +index b864c6c57c6..d94f8fdac14 100644 --- a/pkg/kubelet/cm/cpumanager/policy_static_test.go +++ b/pkg/kubelet/cm/cpumanager/policy_static_test.go -@@ -27,6 +27,7 @@ import ( +@@ -17,16 +17,20 @@ limitations under the License. + package cpumanager + + import ( ++ "errors" + "fmt" + "reflect" + "testing" + ++ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ++ restclient "k8s.io/client-go/rest" + v1 "k8s.io/api/core/v1" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" pkgfeatures "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology" @@ -442,7 +558,7 @@ index b864c6c57c6..cb363bb29ab 100644 "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager" "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask" "k8s.io/utils/cpuset" -@@ -70,8 +71,9 @@ func (spt staticPolicyTest) PseudoClone() staticPolicyTest { +@@ -70,8 +74,9 @@ func (spt staticPolicyTest) PseudoClone() staticPolicyTest { } func TestStaticPolicyName(t *testing.T) { @@ -453,7 +569,7 @@ index b864c6c57c6..cb363bb29ab 100644 policyName := policy.Name() if policyName != "static" { -@@ -81,6 +83,7 @@ func TestStaticPolicyName(t *testing.T) { +@@ -81,6 +86,7 @@ func TestStaticPolicyName(t *testing.T) { } func TestStaticPolicyStart(t *testing.T) { @@ -461,7 +577,7 @@ index b864c6c57c6..cb363bb29ab 100644 testCases := []staticPolicyTest{ { description: "non-corrupted state", -@@ -156,7 +159,7 @@ func TestStaticPolicyStart(t *testing.T) { +@@ -156,7 +162,7 @@ func TestStaticPolicyStart(t *testing.T) { } for _, testCase := range testCases { t.Run(testCase.description, func(t *testing.T) { @@ -470,7 +586,7 @@ index b864c6c57c6..cb363bb29ab 100644 policy := p.(*staticPolicy) st := &mockState{ -@@ -204,7 +207,6 @@ func TestStaticPolicyAdd(t *testing.T) { +@@ -204,7 +210,6 @@ func TestStaticPolicyAdd(t *testing.T) { largeTopoCPUSet := cpuset.New(largeTopoCPUids...) largeTopoSock0CPUSet := cpuset.New(largeTopoSock0CPUids...) largeTopoSock1CPUSet := cpuset.New(largeTopoSock1CPUids...) @@ -478,7 +594,7 @@ index b864c6c57c6..cb363bb29ab 100644 // these are the cases which must behave the same regardless the policy options. // So we will permutate the options to ensure this holds true. -@@ -627,7 +629,9 @@ func runStaticPolicyTestCase(t *testing.T, testCase staticPolicyTest) { +@@ -627,7 +632,9 @@ func runStaticPolicyTestCase(t *testing.T, testCase staticPolicyTest) { cpus = testCase.reservedCPUs.Clone() } testExcl := false @@ -489,7 +605,7 @@ index b864c6c57c6..cb363bb29ab 100644 st := &mockState{ assignments: testCase.stAssignments, -@@ -674,6 +678,8 @@ func runStaticPolicyTestCaseWithFeatureGate(t *testing.T, testCase staticPolicyT +@@ -674,6 +681,8 @@ func runStaticPolicyTestCaseWithFeatureGate(t *testing.T, testCase staticPolicyT } func TestStaticPolicyReuseCPUs(t *testing.T) { @@ -498,7 +614,7 @@ index b864c6c57c6..cb363bb29ab 100644 testCases := []struct { staticPolicyTest expCSetAfterAlloc cpuset.CPUSet -@@ -698,7 +704,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) { +@@ -698,7 +707,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) { } for _, testCase := range testCases { @@ -507,7 +623,7 @@ index b864c6c57c6..cb363bb29ab 100644 st := &mockState{ assignments: testCase.stAssignments, -@@ -731,6 +737,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) { +@@ -731,6 +740,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) { func TestStaticPolicyRemove(t *testing.T) { excludeReserved := false @@ -515,7 +631,7 @@ index b864c6c57c6..cb363bb29ab 100644 testCases := []staticPolicyTest{ { description: "SingleSocketHT, DeAllocOneContainer", -@@ -789,7 +796,7 @@ func TestStaticPolicyRemove(t *testing.T) { +@@ -789,7 +799,7 @@ func TestStaticPolicyRemove(t *testing.T) { } for _, testCase := range testCases { @@ -524,7 +640,7 @@ index b864c6c57c6..cb363bb29ab 100644 st := &mockState{ assignments: testCase.stAssignments, -@@ -812,6 +819,7 @@ func TestStaticPolicyRemove(t *testing.T) { +@@ -812,6 +822,7 @@ func TestStaticPolicyRemove(t *testing.T) { func TestTopologyAwareAllocateCPUs(t *testing.T) { excludeReserved := false @@ -532,7 +648,7 @@ index b864c6c57c6..cb363bb29ab 100644 testCases := []struct { description string topo *topology.CPUTopology -@@ -880,7 +888,8 @@ func TestTopologyAwareAllocateCPUs(t *testing.T) { +@@ -880,7 +891,8 @@ func TestTopologyAwareAllocateCPUs(t *testing.T) { }, } for _, tc := range testCases { @@ -542,7 +658,7 @@ index b864c6c57c6..cb363bb29ab 100644 policy := p.(*staticPolicy) st := &mockState{ assignments: tc.stAssignments, -@@ -913,6 +922,7 @@ type staticPolicyTestWithResvList struct { +@@ -913,6 +925,7 @@ type staticPolicyTestWithResvList struct { topo *topology.CPUTopology numReservedCPUs int reserved cpuset.CPUSet @@ -550,7 +666,7 @@ index b864c6c57c6..cb363bb29ab 100644 stAssignments state.ContainerCPUAssignments stDefaultCPUSet cpuset.CPUSet pod *v1.Pod -@@ -923,6 +933,8 @@ type staticPolicyTestWithResvList struct { +@@ -923,6 +936,8 @@ type staticPolicyTestWithResvList struct { } func TestStaticPolicyStartWithResvList(t *testing.T) { @@ -559,7 +675,7 @@ index b864c6c57c6..cb363bb29ab 100644 testCases := []staticPolicyTestWithResvList{ { description: "empty cpuset", -@@ -952,10 +964,9 @@ func TestStaticPolicyStartWithResvList(t *testing.T) { +@@ -952,10 +967,9 @@ func TestStaticPolicyStartWithResvList(t *testing.T) { expNewErr: fmt.Errorf("[cpumanager] unable to reserve the required amount of CPUs (size of 0-1 did not equal 1)"), }, } @@ -571,7 +687,7 @@ index b864c6c57c6..cb363bb29ab 100644 if !reflect.DeepEqual(err, testCase.expNewErr) { t.Errorf("StaticPolicy Start() error (%v). expected error: %v but got: %v", -@@ -996,6 +1007,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { +@@ -996,6 +1010,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { topo: topoSingleSocketHT, numReservedCPUs: 1, reserved: cpuset.New(0), @@ -579,7 +695,7 @@ index b864c6c57c6..cb363bb29ab 100644 stAssignments: state.ContainerCPUAssignments{}, stDefaultCPUSet: cpuset.New(1, 2, 3, 4, 5, 6, 7), pod: makePod("fakePod", "fakeContainer2", "8000m", "8000m"), -@@ -1008,6 +1020,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { +@@ -1008,6 +1023,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { topo: topoSingleSocketHT, numReservedCPUs: 2, reserved: cpuset.New(0, 1), @@ -587,7 +703,7 @@ index b864c6c57c6..cb363bb29ab 100644 stAssignments: state.ContainerCPUAssignments{}, stDefaultCPUSet: cpuset.New(2, 3, 4, 5, 6, 7), pod: makePod("fakePod", "fakeContainer2", "1000m", "1000m"), -@@ -1020,6 +1033,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { +@@ -1020,6 +1036,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { topo: topoSingleSocketHT, numReservedCPUs: 2, reserved: cpuset.New(0, 1), @@ -595,7 +711,7 @@ index b864c6c57c6..cb363bb29ab 100644 stAssignments: state.ContainerCPUAssignments{ "fakePod": map[string]cpuset.CPUSet{ "fakeContainer100": cpuset.New(2, 3, 6, 7), -@@ -1036,6 +1050,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { +@@ -1036,6 +1053,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { topo: topoSingleSocketHT, numReservedCPUs: 2, reserved: cpuset.New(0, 1), @@ -603,7 +719,7 @@ index b864c6c57c6..cb363bb29ab 100644 stAssignments: state.ContainerCPUAssignments{ "fakePod": map[string]cpuset.CPUSet{ "fakeContainer100": cpuset.New(2, 3, 6, 7), -@@ -1047,11 +1062,29 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { +@@ -1047,11 +1065,29 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { expCPUAlloc: true, expCSet: cpuset.New(0, 1), }, @@ -634,6 +750,132 @@ index b864c6c57c6..cb363bb29ab 100644 st := &mockState{ assignments: testCase.stAssignments, +@@ -1182,3 +1218,125 @@ func newCPUSetPtr(cpus ...int) *cpuset.CPUSet { + ret := cpuset.New(cpus...) + return &ret + } ++ ++func makePodWithLabels(podLabels map[string]string) *v1.Pod { ++ return &v1.Pod{ ++ ObjectMeta: metav1.ObjectMeta{ ++ Name: "test-pod", ++ Namespace: "test-namespace", ++ Labels: podLabels, ++ }, ++ } ++} ++ ++func fakeBuildConfigFromFlags(masterUrl string, kubeconfigPath string) (*restclient.Config, error) { ++ ++ return &restclient.Config{}, nil ++} ++ ++func fakeBuildConfigFromFlagsError(masterUrl string, kubeconfigPath string) (*restclient.Config, error) { ++ ++ errString := fmt.Sprintf("%s file not found", kubeconfigPath) ++ return nil, errors.New(errString) ++ ++} ++ ++func getFakeInfraPodNamespace(_ string) (*v1.Namespace, error) { ++ ++ return &v1.Namespace{ ++ ObjectMeta: metav1.ObjectMeta{ ++ Name: "test-namespace", ++ Labels: map[string]string{ ++ "app.starlingx.io/component": "platform", ++ }, ++ }}, nil ++} ++ ++func getFakeNonInfraPodNamespace(_ string) (*v1.Namespace, error) { ++ ++ return &v1.Namespace{ ++ ObjectMeta: metav1.ObjectMeta{ ++ Name: "test-namespace", ++ Labels: map[string]string{ ++ "fake": "label", ++ }}}, nil ++ ++} ++ ++type kubeInfraPodTestCase struct { ++ description string ++ pod *v1.Pod ++ namespaceFunc getPodNamespace ++ expectedValue bool ++} ++ ++func TestKubeInfraPod(t *testing.T) { ++ testCases := []kubeInfraPodTestCase{ ++ { ++ description: "Pod with platform label and namespace without platform label", ++ pod: makePodWithLabels(map[string]string{ ++ "app.starlingx.io/component": "platform", ++ }), ++ namespaceFunc: getFakeNonInfraPodNamespace, ++ expectedValue: true, ++ ++ }, ++ { ++ description: "Pod without platform label and namespace with platform label", ++ pod: makePodWithLabels(map[string]string{ ++ "test": "label", ++ }), ++ namespaceFunc: getFakeInfraPodNamespace, ++ expectedValue: true, ++ }, ++ { ++ description: "Pod without platform label and namespace without platform label", ++ pod: makePodWithLabels(map[string]string{ ++ "test": "namespace", ++ }), ++ namespaceFunc: getFakeNonInfraPodNamespace, ++ expectedValue: false, ++ }, ++ ++ } ++ ++ for _, testCase := range testCases { ++ t.Run(testCase.description, func(t *testing.T) { ++ ++ varGetNamespaceObject = testCase.namespaceFunc ++ varBuildConfigFromFlags = fakeBuildConfigFromFlags ++ gotValue := isKubeInfra(testCase.pod) ++ ++ if gotValue != testCase.expectedValue { ++ t.Errorf("StaticPolicy isKubeInfraPod() error %v. expected value %v actual value %v", ++ testCase.description, testCase.expectedValue, gotValue) ++ } else { ++ fmt.Printf("StaticPolicy isKubeInfraPod() test successful. : %v ", testCase.description) ++ } ++ ++ }) ++ } ++ ++ test := kubeInfraPodTestCase{ ++ description: "Failure reading kubeconfig file", ++ pod: makePodWithLabels(map[string]string{ ++ "test": "namespace", ++ }), ++ namespaceFunc: getFakeNonInfraPodNamespace, ++ expectedValue: false, ++ } ++ ++ varGetNamespaceObject = getPodNamespaceObject ++ varBuildConfigFromFlags = fakeBuildConfigFromFlagsError ++ ++ gotValue := isKubeInfra(test.pod) ++ ++ if gotValue != test.expectedValue { ++ t.Errorf("StaticPolicy isKubeInfraPod() error %v. expected value %v actual value %v", ++ test.description, test.expectedValue, gotValue) ++ } else { ++ fmt.Printf("StaticPolicy isKubeInfraPod() test successful. : %v ", test.description) ++ } ++ ++} ++ diff --git a/pkg/kubelet/cm/devicemanager/manager_stub.go b/pkg/kubelet/cm/devicemanager/manager_stub.go new file mode 100644 index 00000000000..98abcde2519