Merge "Identify platform pods based on pod/namespace labels"

This commit is contained in:
Zuul 2024-03-07 20:33:00 +00:00 committed by Gerrit Code Review
commit a933669618
5 changed files with 1427 additions and 219 deletions

View File

@ -1,4 +1,4 @@
From ae7fc7b39bfde784340068b388a13a28b4e76398 Mon Sep 17 00:00:00 2001 From 66130d332c561ab95853e1277f5076f6070c3002 Mon Sep 17 00:00:00 2001
From: Gleb Aronsky <gleb.aronsky@windriver.com> From: Gleb Aronsky <gleb.aronsky@windriver.com>
Date: Tue, 25 Jan 2022 13:27:25 -0500 Date: Tue, 25 Jan 2022 13:27:25 -0500
Subject: [PATCH] kubelet cpumanager introduce concept of isolated CPUs Subject: [PATCH] kubelet cpumanager introduce concept of isolated CPUs
@ -44,13 +44,14 @@ to the right.
Co-authored-by: Jim Gauld <james.gauld@windriver.com> Co-authored-by: Jim Gauld <james.gauld@windriver.com>
Co-authored-by: Chris Friesen <chris.friesen@windriver.com> Co-authored-by: Chris Friesen <chris.friesen@windriver.com>
Signed-off-by: Gleb Aronsky <gleb.aronsky@windriver.com> Signed-off-by: Gleb Aronsky <gleb.aronsky@windriver.com>
Signed-off-by: Kaustubh Dhokte <kaustubh.dhokte@windriver.com>
--- ---
pkg/kubelet/cm/container_manager_linux.go | 1 + pkg/kubelet/cm/container_manager_linux.go | 1 +
pkg/kubelet/cm/cpumanager/cpu_manager.go | 35 +++++++- pkg/kubelet/cm/cpumanager/cpu_manager.go | 35 +++-
pkg/kubelet/cm/cpumanager/cpu_manager_test.go | 14 +++- pkg/kubelet/cm/cpumanager/cpu_manager_test.go | 14 +-
pkg/kubelet/cm/cpumanager/policy_static.go | 83 +++++++++++++++++-- pkg/kubelet/cm/cpumanager/policy_static.go | 157 ++++++++++++++--
.../cm/cpumanager/policy_static_test.go | 50 ++++++++--- .../cm/cpumanager/policy_static_test.go | 176 +++++++++++++++++-
5 files changed, 164 insertions(+), 19 deletions(-) 5 files changed, 355 insertions(+), 28 deletions(-)
diff --git a/pkg/kubelet/cm/container_manager_linux.go b/pkg/kubelet/cm/container_manager_linux.go diff --git a/pkg/kubelet/cm/container_manager_linux.go b/pkg/kubelet/cm/container_manager_linux.go
index 0f09f3eb331..770922ca55d 100644 index 0f09f3eb331..770922ca55d 100644
@ -212,18 +213,24 @@ index 2c8349662c4..31e4d0585fb 100644
testCases := []struct { testCases := []struct {
description string description string
diff --git a/pkg/kubelet/cm/cpumanager/policy_static.go b/pkg/kubelet/cm/cpumanager/policy_static.go diff --git a/pkg/kubelet/cm/cpumanager/policy_static.go b/pkg/kubelet/cm/cpumanager/policy_static.go
index a3c93a896df..d6fe69e7165 100644 index 216b6ce9bf8..30c0afaca32 100644
--- a/pkg/kubelet/cm/cpumanager/policy_static.go --- a/pkg/kubelet/cm/cpumanager/policy_static.go
+++ b/pkg/kubelet/cm/cpumanager/policy_static.go +++ b/pkg/kubelet/cm/cpumanager/policy_static.go
@@ -18,6 +18,7 @@ package cpumanager @@ -17,14 +17,21 @@ limitations under the License.
package cpumanager
import ( import (
+ "context"
"fmt" "fmt"
+ "strconv" + "strconv"
+ k8sclient "k8s.io/client-go/kubernetes"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ restclient "k8s.io/client-go/rest"
v1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1"
+ "k8s.io/client-go/tools/clientcmd"
"k8s.io/klog/v2" "k8s.io/klog/v2"
@@ -25,6 +26,7 @@ import ( v1qos "k8s.io/kubernetes/pkg/apis/core/v1/helper/qos"
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state"
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
"k8s.io/kubernetes/pkg/kubelet/cm/cpuset" "k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
@ -231,7 +238,32 @@ index a3c93a896df..d6fe69e7165 100644
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager" "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager"
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask" "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask"
) )
@@ -101,6 +103,10 @@ type staticPolicy struct { @@ -39,6 +46,12 @@ const (
ErrorSMTAlignment = "SMTAlignmentError"
)
+type getPodNamespace func(string) (*v1.Namespace, error)
+type buildFromConfigFlag func(masterUrl string, kubeconfigPath string) (*restclient.Config, error)
+
+var varGetNamespaceObject getPodNamespace
+var varBuildConfigFromFlags buildFromConfigFlag
+
// SMTAlignmentError represents an error due to SMT alignment
type SMTAlignmentError struct {
RequestedCPUs int
@@ -53,11 +66,6 @@ func (e SMTAlignmentError) Type() string {
return ErrorSMTAlignment
}
-// Define namespaces used by platform infrastructure pods
-var infraNamespaces = [...]string{
- "kube-system", "armada", "cert-manager", "platform-deployment-manager", "portieris", "vault", "notification", "flux-helm", "metrics-server", "node-feature-discovery", "intel-power", "power-metrics", "sriov-fec-system",
-}
-
// staticPolicy is a CPU manager policy that does not change CPU
// assignments for exclusively pinned guaranteed containers after the main
// container process starts.
@@ -101,6 +109,10 @@ type staticPolicy struct {
topology *topology.CPUTopology topology *topology.CPUTopology
// set of CPUs that is not available for exclusive assignment // set of CPUs that is not available for exclusive assignment
reserved cpuset.CPUSet reserved cpuset.CPUSet
@ -242,7 +274,7 @@ index a3c93a896df..d6fe69e7165 100644
// If true, default CPUSet should exclude reserved CPUs // If true, default CPUSet should exclude reserved CPUs
excludeReserved bool excludeReserved bool
// topology manager reference to get container Topology affinity // topology manager reference to get container Topology affinity
@@ -117,7 +123,8 @@ var _ Policy = &staticPolicy{} @@ -117,7 +129,8 @@ var _ Policy = &staticPolicy{}
// NewStaticPolicy returns a CPU manager policy that does not change CPU // NewStaticPolicy returns a CPU manager policy that does not change CPU
// assignments for exclusively pinned guaranteed containers after the main // assignments for exclusively pinned guaranteed containers after the main
// container process starts. // container process starts.
@ -252,7 +284,7 @@ index a3c93a896df..d6fe69e7165 100644
opts, err := NewStaticPolicyOptions(cpuPolicyOptions) opts, err := NewStaticPolicyOptions(cpuPolicyOptions)
if err != nil { if err != nil {
return nil, err return nil, err
@@ -128,6 +135,8 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv @@ -128,6 +141,8 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv
policy := &staticPolicy{ policy := &staticPolicy{
topology: topology, topology: topology,
affinity: affinity, affinity: affinity,
@ -261,7 +293,7 @@ index a3c93a896df..d6fe69e7165 100644
excludeReserved: excludeReserved, excludeReserved: excludeReserved,
cpusToReuse: make(map[string]cpuset.CPUSet), cpusToReuse: make(map[string]cpuset.CPUSet),
options: opts, options: opts,
@@ -154,6 +163,12 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv @@ -154,6 +169,12 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv
klog.InfoS("Reserved CPUs not available for exclusive assignment", "reservedSize", reserved.Size(), "reserved", reserved) klog.InfoS("Reserved CPUs not available for exclusive assignment", "reservedSize", reserved.Size(), "reserved", reserved)
policy.reserved = reserved policy.reserved = reserved
@ -274,7 +306,7 @@ index a3c93a896df..d6fe69e7165 100644
return policy, nil return policy, nil
} }
@@ -187,8 +202,9 @@ func (p *staticPolicy) validateState(s state.State) error { @@ -187,8 +208,9 @@ func (p *staticPolicy) validateState(s state.State) error {
} else { } else {
s.SetDefaultCPUSet(allCPUs) s.SetDefaultCPUSet(allCPUs)
} }
@ -286,7 +318,17 @@ index a3c93a896df..d6fe69e7165 100644
return nil return nil
} }
@@ -278,10 +294,11 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai @@ -269,6 +291,9 @@ func (p *staticPolicy) updateCPUsToReuse(pod *v1.Pod, container *v1.Container, c
}
func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Container) error {
+
+ varGetNamespaceObject = getPodNamespaceObject
+ varBuildConfigFromFlags = clientcmd.BuildConfigFromFlags
// Process infra pods before guaranteed pods
if isKubeInfra(pod) {
// Container belongs in reserved pool.
@@ -278,10 +303,11 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai
return nil return nil
} }
@ -300,7 +342,7 @@ index a3c93a896df..d6fe69e7165 100644
} }
s.SetCPUSet(string(pod.UID), container.Name, cpuset) s.SetCPUSet(string(pod.UID), container.Name, cpuset)
klog.Infof("[cpumanager] static policy: reserved: AddContainer (namespace: %s, pod UID: %s, pod: %s, container: %s); cpuset=%v", pod.Namespace, string(pod.UID), pod.Name, container.Name, cpuset) klog.Infof("[cpumanager] static policy: reserved: AddContainer (namespace: %s, pod UID: %s, pod: %s, container: %s); cpuset=%v", pod.Namespace, string(pod.UID), pod.Name, container.Name, cpuset)
@@ -325,8 +342,34 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai @@ -325,8 +351,34 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai
} }
s.SetCPUSet(string(pod.UID), container.Name, cpuset) s.SetCPUSet(string(pod.UID), container.Name, cpuset)
p.updateCPUsToReuse(pod, container, cpuset) p.updateCPUsToReuse(pod, container, cpuset)
@ -335,10 +377,70 @@ index a3c93a896df..d6fe69e7165 100644
// container belongs in the shared pool (nothing to do; use default cpuset) // container belongs in the shared pool (nothing to do; use default cpuset)
return nil return nil
} }
@@ -634,3 +677,33 @@ func isKubeInfra(pod *v1.Pod) bool { @@ -625,12 +677,89 @@ func (p *staticPolicy) generateCPUTopologyHints(availableCPUs cpuset.CPUSet, reu
} return hints
return false
} }
+func getPodNamespaceObject(podNamespaceName string) (*v1.Namespace, error) {
+
+ cfg, err := varBuildConfigFromFlags("", "/etc/kubernetes/kubelet.conf")
+ if err != nil {
+ klog.Error("Failed to build client config from /etc/kubernetes/kubelet.conf: ", err.Error())
+ return nil, err
+ }
+
+ clientset, err := k8sclient.NewForConfig(cfg)
+ if err != nil {
+ klog.Error("Failed to get clientset for KUBECONFIG /etc/kubernetes/kubelet.conf: ", err.Error())
+ return nil, err
+ }
+
+ namespaceObj, err := clientset.CoreV1().Namespaces().Get(context.TODO(), podNamespaceName, metav1.GetOptions{})
+ if err != nil {
+ klog.Error("Error getting namespace object:", err.Error())
+ return nil, err
+ }
+
+ return namespaceObj, nil
+
+}
+
// check if a given pod is in a platform infrastructure namespace
func isKubeInfra(pod *v1.Pod) bool {
- for _, namespace := range infraNamespaces {
- if namespace == pod.Namespace {
- return true
- }
+
+ podName := pod.GetName()
+ podNamespaceName := pod.GetNamespace()
+
+ klog.InfoS("Checking pod ", podName , " for label 'app.starlingx.io/component=platform'.")
+ podLabels := pod.GetLabels()
+ val, ok := podLabels["app.starlingx.io/component"]
+ if (ok && val == "platform") {
+ klog.InfoS("Pod ", podName, " has 'app.starlingx.io/component=platform' label. Assigning platform CPUs.")
+ return true
+ }
+
+ klog.InfoS("Pod ", pod.GetName(), " does not have 'app.starlingx.io/component=platform' label. Checking its namespace information...")
+
+ namespaceObj, err := varGetNamespaceObject(podNamespaceName)
+ if err != nil {
+ return false
+ }
+
+ namespaceLabels := namespaceObj.GetLabels()
+ val, ok = namespaceLabels["app.starlingx.io/component"]
+ if ok && val == "platform" {
+ klog.InfoS("For pod: ", podName, ", its Namespace ", podNamespaceName, " has 'app.starlingx.io/component=platform' label. Assigning platform CPUs.")
+ return true
}
+
+ klog.InfoS("Neither pod ", podName, " nor its namespace ", podNamespaceName, " has 'app.starlingx.io/component=platform' label. Not assigning platform CPUs.")
return false
+
+}
+ +
+// get the isolated CPUs (if any) from the devices associated with a specific container +// get the isolated CPUs (if any) from the devices associated with a specific container
+func (p *staticPolicy) podIsolCPUs(pod *v1.Pod, container *v1.Container) cpuset.CPUSet { +func (p *staticPolicy) podIsolCPUs(pod *v1.Pod, container *v1.Container) cpuset.CPUSet {
@ -368,12 +470,24 @@ index a3c93a896df..d6fe69e7165 100644
+ } + }
+ } + }
+ return cpuSet + return cpuSet
+} }
diff --git a/pkg/kubelet/cm/cpumanager/policy_static_test.go b/pkg/kubelet/cm/cpumanager/policy_static_test.go diff --git a/pkg/kubelet/cm/cpumanager/policy_static_test.go b/pkg/kubelet/cm/cpumanager/policy_static_test.go
index d4b4b790210..ecd3e9598d0 100644 index 7938f787a57..fa3e99fb2bb 100644
--- a/pkg/kubelet/cm/cpumanager/policy_static_test.go --- a/pkg/kubelet/cm/cpumanager/policy_static_test.go
+++ b/pkg/kubelet/cm/cpumanager/policy_static_test.go +++ b/pkg/kubelet/cm/cpumanager/policy_static_test.go
@@ -25,6 +25,7 @@ import ( @@ -17,14 +17,19 @@ limitations under the License.
package cpumanager
import (
+ "errors"
"fmt"
"reflect"
"testing"
v1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ restclient "k8s.io/client-go/rest"
+
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state"
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
"k8s.io/kubernetes/pkg/kubelet/cm/cpuset" "k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
@ -381,7 +495,7 @@ index d4b4b790210..ecd3e9598d0 100644
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager" "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager"
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask" "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask"
) )
@@ -65,8 +66,9 @@ func (spt staticPolicyTest) PseudoClone() staticPolicyTest { @@ -65,8 +70,9 @@ func (spt staticPolicyTest) PseudoClone() staticPolicyTest {
} }
func TestStaticPolicyName(t *testing.T) { func TestStaticPolicyName(t *testing.T) {
@ -392,7 +506,7 @@ index d4b4b790210..ecd3e9598d0 100644
policyName := policy.Name() policyName := policy.Name()
if policyName != "static" { if policyName != "static" {
@@ -76,6 +78,7 @@ func TestStaticPolicyName(t *testing.T) { @@ -76,6 +82,7 @@ func TestStaticPolicyName(t *testing.T) {
} }
func TestStaticPolicyStart(t *testing.T) { func TestStaticPolicyStart(t *testing.T) {
@ -400,7 +514,7 @@ index d4b4b790210..ecd3e9598d0 100644
testCases := []staticPolicyTest{ testCases := []staticPolicyTest{
{ {
description: "non-corrupted state", description: "non-corrupted state",
@@ -151,7 +154,7 @@ func TestStaticPolicyStart(t *testing.T) { @@ -151,7 +158,7 @@ func TestStaticPolicyStart(t *testing.T) {
} }
for _, testCase := range testCases { for _, testCase := range testCases {
t.Run(testCase.description, func(t *testing.T) { t.Run(testCase.description, func(t *testing.T) {
@ -409,7 +523,7 @@ index d4b4b790210..ecd3e9598d0 100644
policy := p.(*staticPolicy) policy := p.(*staticPolicy)
st := &mockState{ st := &mockState{
@@ -199,7 +202,7 @@ func TestStaticPolicyAdd(t *testing.T) { @@ -199,7 +206,7 @@ func TestStaticPolicyAdd(t *testing.T) {
largeTopoCPUSet := largeTopoBuilder.Result() largeTopoCPUSet := largeTopoBuilder.Result()
largeTopoSock0CPUSet := largeTopoSock0Builder.Result() largeTopoSock0CPUSet := largeTopoSock0Builder.Result()
largeTopoSock1CPUSet := largeTopoSock1Builder.Result() largeTopoSock1CPUSet := largeTopoSock1Builder.Result()
@ -418,7 +532,7 @@ index d4b4b790210..ecd3e9598d0 100644
// these are the cases which must behave the same regardless the policy options. // these are the cases which must behave the same regardless the policy options.
// So we will permutate the options to ensure this holds true. // So we will permutate the options to ensure this holds true.
optionsInsensitiveTestCases := []staticPolicyTest{ optionsInsensitiveTestCases := []staticPolicyTest{
@@ -529,8 +532,9 @@ func TestStaticPolicyAdd(t *testing.T) { @@ -529,8 +536,9 @@ func TestStaticPolicyAdd(t *testing.T) {
} }
func runStaticPolicyTestCase(t *testing.T, testCase staticPolicyTest) { func runStaticPolicyTestCase(t *testing.T, testCase staticPolicyTest) {
@ -429,7 +543,7 @@ index d4b4b790210..ecd3e9598d0 100644
st := &mockState{ st := &mockState{
assignments: testCase.stAssignments, assignments: testCase.stAssignments,
@@ -596,7 +600,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) { @@ -596,7 +604,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) {
} }
for _, testCase := range testCases { for _, testCase := range testCases {
@ -438,7 +552,7 @@ index d4b4b790210..ecd3e9598d0 100644
st := &mockState{ st := &mockState{
assignments: testCase.stAssignments, assignments: testCase.stAssignments,
@@ -629,6 +633,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) { @@ -629,6 +637,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) {
func TestStaticPolicyRemove(t *testing.T) { func TestStaticPolicyRemove(t *testing.T) {
excludeReserved := false excludeReserved := false
@ -446,7 +560,7 @@ index d4b4b790210..ecd3e9598d0 100644
testCases := []staticPolicyTest{ testCases := []staticPolicyTest{
{ {
description: "SingleSocketHT, DeAllocOneContainer", description: "SingleSocketHT, DeAllocOneContainer",
@@ -710,6 +715,7 @@ func TestStaticPolicyRemove(t *testing.T) { @@ -710,6 +719,7 @@ func TestStaticPolicyRemove(t *testing.T) {
func TestTopologyAwareAllocateCPUs(t *testing.T) { func TestTopologyAwareAllocateCPUs(t *testing.T) {
excludeReserved := false excludeReserved := false
@ -454,7 +568,7 @@ index d4b4b790210..ecd3e9598d0 100644
testCases := []struct { testCases := []struct {
description string description string
topo *topology.CPUTopology topo *topology.CPUTopology
@@ -778,7 +784,8 @@ func TestTopologyAwareAllocateCPUs(t *testing.T) { @@ -778,7 +788,8 @@ func TestTopologyAwareAllocateCPUs(t *testing.T) {
}, },
} }
for _, tc := range testCases { for _, tc := range testCases {
@ -464,7 +578,7 @@ index d4b4b790210..ecd3e9598d0 100644
policy := p.(*staticPolicy) policy := p.(*staticPolicy)
st := &mockState{ st := &mockState{
assignments: tc.stAssignments, assignments: tc.stAssignments,
@@ -811,6 +818,7 @@ type staticPolicyTestWithResvList struct { @@ -811,6 +822,7 @@ type staticPolicyTestWithResvList struct {
topo *topology.CPUTopology topo *topology.CPUTopology
numReservedCPUs int numReservedCPUs int
reserved cpuset.CPUSet reserved cpuset.CPUSet
@ -472,7 +586,7 @@ index d4b4b790210..ecd3e9598d0 100644
stAssignments state.ContainerCPUAssignments stAssignments state.ContainerCPUAssignments
stDefaultCPUSet cpuset.CPUSet stDefaultCPUSet cpuset.CPUSet
pod *v1.Pod pod *v1.Pod
@@ -821,6 +829,8 @@ type staticPolicyTestWithResvList struct { @@ -821,6 +833,8 @@ type staticPolicyTestWithResvList struct {
} }
func TestStaticPolicyStartWithResvList(t *testing.T) { func TestStaticPolicyStartWithResvList(t *testing.T) {
@ -481,7 +595,7 @@ index d4b4b790210..ecd3e9598d0 100644
testCases := []staticPolicyTestWithResvList{ testCases := []staticPolicyTestWithResvList{
{ {
description: "empty cpuset", description: "empty cpuset",
@@ -850,11 +860,9 @@ func TestStaticPolicyStartWithResvList(t *testing.T) { @@ -850,11 +864,9 @@ func TestStaticPolicyStartWithResvList(t *testing.T) {
expNewErr: fmt.Errorf("[cpumanager] unable to reserve the required amount of CPUs (size of 0-1 did not equal 1)"), expNewErr: fmt.Errorf("[cpumanager] unable to reserve the required amount of CPUs (size of 0-1 did not equal 1)"),
}, },
} }
@ -494,7 +608,7 @@ index d4b4b790210..ecd3e9598d0 100644
if !reflect.DeepEqual(err, testCase.expNewErr) { if !reflect.DeepEqual(err, testCase.expNewErr) {
t.Errorf("StaticPolicy Start() error (%v). expected error: %v but got: %v", t.Errorf("StaticPolicy Start() error (%v). expected error: %v but got: %v",
testCase.description, testCase.expNewErr, err) testCase.description, testCase.expNewErr, err)
@@ -894,6 +902,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { @@ -894,6 +906,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
topo: topoSingleSocketHT, topo: topoSingleSocketHT,
numReservedCPUs: 1, numReservedCPUs: 1,
reserved: cpuset.NewCPUSet(0), reserved: cpuset.NewCPUSet(0),
@ -502,7 +616,7 @@ index d4b4b790210..ecd3e9598d0 100644
stAssignments: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{},
stDefaultCPUSet: cpuset.NewCPUSet(1, 2, 3, 4, 5, 6, 7), stDefaultCPUSet: cpuset.NewCPUSet(1, 2, 3, 4, 5, 6, 7),
pod: makePod("fakePod", "fakeContainer2", "8000m", "8000m"), pod: makePod("fakePod", "fakeContainer2", "8000m", "8000m"),
@@ -906,6 +915,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { @@ -906,6 +919,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
topo: topoSingleSocketHT, topo: topoSingleSocketHT,
numReservedCPUs: 2, numReservedCPUs: 2,
reserved: cpuset.NewCPUSet(0, 1), reserved: cpuset.NewCPUSet(0, 1),
@ -510,7 +624,7 @@ index d4b4b790210..ecd3e9598d0 100644
stAssignments: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{},
stDefaultCPUSet: cpuset.NewCPUSet(2, 3, 4, 5, 6, 7), stDefaultCPUSet: cpuset.NewCPUSet(2, 3, 4, 5, 6, 7),
pod: makePod("fakePod", "fakeContainer2", "1000m", "1000m"), pod: makePod("fakePod", "fakeContainer2", "1000m", "1000m"),
@@ -918,6 +928,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { @@ -918,6 +932,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
topo: topoSingleSocketHT, topo: topoSingleSocketHT,
numReservedCPUs: 2, numReservedCPUs: 2,
reserved: cpuset.NewCPUSet(0, 1), reserved: cpuset.NewCPUSet(0, 1),
@ -518,7 +632,7 @@ index d4b4b790210..ecd3e9598d0 100644
stAssignments: state.ContainerCPUAssignments{ stAssignments: state.ContainerCPUAssignments{
"fakePod": map[string]cpuset.CPUSet{ "fakePod": map[string]cpuset.CPUSet{
"fakeContainer100": cpuset.NewCPUSet(2, 3, 6, 7), "fakeContainer100": cpuset.NewCPUSet(2, 3, 6, 7),
@@ -934,6 +945,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { @@ -934,6 +949,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
topo: topoSingleSocketHT, topo: topoSingleSocketHT,
numReservedCPUs: 2, numReservedCPUs: 2,
reserved: cpuset.NewCPUSet(0, 1), reserved: cpuset.NewCPUSet(0, 1),
@ -526,7 +640,7 @@ index d4b4b790210..ecd3e9598d0 100644
stAssignments: state.ContainerCPUAssignments{ stAssignments: state.ContainerCPUAssignments{
"fakePod": map[string]cpuset.CPUSet{ "fakePod": map[string]cpuset.CPUSet{
"fakeContainer100": cpuset.NewCPUSet(2, 3, 6, 7), "fakeContainer100": cpuset.NewCPUSet(2, 3, 6, 7),
@@ -945,11 +957,29 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { @@ -945,11 +961,29 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
expCPUAlloc: true, expCPUAlloc: true,
expCSet: cpuset.NewCPUSet(0, 1), expCSet: cpuset.NewCPUSet(0, 1),
}, },
@ -557,6 +671,132 @@ index d4b4b790210..ecd3e9598d0 100644
st := &mockState{ st := &mockState{
assignments: testCase.stAssignments, assignments: testCase.stAssignments,
@@ -1075,3 +1109,125 @@ func TestStaticPolicyOptions(t *testing.T) {
})
}
}
+
+func makePodWithLabels(podLabels map[string]string) *v1.Pod {
+ return &v1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-pod",
+ Namespace: "test-namespace",
+ Labels: podLabels,
+ },
+ }
+}
+
+func fakeBuildConfigFromFlags(masterUrl string, kubeconfigPath string) (*restclient.Config, error) {
+
+ return &restclient.Config{}, nil
+}
+
+func fakeBuildConfigFromFlagsError(masterUrl string, kubeconfigPath string) (*restclient.Config, error) {
+
+ errString := fmt.Sprintf("%s file not found", kubeconfigPath)
+ return nil, errors.New(errString)
+
+}
+
+func getFakeInfraPodNamespace(_ string) (*v1.Namespace, error) {
+
+ return &v1.Namespace{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-namespace",
+ Labels: map[string]string{
+ "app.starlingx.io/component": "platform",
+ },
+ }}, nil
+}
+
+func getFakeNonInfraPodNamespace(_ string) (*v1.Namespace, error) {
+
+ return &v1.Namespace{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-namespace",
+ Labels: map[string]string{
+ "fake": "label",
+ }}}, nil
+
+}
+
+type kubeInfraPodTestCase struct {
+ description string
+ pod *v1.Pod
+ namespaceFunc getPodNamespace
+ expectedValue bool
+}
+
+func TestKubeInfraPod(t *testing.T) {
+ testCases := []kubeInfraPodTestCase{
+ {
+ description: "Pod with platform label and namespace without platform label",
+ pod: makePodWithLabels(map[string]string{
+ "app.starlingx.io/component": "platform",
+ }),
+ namespaceFunc: getFakeNonInfraPodNamespace,
+ expectedValue: true,
+
+ },
+ {
+ description: "Pod without platform label and namespace with platform label",
+ pod: makePodWithLabels(map[string]string{
+ "test": "label",
+ }),
+ namespaceFunc: getFakeInfraPodNamespace,
+ expectedValue: true,
+ },
+ {
+ description: "Pod without platform label and namespace without platform label",
+ pod: makePodWithLabels(map[string]string{
+ "test": "namespace",
+ }),
+ namespaceFunc: getFakeNonInfraPodNamespace,
+ expectedValue: false,
+ },
+
+ }
+
+ for _, testCase := range testCases {
+ t.Run(testCase.description, func(t *testing.T) {
+
+ varGetNamespaceObject = testCase.namespaceFunc
+ varBuildConfigFromFlags = fakeBuildConfigFromFlags
+ gotValue := isKubeInfra(testCase.pod)
+
+ if gotValue != testCase.expectedValue {
+ t.Errorf("StaticPolicy isKubeInfraPod() error %v. expected value %v actual value %v",
+ testCase.description, testCase.expectedValue, gotValue)
+ } else {
+ fmt.Printf("StaticPolicy isKubeInfraPod() test successful. : %v ", testCase.description)
+ }
+
+ })
+ }
+
+ test := kubeInfraPodTestCase{
+ description: "Failure reading kubeconfig file",
+ pod: makePodWithLabels(map[string]string{
+ "test": "namespace",
+ }),
+ namespaceFunc: getFakeNonInfraPodNamespace,
+ expectedValue: false,
+ }
+
+ varGetNamespaceObject = getPodNamespaceObject
+ varBuildConfigFromFlags = fakeBuildConfigFromFlagsError
+
+ gotValue := isKubeInfra(test.pod)
+
+ if gotValue != test.expectedValue {
+ t.Errorf("StaticPolicy isKubeInfraPod() error %v. expected value %v actual value %v",
+ test.description, test.expectedValue, gotValue)
+ } else {
+ fmt.Printf("StaticPolicy isKubeInfraPod() test successful. : %v ", test.description)
+ }
+
+}
+
-- --
2.25.1 2.25.1

View File

@ -1,4 +1,4 @@
From e8608eb7c300b6e0503885a4848fafc75f20d909 Mon Sep 17 00:00:00 2001 From 27f0c5ba4cdd5f8f850fa4a5b110a39eaba7cd65 Mon Sep 17 00:00:00 2001
From: Ramesh Kumar Sivanandam <rameshkumar.sivanandam@windriver.com> From: Ramesh Kumar Sivanandam <rameshkumar.sivanandam@windriver.com>
Date: Mon, 7 Nov 2022 13:33:03 -0500 Date: Mon, 7 Nov 2022 13:33:03 -0500
Subject: [PATCH] kubelet cpumanager introduce concept of isolated CPUs Subject: [PATCH] kubelet cpumanager introduce concept of isolated CPUs
@ -45,13 +45,14 @@ Co-authored-by: Jim Gauld <james.gauld@windriver.com>
Co-authored-by: Chris Friesen <chris.friesen@windriver.com> Co-authored-by: Chris Friesen <chris.friesen@windriver.com>
Signed-off-by: Gleb Aronsky <gleb.aronsky@windriver.com> Signed-off-by: Gleb Aronsky <gleb.aronsky@windriver.com>
Signed-off-by: Ramesh Kumar Sivanandam <rameshkumar.sivanandam@windriver.com> Signed-off-by: Ramesh Kumar Sivanandam <rameshkumar.sivanandam@windriver.com>
Signed-off-by: Kaustubh Dhokte <kaustubh.dhokte@windriver.com>
--- ---
pkg/kubelet/cm/container_manager_linux.go | 1 + pkg/kubelet/cm/container_manager_linux.go | 1 +
pkg/kubelet/cm/cpumanager/cpu_manager.go | 35 +++++++- pkg/kubelet/cm/cpumanager/cpu_manager.go | 35 +++-
pkg/kubelet/cm/cpumanager/cpu_manager_test.go | 20 ++++- pkg/kubelet/cm/cpumanager/cpu_manager_test.go | 20 +-
pkg/kubelet/cm/cpumanager/policy_static.go | 83 +++++++++++++++++-- pkg/kubelet/cm/cpumanager/policy_static.go | 158 ++++++++++++++--
.../cm/cpumanager/policy_static_test.go | 53 +++++++++--- .../cm/cpumanager/policy_static_test.go | 177 +++++++++++++++++-
5 files changed, 172 insertions(+), 20 deletions(-) 5 files changed, 362 insertions(+), 29 deletions(-)
diff --git a/pkg/kubelet/cm/container_manager_linux.go b/pkg/kubelet/cm/container_manager_linux.go diff --git a/pkg/kubelet/cm/container_manager_linux.go b/pkg/kubelet/cm/container_manager_linux.go
index 44c8cda6c40..a3f92b23c69 100644 index 44c8cda6c40..a3f92b23c69 100644
@ -242,18 +243,24 @@ index d553b182e0b..57f3f9a1c97 100644
t.Errorf("Expected error, but NewManager succeeded") t.Errorf("Expected error, but NewManager succeeded")
} }
diff --git a/pkg/kubelet/cm/cpumanager/policy_static.go b/pkg/kubelet/cm/cpumanager/policy_static.go diff --git a/pkg/kubelet/cm/cpumanager/policy_static.go b/pkg/kubelet/cm/cpumanager/policy_static.go
index 341e9f3dffe..802e289bfaf 100644 index 4fc96303622..4631841fe01 100644
--- a/pkg/kubelet/cm/cpumanager/policy_static.go --- a/pkg/kubelet/cm/cpumanager/policy_static.go
+++ b/pkg/kubelet/cm/cpumanager/policy_static.go +++ b/pkg/kubelet/cm/cpumanager/policy_static.go
@@ -18,6 +18,7 @@ package cpumanager @@ -17,14 +17,21 @@ limitations under the License.
package cpumanager
import ( import (
+ "context"
"fmt" "fmt"
+ "strconv" + "strconv"
+ k8sclient "k8s.io/client-go/kubernetes"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ restclient "k8s.io/client-go/rest"
v1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1"
+ "k8s.io/client-go/tools/clientcmd"
"k8s.io/klog/v2" "k8s.io/klog/v2"
@@ -25,6 +26,7 @@ import ( v1qos "k8s.io/kubernetes/pkg/apis/core/v1/helper/qos"
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state"
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
"k8s.io/kubernetes/pkg/kubelet/cm/cpuset" "k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
@ -261,7 +268,33 @@ index 341e9f3dffe..802e289bfaf 100644
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager" "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager"
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask" "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask"
) )
@@ -101,6 +103,10 @@ type staticPolicy struct { @@ -39,6 +46,13 @@ const (
ErrorSMTAlignment = "SMTAlignmentError"
)
+type getPodNamespace func(string) (*v1.Namespace, error)
+type buildFromConfigFlag func(masterUrl string, kubeconfigPath string) (*restclient.Config, error)
+
+var varGetNamespaceObject getPodNamespace
+var varBuildConfigFromFlags buildFromConfigFlag
+
+
// SMTAlignmentError represents an error due to SMT alignment
type SMTAlignmentError struct {
RequestedCPUs int
@@ -53,11 +67,6 @@ func (e SMTAlignmentError) Type() string {
return ErrorSMTAlignment
}
-// Define namespaces used by platform infrastructure pods
-var infraNamespaces = [...]string{
- "kube-system", "armada", "cert-manager", "platform-deployment-manager", "portieris", "vault", "notification", "flux-helm", "metrics-server", "node-feature-discovery", "intel-power", "power-metrics", "sriov-fec-system",
-}
-
// staticPolicy is a CPU manager policy that does not change CPU
// assignments for exclusively pinned guaranteed containers after the main
// container process starts.
@@ -101,6 +110,10 @@ type staticPolicy struct {
topology *topology.CPUTopology topology *topology.CPUTopology
// set of CPUs that is not available for exclusive assignment // set of CPUs that is not available for exclusive assignment
reserved cpuset.CPUSet reserved cpuset.CPUSet
@ -272,7 +305,7 @@ index 341e9f3dffe..802e289bfaf 100644
// If true, default CPUSet should exclude reserved CPUs // If true, default CPUSet should exclude reserved CPUs
excludeReserved bool excludeReserved bool
// topology manager reference to get container Topology affinity // topology manager reference to get container Topology affinity
@@ -117,7 +123,8 @@ var _ Policy = &staticPolicy{} @@ -117,7 +130,8 @@ var _ Policy = &staticPolicy{}
// NewStaticPolicy returns a CPU manager policy that does not change CPU // NewStaticPolicy returns a CPU manager policy that does not change CPU
// assignments for exclusively pinned guaranteed containers after the main // assignments for exclusively pinned guaranteed containers after the main
// container process starts. // container process starts.
@ -282,7 +315,7 @@ index 341e9f3dffe..802e289bfaf 100644
opts, err := NewStaticPolicyOptions(cpuPolicyOptions) opts, err := NewStaticPolicyOptions(cpuPolicyOptions)
if err != nil { if err != nil {
return nil, err return nil, err
@@ -132,6 +139,8 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv @@ -132,6 +146,8 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv
policy := &staticPolicy{ policy := &staticPolicy{
topology: topology, topology: topology,
affinity: affinity, affinity: affinity,
@ -291,7 +324,7 @@ index 341e9f3dffe..802e289bfaf 100644
excludeReserved: excludeReserved, excludeReserved: excludeReserved,
cpusToReuse: make(map[string]cpuset.CPUSet), cpusToReuse: make(map[string]cpuset.CPUSet),
options: opts, options: opts,
@@ -158,6 +167,12 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv @@ -158,6 +174,12 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv
klog.InfoS("Reserved CPUs not available for exclusive assignment", "reservedSize", reserved.Size(), "reserved", reserved) klog.InfoS("Reserved CPUs not available for exclusive assignment", "reservedSize", reserved.Size(), "reserved", reserved)
policy.reserved = reserved policy.reserved = reserved
@ -304,7 +337,7 @@ index 341e9f3dffe..802e289bfaf 100644
return policy, nil return policy, nil
} }
@@ -191,8 +206,9 @@ func (p *staticPolicy) validateState(s state.State) error { @@ -191,8 +213,9 @@ func (p *staticPolicy) validateState(s state.State) error {
} else { } else {
s.SetDefaultCPUSet(allCPUs) s.SetDefaultCPUSet(allCPUs)
} }
@ -316,7 +349,17 @@ index 341e9f3dffe..802e289bfaf 100644
return nil return nil
} }
@@ -282,10 +298,11 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai @@ -273,6 +296,9 @@ func (p *staticPolicy) updateCPUsToReuse(pod *v1.Pod, container *v1.Container, c
}
func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Container) error {
+
+ varGetNamespaceObject = getPodNamespaceObject
+ varBuildConfigFromFlags = clientcmd.BuildConfigFromFlags
// Process infra pods before guaranteed pods
if isKubeInfra(pod) {
// Container belongs in reserved pool.
@@ -282,10 +308,11 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai
return nil return nil
} }
@ -330,7 +373,7 @@ index 341e9f3dffe..802e289bfaf 100644
} }
s.SetCPUSet(string(pod.UID), container.Name, cpuset) s.SetCPUSet(string(pod.UID), container.Name, cpuset)
klog.Infof("[cpumanager] static policy: reserved: AddContainer (namespace: %s, pod UID: %s, pod: %s, container: %s); cpuset=%v", pod.Namespace, string(pod.UID), pod.Name, container.Name, cpuset) klog.Infof("[cpumanager] static policy: reserved: AddContainer (namespace: %s, pod UID: %s, pod: %s, container: %s); cpuset=%v", pod.Namespace, string(pod.UID), pod.Name, container.Name, cpuset)
@@ -329,8 +346,34 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai @@ -329,8 +356,34 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai
} }
s.SetCPUSet(string(pod.UID), container.Name, cpuset) s.SetCPUSet(string(pod.UID), container.Name, cpuset)
p.updateCPUsToReuse(pod, container, cpuset) p.updateCPUsToReuse(pod, container, cpuset)
@ -365,10 +408,71 @@ index 341e9f3dffe..802e289bfaf 100644
// container belongs in the shared pool (nothing to do; use default cpuset) // container belongs in the shared pool (nothing to do; use default cpuset)
return nil return nil
} }
@@ -640,6 +683,36 @@ func isKubeInfra(pod *v1.Pod) bool { @@ -630,14 +683,91 @@ func (p *staticPolicy) generateCPUTopologyHints(availableCPUs cpuset.CPUSet, reu
return false return hints
} }
+func getPodNamespaceObject(podNamespaceName string) (*v1.Namespace, error) {
+
+ cfg, err := varBuildConfigFromFlags("", "/etc/kubernetes/kubelet.conf")
+ if err != nil {
+ klog.Error("Failed to build client config from /etc/kubernetes/kubelet.conf: ", err.Error())
+ return nil, err
+ }
+
+ clientset, err := k8sclient.NewForConfig(cfg)
+ if err != nil {
+ klog.Error("Failed to get clientset for KUBECONFIG /etc/kubernetes/kubelet.conf: ", err.Error())
+ return nil, err
+ }
+
+ namespaceObj, err := clientset.CoreV1().Namespaces().Get(context.TODO(), podNamespaceName, metav1.GetOptions{})
+ if err != nil {
+ klog.Error("Error getting namespace object:", err.Error())
+ return nil, err
+ }
+
+ return namespaceObj, nil
+
+}
+
// check if a given pod is in a platform infrastructure namespace
func isKubeInfra(pod *v1.Pod) bool {
- for _, namespace := range infraNamespaces {
- if namespace == pod.Namespace {
- return true
- }
+
+ podName := pod.GetName()
+ podNamespaceName := pod.GetNamespace()
+
+ klog.InfoS("Checking pod ", podName , " for label 'app.starlingx.io/component=platform'.")
+ podLabels := pod.GetLabels()
+ val, ok := podLabels["app.starlingx.io/component"]
+ if (ok && val == "platform") {
+ klog.InfoS("Pod ", podName, " has 'app.starlingx.io/component=platform' label. Assigning platform CPUs.")
+ return true
}
+
+ klog.InfoS("Pod ", pod.GetName(), " does not have 'app.starlingx.io/component=platform' label. Checking its namespace information...")
+
+ namespaceObj, err := varGetNamespaceObject(podNamespaceName)
+ if err != nil {
+ return false
+ }
+
+ namespaceLabels := namespaceObj.GetLabels()
+ val, ok = namespaceLabels["app.starlingx.io/component"]
+ if ok && val == "platform" {
+ klog.InfoS("For pod: ", podName, ", its Namespace ", podNamespaceName, " has 'app.starlingx.io/component=platform' label. Assigning platform CPUs.")
+ return true
+ }
+
+ klog.InfoS("Neither pod ", podName, " nor its namespace ", podNamespaceName, " has 'app.starlingx.io/component=platform' label. Not assigning platform CPUs.")
return false
+
+}
+
+// get the isolated CPUs (if any) from the devices associated with a specific container +// get the isolated CPUs (if any) from the devices associated with a specific container
+func (p *staticPolicy) podIsolCPUs(pod *v1.Pod, container *v1.Container) cpuset.CPUSet { +func (p *staticPolicy) podIsolCPUs(pod *v1.Pod, container *v1.Container) cpuset.CPUSet {
+ // NOTE: This is required for TestStaticPolicyAdd() since makePod() does + // NOTE: This is required for TestStaticPolicyAdd() since makePod() does
@ -397,16 +501,28 @@ index 341e9f3dffe..802e289bfaf 100644
+ } + }
+ } + }
+ return cpuSet + return cpuSet
+} }
+
// isHintSocketAligned function return true if numa nodes in hint are socket aligned. // isHintSocketAligned function return true if numa nodes in hint are socket aligned.
func (p *staticPolicy) isHintSocketAligned(hint topologymanager.TopologyHint, minAffinitySize int) bool {
numaNodesBitMask := hint.NUMANodeAffinity.GetBits()
diff --git a/pkg/kubelet/cm/cpumanager/policy_static_test.go b/pkg/kubelet/cm/cpumanager/policy_static_test.go diff --git a/pkg/kubelet/cm/cpumanager/policy_static_test.go b/pkg/kubelet/cm/cpumanager/policy_static_test.go
index 414e5ce144c..1c43df3b85f 100644 index 414e5ce144c..b6aad48576f 100644
--- a/pkg/kubelet/cm/cpumanager/policy_static_test.go --- a/pkg/kubelet/cm/cpumanager/policy_static_test.go
+++ b/pkg/kubelet/cm/cpumanager/policy_static_test.go +++ b/pkg/kubelet/cm/cpumanager/policy_static_test.go
@@ -28,6 +28,7 @@ import ( @@ -17,10 +17,13 @@ limitations under the License.
package cpumanager
import (
+ "errors"
"fmt"
"reflect"
"testing"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ restclient "k8s.io/client-go/rest"
v1 "k8s.io/api/core/v1"
utilfeature "k8s.io/apiserver/pkg/util/feature"
featuregatetesting "k8s.io/component-base/featuregate/testing"
@@ -28,6 +31,7 @@ import (
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state"
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
"k8s.io/kubernetes/pkg/kubelet/cm/cpuset" "k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
@ -414,7 +530,7 @@ index 414e5ce144c..1c43df3b85f 100644
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager" "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager"
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask" "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask"
) )
@@ -69,8 +70,9 @@ func (spt staticPolicyTest) PseudoClone() staticPolicyTest { @@ -69,8 +73,9 @@ func (spt staticPolicyTest) PseudoClone() staticPolicyTest {
} }
func TestStaticPolicyName(t *testing.T) { func TestStaticPolicyName(t *testing.T) {
@ -425,7 +541,7 @@ index 414e5ce144c..1c43df3b85f 100644
policyName := policy.Name() policyName := policy.Name()
if policyName != "static" { if policyName != "static" {
@@ -80,6 +82,7 @@ func TestStaticPolicyName(t *testing.T) { @@ -80,6 +85,7 @@ func TestStaticPolicyName(t *testing.T) {
} }
func TestStaticPolicyStart(t *testing.T) { func TestStaticPolicyStart(t *testing.T) {
@ -433,7 +549,7 @@ index 414e5ce144c..1c43df3b85f 100644
testCases := []staticPolicyTest{ testCases := []staticPolicyTest{
{ {
description: "non-corrupted state", description: "non-corrupted state",
@@ -155,7 +158,7 @@ func TestStaticPolicyStart(t *testing.T) { @@ -155,7 +161,7 @@ func TestStaticPolicyStart(t *testing.T) {
} }
for _, testCase := range testCases { for _, testCase := range testCases {
t.Run(testCase.description, func(t *testing.T) { t.Run(testCase.description, func(t *testing.T) {
@ -442,7 +558,7 @@ index 414e5ce144c..1c43df3b85f 100644
policy := p.(*staticPolicy) policy := p.(*staticPolicy)
st := &mockState{ st := &mockState{
@@ -203,7 +206,6 @@ func TestStaticPolicyAdd(t *testing.T) { @@ -203,7 +209,6 @@ func TestStaticPolicyAdd(t *testing.T) {
largeTopoCPUSet := largeTopoBuilder.Result() largeTopoCPUSet := largeTopoBuilder.Result()
largeTopoSock0CPUSet := largeTopoSock0Builder.Result() largeTopoSock0CPUSet := largeTopoSock0Builder.Result()
largeTopoSock1CPUSet := largeTopoSock1Builder.Result() largeTopoSock1CPUSet := largeTopoSock1Builder.Result()
@ -450,7 +566,7 @@ index 414e5ce144c..1c43df3b85f 100644
// these are the cases which must behave the same regardless the policy options. // these are the cases which must behave the same regardless the policy options.
// So we will permutate the options to ensure this holds true. // So we will permutate the options to ensure this holds true.
@@ -577,8 +579,10 @@ func runStaticPolicyTestCase(t *testing.T, testCase staticPolicyTest) { @@ -577,8 +582,10 @@ func runStaticPolicyTestCase(t *testing.T, testCase staticPolicyTest) {
if testCase.topologyHint != nil { if testCase.topologyHint != nil {
tm = topologymanager.NewFakeManagerWithHint(testCase.topologyHint) tm = topologymanager.NewFakeManagerWithHint(testCase.topologyHint)
} }
@ -462,7 +578,7 @@ index 414e5ce144c..1c43df3b85f 100644
st := &mockState{ st := &mockState{
assignments: testCase.stAssignments, assignments: testCase.stAssignments,
@@ -625,6 +629,8 @@ func runStaticPolicyTestCaseWithFeatureGate(t *testing.T, testCase staticPolicyT @@ -625,6 +632,8 @@ func runStaticPolicyTestCaseWithFeatureGate(t *testing.T, testCase staticPolicyT
} }
func TestStaticPolicyReuseCPUs(t *testing.T) { func TestStaticPolicyReuseCPUs(t *testing.T) {
@ -471,7 +587,7 @@ index 414e5ce144c..1c43df3b85f 100644
testCases := []struct { testCases := []struct {
staticPolicyTest staticPolicyTest
expCSetAfterAlloc cpuset.CPUSet expCSetAfterAlloc cpuset.CPUSet
@@ -649,7 +655,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) { @@ -649,7 +658,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) {
} }
for _, testCase := range testCases { for _, testCase := range testCases {
@ -480,7 +596,7 @@ index 414e5ce144c..1c43df3b85f 100644
st := &mockState{ st := &mockState{
assignments: testCase.stAssignments, assignments: testCase.stAssignments,
@@ -682,6 +688,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) { @@ -682,6 +691,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) {
func TestStaticPolicyRemove(t *testing.T) { func TestStaticPolicyRemove(t *testing.T) {
excludeReserved := false excludeReserved := false
@ -488,7 +604,7 @@ index 414e5ce144c..1c43df3b85f 100644
testCases := []staticPolicyTest{ testCases := []staticPolicyTest{
{ {
description: "SingleSocketHT, DeAllocOneContainer", description: "SingleSocketHT, DeAllocOneContainer",
@@ -740,7 +747,7 @@ func TestStaticPolicyRemove(t *testing.T) { @@ -740,7 +750,7 @@ func TestStaticPolicyRemove(t *testing.T) {
} }
for _, testCase := range testCases { for _, testCase := range testCases {
@ -497,7 +613,7 @@ index 414e5ce144c..1c43df3b85f 100644
st := &mockState{ st := &mockState{
assignments: testCase.stAssignments, assignments: testCase.stAssignments,
@@ -763,6 +770,7 @@ func TestStaticPolicyRemove(t *testing.T) { @@ -763,6 +773,7 @@ func TestStaticPolicyRemove(t *testing.T) {
func TestTopologyAwareAllocateCPUs(t *testing.T) { func TestTopologyAwareAllocateCPUs(t *testing.T) {
excludeReserved := false excludeReserved := false
@ -505,7 +621,7 @@ index 414e5ce144c..1c43df3b85f 100644
testCases := []struct { testCases := []struct {
description string description string
topo *topology.CPUTopology topo *topology.CPUTopology
@@ -831,7 +839,8 @@ func TestTopologyAwareAllocateCPUs(t *testing.T) { @@ -831,7 +842,8 @@ func TestTopologyAwareAllocateCPUs(t *testing.T) {
}, },
} }
for _, tc := range testCases { for _, tc := range testCases {
@ -515,7 +631,7 @@ index 414e5ce144c..1c43df3b85f 100644
policy := p.(*staticPolicy) policy := p.(*staticPolicy)
st := &mockState{ st := &mockState{
assignments: tc.stAssignments, assignments: tc.stAssignments,
@@ -864,6 +873,7 @@ type staticPolicyTestWithResvList struct { @@ -864,6 +876,7 @@ type staticPolicyTestWithResvList struct {
topo *topology.CPUTopology topo *topology.CPUTopology
numReservedCPUs int numReservedCPUs int
reserved cpuset.CPUSet reserved cpuset.CPUSet
@ -523,7 +639,7 @@ index 414e5ce144c..1c43df3b85f 100644
stAssignments state.ContainerCPUAssignments stAssignments state.ContainerCPUAssignments
stDefaultCPUSet cpuset.CPUSet stDefaultCPUSet cpuset.CPUSet
pod *v1.Pod pod *v1.Pod
@@ -874,6 +884,8 @@ type staticPolicyTestWithResvList struct { @@ -874,6 +887,8 @@ type staticPolicyTestWithResvList struct {
} }
func TestStaticPolicyStartWithResvList(t *testing.T) { func TestStaticPolicyStartWithResvList(t *testing.T) {
@ -532,7 +648,7 @@ index 414e5ce144c..1c43df3b85f 100644
testCases := []staticPolicyTestWithResvList{ testCases := []staticPolicyTestWithResvList{
{ {
description: "empty cpuset", description: "empty cpuset",
@@ -903,11 +915,10 @@ func TestStaticPolicyStartWithResvList(t *testing.T) { @@ -903,11 +918,10 @@ func TestStaticPolicyStartWithResvList(t *testing.T) {
expNewErr: fmt.Errorf("[cpumanager] unable to reserve the required amount of CPUs (size of 0-1 did not equal 1)"), expNewErr: fmt.Errorf("[cpumanager] unable to reserve the required amount of CPUs (size of 0-1 did not equal 1)"),
}, },
} }
@ -545,7 +661,7 @@ index 414e5ce144c..1c43df3b85f 100644
if !reflect.DeepEqual(err, testCase.expNewErr) { if !reflect.DeepEqual(err, testCase.expNewErr) {
t.Errorf("StaticPolicy Start() error (%v). expected error: %v but got: %v", t.Errorf("StaticPolicy Start() error (%v). expected error: %v but got: %v",
testCase.description, testCase.expNewErr, err) testCase.description, testCase.expNewErr, err)
@@ -947,6 +958,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { @@ -947,6 +961,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
topo: topoSingleSocketHT, topo: topoSingleSocketHT,
numReservedCPUs: 1, numReservedCPUs: 1,
reserved: cpuset.NewCPUSet(0), reserved: cpuset.NewCPUSet(0),
@ -553,7 +669,7 @@ index 414e5ce144c..1c43df3b85f 100644
stAssignments: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{},
stDefaultCPUSet: cpuset.NewCPUSet(1, 2, 3, 4, 5, 6, 7), stDefaultCPUSet: cpuset.NewCPUSet(1, 2, 3, 4, 5, 6, 7),
pod: makePod("fakePod", "fakeContainer2", "8000m", "8000m"), pod: makePod("fakePod", "fakeContainer2", "8000m", "8000m"),
@@ -959,6 +971,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { @@ -959,6 +974,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
topo: topoSingleSocketHT, topo: topoSingleSocketHT,
numReservedCPUs: 2, numReservedCPUs: 2,
reserved: cpuset.NewCPUSet(0, 1), reserved: cpuset.NewCPUSet(0, 1),
@ -561,7 +677,7 @@ index 414e5ce144c..1c43df3b85f 100644
stAssignments: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{},
stDefaultCPUSet: cpuset.NewCPUSet(2, 3, 4, 5, 6, 7), stDefaultCPUSet: cpuset.NewCPUSet(2, 3, 4, 5, 6, 7),
pod: makePod("fakePod", "fakeContainer2", "1000m", "1000m"), pod: makePod("fakePod", "fakeContainer2", "1000m", "1000m"),
@@ -971,6 +984,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { @@ -971,6 +987,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
topo: topoSingleSocketHT, topo: topoSingleSocketHT,
numReservedCPUs: 2, numReservedCPUs: 2,
reserved: cpuset.NewCPUSet(0, 1), reserved: cpuset.NewCPUSet(0, 1),
@ -569,7 +685,7 @@ index 414e5ce144c..1c43df3b85f 100644
stAssignments: state.ContainerCPUAssignments{ stAssignments: state.ContainerCPUAssignments{
"fakePod": map[string]cpuset.CPUSet{ "fakePod": map[string]cpuset.CPUSet{
"fakeContainer100": cpuset.NewCPUSet(2, 3, 6, 7), "fakeContainer100": cpuset.NewCPUSet(2, 3, 6, 7),
@@ -987,6 +1001,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { @@ -987,6 +1004,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
topo: topoSingleSocketHT, topo: topoSingleSocketHT,
numReservedCPUs: 2, numReservedCPUs: 2,
reserved: cpuset.NewCPUSet(0, 1), reserved: cpuset.NewCPUSet(0, 1),
@ -577,7 +693,7 @@ index 414e5ce144c..1c43df3b85f 100644
stAssignments: state.ContainerCPUAssignments{ stAssignments: state.ContainerCPUAssignments{
"fakePod": map[string]cpuset.CPUSet{ "fakePod": map[string]cpuset.CPUSet{
"fakeContainer100": cpuset.NewCPUSet(2, 3, 6, 7), "fakeContainer100": cpuset.NewCPUSet(2, 3, 6, 7),
@@ -998,11 +1013,29 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { @@ -998,11 +1016,29 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
expCPUAlloc: true, expCPUAlloc: true,
expCSet: cpuset.NewCPUSet(0, 1), expCSet: cpuset.NewCPUSet(0, 1),
}, },
@ -608,6 +724,131 @@ index 414e5ce144c..1c43df3b85f 100644
st := &mockState{ st := &mockState{
assignments: testCase.stAssignments, assignments: testCase.stAssignments,
@@ -1128,3 +1164,124 @@ func TestStaticPolicyOptions(t *testing.T) {
})
}
}
+
+func makePodWithLabels(podLabels map[string]string) *v1.Pod {
+ return &v1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-pod",
+ Namespace: "test-namespace",
+ Labels: podLabels,
+ },
+ }
+}
+
+func fakeBuildConfigFromFlags(masterUrl string, kubeconfigPath string) (*restclient.Config, error) {
+
+ return &restclient.Config{}, nil
+}
+
+func fakeBuildConfigFromFlagsError(masterUrl string, kubeconfigPath string) (*restclient.Config, error) {
+
+ errString := fmt.Sprintf("%s file not found", kubeconfigPath)
+ return nil, errors.New(errString)
+
+}
+
+func getFakeInfraPodNamespace(_ string) (*v1.Namespace, error) {
+
+ return &v1.Namespace{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-namespace",
+ Labels: map[string]string{
+ "app.starlingx.io/component": "platform",
+ },
+ }}, nil
+}
+
+func getFakeNonInfraPodNamespace(_ string) (*v1.Namespace, error) {
+
+ return &v1.Namespace{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-namespace",
+ Labels: map[string]string{
+ "fake": "label",
+ }}}, nil
+
+}
+
+type kubeInfraPodTestCase struct {
+ description string
+ pod *v1.Pod
+ namespaceFunc getPodNamespace
+ expectedValue bool
+}
+
+func TestKubeInfraPod(t *testing.T) {
+ testCases := []kubeInfraPodTestCase{
+ {
+ description: "Pod with platform label and namespace without platform label",
+ pod: makePodWithLabels(map[string]string{
+ "app.starlingx.io/component": "platform",
+ }),
+ namespaceFunc: getFakeNonInfraPodNamespace,
+ expectedValue: true,
+
+ },
+ {
+ description: "Pod without platform label and namespace with platform label",
+ pod: makePodWithLabels(map[string]string{
+ "test": "label",
+ }),
+ namespaceFunc: getFakeInfraPodNamespace,
+ expectedValue: true,
+ },
+ {
+ description: "Pod without platform label and namespace without platform label",
+ pod: makePodWithLabels(map[string]string{
+ "test": "namespace",
+ }),
+ namespaceFunc: getFakeNonInfraPodNamespace,
+ expectedValue: false,
+ },
+
+ }
+
+ for _, testCase := range testCases {
+ t.Run(testCase.description, func(t *testing.T) {
+
+ varGetNamespaceObject = testCase.namespaceFunc
+ varBuildConfigFromFlags = fakeBuildConfigFromFlags
+ gotValue := isKubeInfra(testCase.pod)
+
+ if gotValue != testCase.expectedValue {
+ t.Errorf("StaticPolicy isKubeInfraPod() error %v. expected value %v actual value %v",
+ testCase.description, testCase.expectedValue, gotValue)
+ } else {
+ fmt.Printf("StaticPolicy isKubeInfraPod() test successful. : %v ", testCase.description)
+ }
+
+ })
+ }
+
+ test := kubeInfraPodTestCase{
+ description: "Failure reading kubeconfig file",
+ pod: makePodWithLabels(map[string]string{
+ "test": "namespace",
+ }),
+ namespaceFunc: getFakeNonInfraPodNamespace,
+ expectedValue: false,
+ }
+
+ varGetNamespaceObject = getPodNamespaceObject
+ varBuildConfigFromFlags = fakeBuildConfigFromFlagsError
+
+ gotValue := isKubeInfra(test.pod)
+
+ if gotValue != test.expectedValue {
+ t.Errorf("StaticPolicy isKubeInfraPod() error %v. expected value %v actual value %v",
+ test.description, test.expectedValue, gotValue)
+ } else {
+ fmt.Printf("StaticPolicy isKubeInfraPod() test successful. : %v ", test.description)
+ }
+
+}
-- --
2.25.1 2.25.1

View File

@ -1,7 +1,7 @@
From ed1f8c6a04e7fed096eaae5081c2b5e0c3bc6fed Mon Sep 17 00:00:00 2001 From 36170864cc9ebb2183e6301cb745e49238d21397 Mon Sep 17 00:00:00 2001
From: Ramesh Kumar Sivanandam <rameshkumar.sivanandam@windriver.com> From: Ramesh Kumar Sivanandam <rameshkumar.sivanandam@windriver.com>
Date: Mon, 7 Nov 2022 13:33:03 -0500 Date: Mon, 7 Nov 2022 13:33:03 -0500
Subject: [PATCH 08/10] kubelet cpumanager introduce concept of isolated CPUs Subject: [PATCH] kubelet cpumanager introduce concept of isolated CPUs
This introduces the concept of "isolated CPUs", which are CPUs that This introduces the concept of "isolated CPUs", which are CPUs that
have been isolated at the kernel level via the "isolcpus" kernel boot have been isolated at the kernel level via the "isolcpus" kernel boot
@ -46,14 +46,15 @@ Co-authored-by: Chris Friesen <chris.friesen@windriver.com>
Signed-off-by: Gleb Aronsky <gleb.aronsky@windriver.com> Signed-off-by: Gleb Aronsky <gleb.aronsky@windriver.com>
Signed-off-by: Ramesh Kumar Sivanandam <rameshkumar.sivanandam@windriver.com> Signed-off-by: Ramesh Kumar Sivanandam <rameshkumar.sivanandam@windriver.com>
Signed-off-by: Sachin Gopala Krishna <saching.krishna@windriver.com> Signed-off-by: Sachin Gopala Krishna <saching.krishna@windriver.com>
Signed-off-by: Kaustubh Dhokte <kaustubh.dhokte@windriver.com>
--- ---
pkg/kubelet/cm/container_manager_linux.go | 1 + pkg/kubelet/cm/container_manager_linux.go | 1 +
pkg/kubelet/cm/cpumanager/cpu_manager.go | 35 ++++++- pkg/kubelet/cm/cpumanager/cpu_manager.go | 35 +++-
pkg/kubelet/cm/cpumanager/cpu_manager_test.go | 23 ++++- pkg/kubelet/cm/cpumanager/cpu_manager_test.go | 23 ++-
pkg/kubelet/cm/cpumanager/policy_static.go | 83 ++++++++++++++-- pkg/kubelet/cm/cpumanager/policy_static.go | 157 +++++++++++++--
.../cm/cpumanager/policy_static_test.go | 53 ++++++++-- .../cm/cpumanager/policy_static_test.go | 178 +++++++++++++++++-
pkg/kubelet/cm/devicemanager/manager_stub.go | 99 +++++++++++++++++++ pkg/kubelet/cm/devicemanager/manager_stub.go | 99 ++++++++++
6 files changed, 273 insertions(+), 21 deletions(-) 6 files changed, 463 insertions(+), 30 deletions(-)
create mode 100644 pkg/kubelet/cm/devicemanager/manager_stub.go create mode 100644 pkg/kubelet/cm/devicemanager/manager_stub.go
diff --git a/pkg/kubelet/cm/container_manager_linux.go b/pkg/kubelet/cm/container_manager_linux.go diff --git a/pkg/kubelet/cm/container_manager_linux.go b/pkg/kubelet/cm/container_manager_linux.go
@ -264,18 +265,25 @@ index e7c74453472..78b4ada1a73 100644
testCases := []struct { testCases := []struct {
diff --git a/pkg/kubelet/cm/cpumanager/policy_static.go b/pkg/kubelet/cm/cpumanager/policy_static.go diff --git a/pkg/kubelet/cm/cpumanager/policy_static.go b/pkg/kubelet/cm/cpumanager/policy_static.go
index 180d018565c..8d18ce65309 100644 index 26eb400cee6..13bcad5a531 100644
--- a/pkg/kubelet/cm/cpumanager/policy_static.go --- a/pkg/kubelet/cm/cpumanager/policy_static.go
+++ b/pkg/kubelet/cm/cpumanager/policy_static.go +++ b/pkg/kubelet/cm/cpumanager/policy_static.go
@@ -18,6 +18,7 @@ package cpumanager @@ -17,15 +17,22 @@ limitations under the License.
package cpumanager
import ( import (
+ "context"
"fmt" "fmt"
+ "strconv" + "strconv"
+ k8sclient "k8s.io/client-go/kubernetes"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ restclient "k8s.io/client-go/rest"
v1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1"
+ "k8s.io/client-go/tools/clientcmd"
"k8s.io/klog/v2" "k8s.io/klog/v2"
@@ -26,6 +27,7 @@ import (
v1qos "k8s.io/kubernetes/pkg/apis/core/v1/helper/qos"
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state"
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
"k8s.io/kubernetes/pkg/kubelet/cm/cpuset" "k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
@ -283,7 +291,32 @@ index 180d018565c..8d18ce65309 100644
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager" "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager"
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask" "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask"
"k8s.io/kubernetes/pkg/kubelet/metrics" "k8s.io/kubernetes/pkg/kubelet/metrics"
@@ -104,6 +106,10 @@ type staticPolicy struct { @@ -41,6 +48,12 @@ const (
ErrorSMTAlignment = "SMTAlignmentError"
)
+type getPodNamespace func(string) (*v1.Namespace, error)
+type buildFromConfigFlag func(masterUrl string, kubeconfigPath string) (*restclient.Config, error)
+
+var varGetNamespaceObject getPodNamespace
+var varBuildConfigFromFlags buildFromConfigFlag
+
// SMTAlignmentError represents an error due to SMT alignment
type SMTAlignmentError struct {
RequestedCPUs int
@@ -56,11 +69,6 @@ func (e SMTAlignmentError) Type() string {
return ErrorSMTAlignment
}
-// Define namespaces used by platform infrastructure pods
-var infraNamespaces = [...]string{
- "kube-system", "armada", "cert-manager", "platform-deployment-manager", "portieris", "vault", "notification", "flux-helm", "metrics-server", "node-feature-discovery", "intel-power", "power-metrics", "sriov-fec-system",
-}
-
// staticPolicy is a CPU manager policy that does not change CPU
// assignments for exclusively pinned guaranteed containers after the main
// container process starts.
@@ -104,6 +112,10 @@ type staticPolicy struct {
topology *topology.CPUTopology topology *topology.CPUTopology
// set of CPUs that is not available for exclusive assignment // set of CPUs that is not available for exclusive assignment
reserved cpuset.CPUSet reserved cpuset.CPUSet
@ -294,7 +327,7 @@ index 180d018565c..8d18ce65309 100644
// If true, default CPUSet should exclude reserved CPUs // If true, default CPUSet should exclude reserved CPUs
excludeReserved bool excludeReserved bool
// topology manager reference to get container Topology affinity // topology manager reference to get container Topology affinity
@@ -120,7 +126,8 @@ var _ Policy = &staticPolicy{} @@ -120,7 +132,8 @@ var _ Policy = &staticPolicy{}
// NewStaticPolicy returns a CPU manager policy that does not change CPU // NewStaticPolicy returns a CPU manager policy that does not change CPU
// assignments for exclusively pinned guaranteed containers after the main // assignments for exclusively pinned guaranteed containers after the main
// container process starts. // container process starts.
@ -304,7 +337,7 @@ index 180d018565c..8d18ce65309 100644
opts, err := NewStaticPolicyOptions(cpuPolicyOptions) opts, err := NewStaticPolicyOptions(cpuPolicyOptions)
if err != nil { if err != nil {
return nil, err return nil, err
@@ -135,6 +142,8 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv @@ -135,6 +148,8 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv
policy := &staticPolicy{ policy := &staticPolicy{
topology: topology, topology: topology,
affinity: affinity, affinity: affinity,
@ -313,7 +346,7 @@ index 180d018565c..8d18ce65309 100644
excludeReserved: excludeReserved, excludeReserved: excludeReserved,
cpusToReuse: make(map[string]cpuset.CPUSet), cpusToReuse: make(map[string]cpuset.CPUSet),
options: opts, options: opts,
@@ -161,6 +170,12 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv @@ -161,6 +176,12 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv
klog.InfoS("Reserved CPUs not available for exclusive assignment", "reservedSize", reserved.Size(), "reserved", reserved) klog.InfoS("Reserved CPUs not available for exclusive assignment", "reservedSize", reserved.Size(), "reserved", reserved)
policy.reserved = reserved policy.reserved = reserved
@ -326,7 +359,7 @@ index 180d018565c..8d18ce65309 100644
return policy, nil return policy, nil
} }
@@ -194,8 +209,9 @@ func (p *staticPolicy) validateState(s state.State) error { @@ -194,8 +215,9 @@ func (p *staticPolicy) validateState(s state.State) error {
} else { } else {
s.SetDefaultCPUSet(allCPUs) s.SetDefaultCPUSet(allCPUs)
} }
@ -338,7 +371,17 @@ index 180d018565c..8d18ce65309 100644
return nil return nil
} }
@@ -290,16 +306,39 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai @@ -281,6 +303,9 @@ func (p *staticPolicy) updateCPUsToReuse(pod *v1.Pod, container *v1.Container, c
}
func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Container) (rerr error) {
+
+ varGetNamespaceObject = getPodNamespaceObject
+ varBuildConfigFromFlags = clientcmd.BuildConfigFromFlags
// Process infra pods before guaranteed pods
if isKubeInfra(pod) {
// Container belongs in reserved pool.
@@ -290,16 +315,39 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai
return nil return nil
} }
@ -380,7 +423,7 @@ index 180d018565c..8d18ce65309 100644
numCPUs := p.guaranteedCPUs(pod, container) numCPUs := p.guaranteedCPUs(pod, container)
if numCPUs == 0 { if numCPUs == 0 {
// container belongs in the shared pool (nothing to do; use default cpuset) // container belongs in the shared pool (nothing to do; use default cpuset)
@@ -348,7 +387,9 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai @@ -348,7 +396,9 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai
} }
s.SetCPUSet(string(pod.UID), container.Name, cpuset) s.SetCPUSet(string(pod.UID), container.Name, cpuset)
p.updateCPUsToReuse(pod, container, cpuset) p.updateCPUsToReuse(pod, container, cpuset)
@ -391,10 +434,71 @@ index 180d018565c..8d18ce65309 100644
return nil return nil
} }
@@ -657,6 +698,36 @@ func isKubeInfra(pod *v1.Pod) bool { @@ -647,14 +697,91 @@ func (p *staticPolicy) generateCPUTopologyHints(availableCPUs cpuset.CPUSet, reu
return false return hints
} }
+func getPodNamespaceObject(podNamespaceName string) (*v1.Namespace, error) {
+
+ cfg, err := varBuildConfigFromFlags("", "/etc/kubernetes/kubelet.conf")
+ if err != nil {
+ klog.Error("Failed to build client config from /etc/kubernetes/kubelet.conf: ", err.Error())
+ return nil, err
+ }
+
+ clientset, err := k8sclient.NewForConfig(cfg)
+ if err != nil {
+ klog.Error("Failed to get clientset for KUBECONFIG /etc/kubernetes/kubelet.conf: ", err.Error())
+ return nil, err
+ }
+
+ namespaceObj, err := clientset.CoreV1().Namespaces().Get(context.TODO(), podNamespaceName, metav1.GetOptions{})
+ if err != nil {
+ klog.Error("Error getting namespace object:", err.Error())
+ return nil, err
+ }
+
+ return namespaceObj, nil
+
+}
+
// check if a given pod is in a platform infrastructure namespace
func isKubeInfra(pod *v1.Pod) bool {
- for _, namespace := range infraNamespaces {
- if namespace == pod.Namespace {
- return true
- }
+
+ podName := pod.GetName()
+ podNamespaceName := pod.GetNamespace()
+
+ klog.InfoS("Checking pod ", podName , " for label 'app.starlingx.io/component=platform'.")
+ podLabels := pod.GetLabels()
+ val, ok := podLabels["app.starlingx.io/component"]
+ if (ok && val == "platform") {
+ klog.InfoS("Pod ", podName, " has 'app.starlingx.io/component=platform' label. Assigning platform CPUs.")
+ return true
}
+
+ klog.InfoS("Pod ", pod.GetName(), " does not have 'app.starlingx.io/component=platform' label. Checking its namespace information...")
+
+ namespaceObj, err := varGetNamespaceObject(podNamespaceName)
+ if err != nil {
+ return false
+ }
+
+ namespaceLabels := namespaceObj.GetLabels()
+ val, ok = namespaceLabels["app.starlingx.io/component"]
+ if ok && val == "platform" {
+ klog.InfoS("For pod: ", podName, ", its Namespace ", podNamespaceName, " has 'app.starlingx.io/component=platform' label. Assigning platform CPUs.")
+ return true
+ }
+
+ klog.InfoS("Neither pod ", podName, " nor its namespace ", podNamespaceName, " has 'app.starlingx.io/component=platform' label. Not assigning platform CPUs.")
return false
+
+}
+
+// get the isolated CPUs (if any) from the devices associated with a specific container +// get the isolated CPUs (if any) from the devices associated with a specific container
+func (p *staticPolicy) podIsolCPUs(pod *v1.Pod, container *v1.Container) cpuset.CPUSet { +func (p *staticPolicy) podIsolCPUs(pod *v1.Pod, container *v1.Container) cpuset.CPUSet {
+ // NOTE: This is required for TestStaticPolicyAdd() since makePod() does + // NOTE: This is required for TestStaticPolicyAdd() since makePod() does
@ -423,16 +527,28 @@ index 180d018565c..8d18ce65309 100644
+ } + }
+ } + }
+ return cpuSet + return cpuSet
+} }
+
// isHintSocketAligned function return true if numa nodes in hint are socket aligned. // isHintSocketAligned function return true if numa nodes in hint are socket aligned.
func (p *staticPolicy) isHintSocketAligned(hint topologymanager.TopologyHint, minAffinitySize int) bool {
numaNodesBitMask := hint.NUMANodeAffinity.GetBits()
diff --git a/pkg/kubelet/cm/cpumanager/policy_static_test.go b/pkg/kubelet/cm/cpumanager/policy_static_test.go diff --git a/pkg/kubelet/cm/cpumanager/policy_static_test.go b/pkg/kubelet/cm/cpumanager/policy_static_test.go
index 414e5ce144c..1c43df3b85f 100644 index 414e5ce144c..f79c23accb4 100644
--- a/pkg/kubelet/cm/cpumanager/policy_static_test.go --- a/pkg/kubelet/cm/cpumanager/policy_static_test.go
+++ b/pkg/kubelet/cm/cpumanager/policy_static_test.go +++ b/pkg/kubelet/cm/cpumanager/policy_static_test.go
@@ -28,6 +28,7 @@ import ( @@ -17,10 +17,13 @@ limitations under the License.
package cpumanager
import (
+ "errors"
"fmt"
"reflect"
"testing"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ restclient "k8s.io/client-go/rest"
v1 "k8s.io/api/core/v1"
utilfeature "k8s.io/apiserver/pkg/util/feature"
featuregatetesting "k8s.io/component-base/featuregate/testing"
@@ -28,6 +31,7 @@ import (
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state"
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
"k8s.io/kubernetes/pkg/kubelet/cm/cpuset" "k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
@ -440,7 +556,7 @@ index 414e5ce144c..1c43df3b85f 100644
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager" "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager"
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask" "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask"
) )
@@ -69,8 +70,9 @@ func (spt staticPolicyTest) PseudoClone() staticPolicyTest { @@ -69,8 +73,9 @@ func (spt staticPolicyTest) PseudoClone() staticPolicyTest {
} }
func TestStaticPolicyName(t *testing.T) { func TestStaticPolicyName(t *testing.T) {
@ -451,7 +567,7 @@ index 414e5ce144c..1c43df3b85f 100644
policyName := policy.Name() policyName := policy.Name()
if policyName != "static" { if policyName != "static" {
@@ -80,6 +82,7 @@ func TestStaticPolicyName(t *testing.T) { @@ -80,6 +85,7 @@ func TestStaticPolicyName(t *testing.T) {
} }
func TestStaticPolicyStart(t *testing.T) { func TestStaticPolicyStart(t *testing.T) {
@ -459,7 +575,7 @@ index 414e5ce144c..1c43df3b85f 100644
testCases := []staticPolicyTest{ testCases := []staticPolicyTest{
{ {
description: "non-corrupted state", description: "non-corrupted state",
@@ -155,7 +158,7 @@ func TestStaticPolicyStart(t *testing.T) { @@ -155,7 +161,7 @@ func TestStaticPolicyStart(t *testing.T) {
} }
for _, testCase := range testCases { for _, testCase := range testCases {
t.Run(testCase.description, func(t *testing.T) { t.Run(testCase.description, func(t *testing.T) {
@ -468,7 +584,7 @@ index 414e5ce144c..1c43df3b85f 100644
policy := p.(*staticPolicy) policy := p.(*staticPolicy)
st := &mockState{ st := &mockState{
@@ -203,7 +206,6 @@ func TestStaticPolicyAdd(t *testing.T) { @@ -203,7 +209,6 @@ func TestStaticPolicyAdd(t *testing.T) {
largeTopoCPUSet := largeTopoBuilder.Result() largeTopoCPUSet := largeTopoBuilder.Result()
largeTopoSock0CPUSet := largeTopoSock0Builder.Result() largeTopoSock0CPUSet := largeTopoSock0Builder.Result()
largeTopoSock1CPUSet := largeTopoSock1Builder.Result() largeTopoSock1CPUSet := largeTopoSock1Builder.Result()
@ -476,7 +592,7 @@ index 414e5ce144c..1c43df3b85f 100644
// these are the cases which must behave the same regardless the policy options. // these are the cases which must behave the same regardless the policy options.
// So we will permutate the options to ensure this holds true. // So we will permutate the options to ensure this holds true.
@@ -577,8 +579,10 @@ func runStaticPolicyTestCase(t *testing.T, testCase staticPolicyTest) { @@ -577,8 +582,10 @@ func runStaticPolicyTestCase(t *testing.T, testCase staticPolicyTest) {
if testCase.topologyHint != nil { if testCase.topologyHint != nil {
tm = topologymanager.NewFakeManagerWithHint(testCase.topologyHint) tm = topologymanager.NewFakeManagerWithHint(testCase.topologyHint)
} }
@ -488,7 +604,7 @@ index 414e5ce144c..1c43df3b85f 100644
st := &mockState{ st := &mockState{
assignments: testCase.stAssignments, assignments: testCase.stAssignments,
@@ -625,6 +629,8 @@ func runStaticPolicyTestCaseWithFeatureGate(t *testing.T, testCase staticPolicyT @@ -625,6 +632,8 @@ func runStaticPolicyTestCaseWithFeatureGate(t *testing.T, testCase staticPolicyT
} }
func TestStaticPolicyReuseCPUs(t *testing.T) { func TestStaticPolicyReuseCPUs(t *testing.T) {
@ -497,7 +613,7 @@ index 414e5ce144c..1c43df3b85f 100644
testCases := []struct { testCases := []struct {
staticPolicyTest staticPolicyTest
expCSetAfterAlloc cpuset.CPUSet expCSetAfterAlloc cpuset.CPUSet
@@ -649,7 +655,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) { @@ -649,7 +658,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) {
} }
for _, testCase := range testCases { for _, testCase := range testCases {
@ -506,7 +622,7 @@ index 414e5ce144c..1c43df3b85f 100644
st := &mockState{ st := &mockState{
assignments: testCase.stAssignments, assignments: testCase.stAssignments,
@@ -682,6 +688,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) { @@ -682,6 +691,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) {
func TestStaticPolicyRemove(t *testing.T) { func TestStaticPolicyRemove(t *testing.T) {
excludeReserved := false excludeReserved := false
@ -514,7 +630,7 @@ index 414e5ce144c..1c43df3b85f 100644
testCases := []staticPolicyTest{ testCases := []staticPolicyTest{
{ {
description: "SingleSocketHT, DeAllocOneContainer", description: "SingleSocketHT, DeAllocOneContainer",
@@ -740,7 +747,7 @@ func TestStaticPolicyRemove(t *testing.T) { @@ -740,7 +750,7 @@ func TestStaticPolicyRemove(t *testing.T) {
} }
for _, testCase := range testCases { for _, testCase := range testCases {
@ -523,7 +639,7 @@ index 414e5ce144c..1c43df3b85f 100644
st := &mockState{ st := &mockState{
assignments: testCase.stAssignments, assignments: testCase.stAssignments,
@@ -763,6 +770,7 @@ func TestStaticPolicyRemove(t *testing.T) { @@ -763,6 +773,7 @@ func TestStaticPolicyRemove(t *testing.T) {
func TestTopologyAwareAllocateCPUs(t *testing.T) { func TestTopologyAwareAllocateCPUs(t *testing.T) {
excludeReserved := false excludeReserved := false
@ -531,7 +647,7 @@ index 414e5ce144c..1c43df3b85f 100644
testCases := []struct { testCases := []struct {
description string description string
topo *topology.CPUTopology topo *topology.CPUTopology
@@ -831,7 +839,8 @@ func TestTopologyAwareAllocateCPUs(t *testing.T) { @@ -831,7 +842,8 @@ func TestTopologyAwareAllocateCPUs(t *testing.T) {
}, },
} }
for _, tc := range testCases { for _, tc := range testCases {
@ -541,7 +657,7 @@ index 414e5ce144c..1c43df3b85f 100644
policy := p.(*staticPolicy) policy := p.(*staticPolicy)
st := &mockState{ st := &mockState{
assignments: tc.stAssignments, assignments: tc.stAssignments,
@@ -864,6 +873,7 @@ type staticPolicyTestWithResvList struct { @@ -864,6 +876,7 @@ type staticPolicyTestWithResvList struct {
topo *topology.CPUTopology topo *topology.CPUTopology
numReservedCPUs int numReservedCPUs int
reserved cpuset.CPUSet reserved cpuset.CPUSet
@ -549,7 +665,7 @@ index 414e5ce144c..1c43df3b85f 100644
stAssignments state.ContainerCPUAssignments stAssignments state.ContainerCPUAssignments
stDefaultCPUSet cpuset.CPUSet stDefaultCPUSet cpuset.CPUSet
pod *v1.Pod pod *v1.Pod
@@ -874,6 +884,8 @@ type staticPolicyTestWithResvList struct { @@ -874,6 +887,8 @@ type staticPolicyTestWithResvList struct {
} }
func TestStaticPolicyStartWithResvList(t *testing.T) { func TestStaticPolicyStartWithResvList(t *testing.T) {
@ -558,7 +674,7 @@ index 414e5ce144c..1c43df3b85f 100644
testCases := []staticPolicyTestWithResvList{ testCases := []staticPolicyTestWithResvList{
{ {
description: "empty cpuset", description: "empty cpuset",
@@ -903,11 +915,10 @@ func TestStaticPolicyStartWithResvList(t *testing.T) { @@ -903,11 +918,10 @@ func TestStaticPolicyStartWithResvList(t *testing.T) {
expNewErr: fmt.Errorf("[cpumanager] unable to reserve the required amount of CPUs (size of 0-1 did not equal 1)"), expNewErr: fmt.Errorf("[cpumanager] unable to reserve the required amount of CPUs (size of 0-1 did not equal 1)"),
}, },
} }
@ -571,7 +687,7 @@ index 414e5ce144c..1c43df3b85f 100644
if !reflect.DeepEqual(err, testCase.expNewErr) { if !reflect.DeepEqual(err, testCase.expNewErr) {
t.Errorf("StaticPolicy Start() error (%v). expected error: %v but got: %v", t.Errorf("StaticPolicy Start() error (%v). expected error: %v but got: %v",
testCase.description, testCase.expNewErr, err) testCase.description, testCase.expNewErr, err)
@@ -947,6 +958,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { @@ -947,6 +961,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
topo: topoSingleSocketHT, topo: topoSingleSocketHT,
numReservedCPUs: 1, numReservedCPUs: 1,
reserved: cpuset.NewCPUSet(0), reserved: cpuset.NewCPUSet(0),
@ -579,7 +695,7 @@ index 414e5ce144c..1c43df3b85f 100644
stAssignments: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{},
stDefaultCPUSet: cpuset.NewCPUSet(1, 2, 3, 4, 5, 6, 7), stDefaultCPUSet: cpuset.NewCPUSet(1, 2, 3, 4, 5, 6, 7),
pod: makePod("fakePod", "fakeContainer2", "8000m", "8000m"), pod: makePod("fakePod", "fakeContainer2", "8000m", "8000m"),
@@ -959,6 +971,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { @@ -959,6 +974,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
topo: topoSingleSocketHT, topo: topoSingleSocketHT,
numReservedCPUs: 2, numReservedCPUs: 2,
reserved: cpuset.NewCPUSet(0, 1), reserved: cpuset.NewCPUSet(0, 1),
@ -587,7 +703,7 @@ index 414e5ce144c..1c43df3b85f 100644
stAssignments: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{},
stDefaultCPUSet: cpuset.NewCPUSet(2, 3, 4, 5, 6, 7), stDefaultCPUSet: cpuset.NewCPUSet(2, 3, 4, 5, 6, 7),
pod: makePod("fakePod", "fakeContainer2", "1000m", "1000m"), pod: makePod("fakePod", "fakeContainer2", "1000m", "1000m"),
@@ -971,6 +984,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { @@ -971,6 +987,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
topo: topoSingleSocketHT, topo: topoSingleSocketHT,
numReservedCPUs: 2, numReservedCPUs: 2,
reserved: cpuset.NewCPUSet(0, 1), reserved: cpuset.NewCPUSet(0, 1),
@ -595,7 +711,7 @@ index 414e5ce144c..1c43df3b85f 100644
stAssignments: state.ContainerCPUAssignments{ stAssignments: state.ContainerCPUAssignments{
"fakePod": map[string]cpuset.CPUSet{ "fakePod": map[string]cpuset.CPUSet{
"fakeContainer100": cpuset.NewCPUSet(2, 3, 6, 7), "fakeContainer100": cpuset.NewCPUSet(2, 3, 6, 7),
@@ -987,6 +1001,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { @@ -987,6 +1004,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
topo: topoSingleSocketHT, topo: topoSingleSocketHT,
numReservedCPUs: 2, numReservedCPUs: 2,
reserved: cpuset.NewCPUSet(0, 1), reserved: cpuset.NewCPUSet(0, 1),
@ -603,7 +719,7 @@ index 414e5ce144c..1c43df3b85f 100644
stAssignments: state.ContainerCPUAssignments{ stAssignments: state.ContainerCPUAssignments{
"fakePod": map[string]cpuset.CPUSet{ "fakePod": map[string]cpuset.CPUSet{
"fakeContainer100": cpuset.NewCPUSet(2, 3, 6, 7), "fakeContainer100": cpuset.NewCPUSet(2, 3, 6, 7),
@@ -998,11 +1013,29 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { @@ -998,11 +1016,29 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
expCPUAlloc: true, expCPUAlloc: true,
expCSet: cpuset.NewCPUSet(0, 1), expCSet: cpuset.NewCPUSet(0, 1),
}, },
@ -634,6 +750,132 @@ index 414e5ce144c..1c43df3b85f 100644
st := &mockState{ st := &mockState{
assignments: testCase.stAssignments, assignments: testCase.stAssignments,
@@ -1128,3 +1164,125 @@ func TestStaticPolicyOptions(t *testing.T) {
})
}
}
+
+func makePodWithLabels(podLabels map[string]string) *v1.Pod {
+ return &v1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-pod",
+ Namespace: "test-namespace",
+ Labels: podLabels,
+ },
+ }
+}
+
+func fakeBuildConfigFromFlags(masterUrl string, kubeconfigPath string) (*restclient.Config, error) {
+
+ return &restclient.Config{}, nil
+}
+
+func fakeBuildConfigFromFlagsError(masterUrl string, kubeconfigPath string) (*restclient.Config, error) {
+
+ errString := fmt.Sprintf("%s file not found", kubeconfigPath)
+ return nil, errors.New(errString)
+
+}
+
+func getFakeInfraPodNamespace(_ string) (*v1.Namespace, error) {
+
+ return &v1.Namespace{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-namespace",
+ Labels: map[string]string{
+ "app.starlingx.io/component": "platform",
+ },
+ }}, nil
+}
+
+func getFakeNonInfraPodNamespace(_ string) (*v1.Namespace, error) {
+
+ return &v1.Namespace{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-namespace",
+ Labels: map[string]string{
+ "fake": "label",
+ }}}, nil
+
+}
+
+type kubeInfraPodTestCase struct {
+ description string
+ pod *v1.Pod
+ namespaceFunc getPodNamespace
+ expectedValue bool
+}
+
+func TestKubeInfraPod(t *testing.T) {
+ testCases := []kubeInfraPodTestCase{
+ {
+ description: "Pod with platform label and namespace without platform label",
+ pod: makePodWithLabels(map[string]string{
+ "app.starlingx.io/component": "platform",
+ }),
+ namespaceFunc: getFakeNonInfraPodNamespace,
+ expectedValue: true,
+
+ },
+ {
+ description: "Pod without platform label and namespace with platform label",
+ pod: makePodWithLabels(map[string]string{
+ "test": "label",
+ }),
+ namespaceFunc: getFakeInfraPodNamespace,
+ expectedValue: true,
+ },
+ {
+ description: "Pod without platform label and namespace without platform label",
+ pod: makePodWithLabels(map[string]string{
+ "test": "namespace",
+ }),
+ namespaceFunc: getFakeNonInfraPodNamespace,
+ expectedValue: false,
+ },
+
+ }
+
+ for _, testCase := range testCases {
+ t.Run(testCase.description, func(t *testing.T) {
+
+ varGetNamespaceObject = testCase.namespaceFunc
+ varBuildConfigFromFlags = fakeBuildConfigFromFlags
+ gotValue := isKubeInfra(testCase.pod)
+
+ if gotValue != testCase.expectedValue {
+ t.Errorf("StaticPolicy isKubeInfraPod() error %v. expected value %v actual value %v",
+ testCase.description, testCase.expectedValue, gotValue)
+ } else {
+ fmt.Printf("StaticPolicy isKubeInfraPod() test successful. : %v ", testCase.description)
+ }
+
+ })
+ }
+
+ test := kubeInfraPodTestCase{
+ description: "Failure reading kubeconfig file",
+ pod: makePodWithLabels(map[string]string{
+ "test": "namespace",
+ }),
+ namespaceFunc: getFakeNonInfraPodNamespace,
+ expectedValue: false,
+ }
+
+ varGetNamespaceObject = getPodNamespaceObject
+ varBuildConfigFromFlags = fakeBuildConfigFromFlagsError
+
+ gotValue := isKubeInfra(test.pod)
+
+ if gotValue != test.expectedValue {
+ t.Errorf("StaticPolicy isKubeInfraPod() error %v. expected value %v actual value %v",
+ test.description, test.expectedValue, gotValue)
+ } else {
+ fmt.Printf("StaticPolicy isKubeInfraPod() test successful. : %v ", test.description)
+ }
+
+}
+
diff --git a/pkg/kubelet/cm/devicemanager/manager_stub.go b/pkg/kubelet/cm/devicemanager/manager_stub.go diff --git a/pkg/kubelet/cm/devicemanager/manager_stub.go b/pkg/kubelet/cm/devicemanager/manager_stub.go
new file mode 100644 new file mode 100644
index 00000000000..e6874f88d8a index 00000000000..e6874f88d8a

View File

@ -1,4 +1,4 @@
From b51d6c0ba6dfd9a34c7f6832d17840820f9985eb Mon Sep 17 00:00:00 2001 From 8cdc168daa7fa8adc3a47c2e40900e4bf435babe Mon Sep 17 00:00:00 2001
From: Boovan Rajendran <boovan.rajendran@windriver.com> From: Boovan Rajendran <boovan.rajendran@windriver.com>
Date: Fri, 8 Sep 2023 10:46:07 -0400 Date: Fri, 8 Sep 2023 10:46:07 -0400
Subject: [PATCH] kubelet cpumanager introduce concept of isolated CPUs Subject: [PATCH] kubelet cpumanager introduce concept of isolated CPUs
@ -47,14 +47,15 @@ Signed-off-by: Gleb Aronsky <gleb.aronsky@windriver.com>
Signed-off-by: Ramesh Kumar Sivanandam <rameshkumar.sivanandam@windriver.com> Signed-off-by: Ramesh Kumar Sivanandam <rameshkumar.sivanandam@windriver.com>
Signed-off-by: Sachin Gopala Krishna <saching.krishna@windriver.com> Signed-off-by: Sachin Gopala Krishna <saching.krishna@windriver.com>
Signed-off-by: Boovan Rajendran <boovan.rajendran@windriver.com> Signed-off-by: Boovan Rajendran <boovan.rajendran@windriver.com>
Signed-off-by: Kaustubh Dhokte <kaustubh.dhokte@windriver.com>
--- ---
pkg/kubelet/cm/container_manager_linux.go | 1 + pkg/kubelet/cm/container_manager_linux.go | 1 +
pkg/kubelet/cm/cpumanager/cpu_manager.go | 35 ++++++- pkg/kubelet/cm/cpumanager/cpu_manager.go | 35 +++-
pkg/kubelet/cm/cpumanager/cpu_manager_test.go | 23 ++++- pkg/kubelet/cm/cpumanager/cpu_manager_test.go | 23 ++-
pkg/kubelet/cm/cpumanager/policy_static.go | 83 ++++++++++++++-- pkg/kubelet/cm/cpumanager/policy_static.go | 157 ++++++++++++++--
.../cm/cpumanager/policy_static_test.go | 53 ++++++++-- .../cm/cpumanager/policy_static_test.go | 177 +++++++++++++++++-
pkg/kubelet/cm/devicemanager/manager_stub.go | 99 +++++++++++++++++++ pkg/kubelet/cm/devicemanager/manager_stub.go | 99 ++++++++++
6 files changed, 273 insertions(+), 21 deletions(-) 6 files changed, 462 insertions(+), 30 deletions(-)
create mode 100644 pkg/kubelet/cm/devicemanager/manager_stub.go create mode 100644 pkg/kubelet/cm/devicemanager/manager_stub.go
diff --git a/pkg/kubelet/cm/container_manager_linux.go b/pkg/kubelet/cm/container_manager_linux.go diff --git a/pkg/kubelet/cm/container_manager_linux.go b/pkg/kubelet/cm/container_manager_linux.go
@ -265,18 +266,27 @@ index bb69b0ac084..44a88429a12 100644
testCases := []struct { testCases := []struct {
diff --git a/pkg/kubelet/cm/cpumanager/policy_static.go b/pkg/kubelet/cm/cpumanager/policy_static.go diff --git a/pkg/kubelet/cm/cpumanager/policy_static.go b/pkg/kubelet/cm/cpumanager/policy_static.go
index 1fdb49b52ad..49f63dd9efd 100644 index 1fdb49b52ad..99990fb596a 100644
--- a/pkg/kubelet/cm/cpumanager/policy_static.go --- a/pkg/kubelet/cm/cpumanager/policy_static.go
+++ b/pkg/kubelet/cm/cpumanager/policy_static.go +++ b/pkg/kubelet/cm/cpumanager/policy_static.go
@@ -18,6 +18,7 @@ package cpumanager @@ -17,10 +17,16 @@ limitations under the License.
package cpumanager
import ( import (
+ "context"
"fmt" "fmt"
+ "strconv" + "strconv"
+ k8sclient "k8s.io/client-go/kubernetes"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ restclient "k8s.io/client-go/rest"
v1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1"
utilfeature "k8s.io/apiserver/pkg/util/feature" utilfeature "k8s.io/apiserver/pkg/util/feature"
@@ -28,6 +29,7 @@ import ( + "k8s.io/client-go/tools/clientcmd"
"k8s.io/klog/v2"
podutil "k8s.io/kubernetes/pkg/api/v1/pod"
v1qos "k8s.io/kubernetes/pkg/apis/core/v1/helper/qos"
@@ -28,6 +34,7 @@ import (
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state"
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
"k8s.io/kubernetes/pkg/kubelet/cm/cpuset" "k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
@ -284,7 +294,32 @@ index 1fdb49b52ad..49f63dd9efd 100644
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager" "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager"
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask" "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask"
"k8s.io/kubernetes/pkg/kubelet/metrics" "k8s.io/kubernetes/pkg/kubelet/metrics"
@@ -110,6 +112,10 @@ type staticPolicy struct { @@ -43,6 +50,12 @@ const (
ErrorSMTAlignment = "SMTAlignmentError"
)
+type getPodNamespace func(string) (*v1.Namespace, error)
+type buildFromConfigFlag func(masterUrl string, kubeconfigPath string) (*restclient.Config, error)
+
+var varGetNamespaceObject getPodNamespace
+var varBuildConfigFromFlags buildFromConfigFlag
+
// SMTAlignmentError represents an error due to SMT alignment
type SMTAlignmentError struct {
RequestedCPUs int
@@ -62,11 +75,6 @@ func (e SMTAlignmentError) Type() string {
return ErrorSMTAlignment
}
-// Define namespaces used by platform infrastructure pods
-var infraNamespaces = [...]string{
- "kube-system", "armada", "cert-manager", "platform-deployment-manager", "portieris", "vault", "notification", "flux-helm", "metrics-server",
-}
-
// staticPolicy is a CPU manager policy that does not change CPU
// assignments for exclusively pinned guaranteed containers after the main
// container process starts.
@@ -110,6 +118,10 @@ type staticPolicy struct {
topology *topology.CPUTopology topology *topology.CPUTopology
// set of CPUs that is not available for exclusive assignment // set of CPUs that is not available for exclusive assignment
reservedCPUs cpuset.CPUSet reservedCPUs cpuset.CPUSet
@ -295,7 +330,7 @@ index 1fdb49b52ad..49f63dd9efd 100644
// If true, default CPUSet should exclude reserved CPUs // If true, default CPUSet should exclude reserved CPUs
excludeReserved bool excludeReserved bool
// Superset of reservedCPUs. It includes not just the reservedCPUs themselves, // Superset of reservedCPUs. It includes not just the reservedCPUs themselves,
@@ -132,7 +138,8 @@ var _ Policy = &staticPolicy{} @@ -132,7 +144,8 @@ var _ Policy = &staticPolicy{}
// NewStaticPolicy returns a CPU manager policy that does not change CPU // NewStaticPolicy returns a CPU manager policy that does not change CPU
// assignments for exclusively pinned guaranteed containers after the main // assignments for exclusively pinned guaranteed containers after the main
// container process starts. // container process starts.
@ -305,7 +340,7 @@ index 1fdb49b52ad..49f63dd9efd 100644
opts, err := NewStaticPolicyOptions(cpuPolicyOptions) opts, err := NewStaticPolicyOptions(cpuPolicyOptions)
if err != nil { if err != nil {
return nil, err return nil, err
@@ -147,6 +154,8 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv @@ -147,6 +160,8 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv
policy := &staticPolicy{ policy := &staticPolicy{
topology: topology, topology: topology,
affinity: affinity, affinity: affinity,
@ -314,7 +349,7 @@ index 1fdb49b52ad..49f63dd9efd 100644
excludeReserved: excludeReserved, excludeReserved: excludeReserved,
cpusToReuse: make(map[string]cpuset.CPUSet), cpusToReuse: make(map[string]cpuset.CPUSet),
options: opts, options: opts,
@@ -183,6 +192,12 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv @@ -183,6 +198,12 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv
policy.reservedCPUs = reserved policy.reservedCPUs = reserved
policy.reservedPhysicalCPUs = reservedPhysicalCPUs policy.reservedPhysicalCPUs = reservedPhysicalCPUs
@ -327,7 +362,7 @@ index 1fdb49b52ad..49f63dd9efd 100644
return policy, nil return policy, nil
} }
@@ -216,8 +231,9 @@ func (p *staticPolicy) validateState(s state.State) error { @@ -216,8 +237,9 @@ func (p *staticPolicy) validateState(s state.State) error {
} else { } else {
s.SetDefaultCPUSet(allCPUs) s.SetDefaultCPUSet(allCPUs)
} }
@ -339,7 +374,17 @@ index 1fdb49b52ad..49f63dd9efd 100644
return nil return nil
} }
@@ -316,16 +332,39 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai @@ -307,6 +329,9 @@ func (p *staticPolicy) updateCPUsToReuse(pod *v1.Pod, container *v1.Container, c
}
func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Container) (rerr error) {
+
+ varGetNamespaceObject = getPodNamespaceObject
+ varBuildConfigFromFlags = clientcmd.BuildConfigFromFlags
// Process infra pods before guaranteed pods
if isKubeInfra(pod) {
// Container belongs in reserved pool.
@@ -316,16 +341,39 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai
return nil return nil
} }
@ -381,7 +426,7 @@ index 1fdb49b52ad..49f63dd9efd 100644
numCPUs := p.guaranteedCPUs(pod, container) numCPUs := p.guaranteedCPUs(pod, container)
if numCPUs == 0 { if numCPUs == 0 {
// container belongs in the shared pool (nothing to do; use default cpuset) // container belongs in the shared pool (nothing to do; use default cpuset)
@@ -391,7 +430,9 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai @@ -391,7 +439,9 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai
} }
s.SetCPUSet(string(pod.UID), container.Name, cpuset) s.SetCPUSet(string(pod.UID), container.Name, cpuset)
p.updateCPUsToReuse(pod, container, cpuset) p.updateCPUsToReuse(pod, container, cpuset)
@ -392,10 +437,71 @@ index 1fdb49b52ad..49f63dd9efd 100644
return nil return nil
} }
@@ -709,6 +750,36 @@ func isKubeInfra(pod *v1.Pod) bool { @@ -699,14 +749,91 @@ func (p *staticPolicy) generateCPUTopologyHints(availableCPUs cpuset.CPUSet, reu
return false return hints
} }
+func getPodNamespaceObject(podNamespaceName string) (*v1.Namespace, error) {
+
+ cfg, err := varBuildConfigFromFlags("", "/etc/kubernetes/kubelet.conf")
+ if err != nil {
+ klog.Error("Failed to build client config from /etc/kubernetes/kubelet.conf: ", err.Error())
+ return nil, err
+ }
+
+ clientset, err := k8sclient.NewForConfig(cfg)
+ if err != nil {
+ klog.Error("Failed to get clientset for KUBECONFIG /etc/kubernetes/kubelet.conf: ", err.Error())
+ return nil, err
+ }
+
+ namespaceObj, err := clientset.CoreV1().Namespaces().Get(context.TODO(), podNamespaceName, metav1.GetOptions{})
+ if err != nil {
+ klog.Error("Error getting namespace object:", err.Error())
+ return nil, err
+ }
+
+ return namespaceObj, nil
+
+}
+
// check if a given pod is in a platform infrastructure namespace
func isKubeInfra(pod *v1.Pod) bool {
- for _, namespace := range infraNamespaces {
- if namespace == pod.Namespace {
- return true
- }
+
+ podName := pod.GetName()
+ podNamespaceName := pod.GetNamespace()
+
+ klog.InfoS("Checking pod ", podName , " for label 'app.starlingx.io/component=platform'.")
+ podLabels := pod.GetLabels()
+ val, ok := podLabels["app.starlingx.io/component"]
+ if (ok && val == "platform") {
+ klog.InfoS("Pod ", podName, " has 'app.starlingx.io/component=platform' label. Assigning platform CPUs.")
+ return true
}
+
+ klog.InfoS("Pod ", pod.GetName(), " does not have 'app.starlingx.io/component=platform' label. Checking its namespace information...")
+
+ namespaceObj, err := varGetNamespaceObject(podNamespaceName)
+ if err != nil {
+ return false
+ }
+
+ namespaceLabels := namespaceObj.GetLabels()
+ val, ok = namespaceLabels["app.starlingx.io/component"]
+ if ok && val == "platform" {
+ klog.InfoS("For pod: ", podName, ", its Namespace ", podNamespaceName, " has 'app.starlingx.io/component=platform' label. Assigning platform CPUs.")
+ return true
+ }
+
+ klog.InfoS("Neither pod ", podName, " nor its namespace ", podNamespaceName, " has 'app.starlingx.io/component=platform' label. Not assigning platform CPUs.")
return false
+
+}
+
+// get the isolated CPUs (if any) from the devices associated with a specific container +// get the isolated CPUs (if any) from the devices associated with a specific container
+func (p *staticPolicy) podIsolCPUs(pod *v1.Pod, container *v1.Container) cpuset.CPUSet { +func (p *staticPolicy) podIsolCPUs(pod *v1.Pod, container *v1.Container) cpuset.CPUSet {
+ // NOTE: This is required for TestStaticPolicyAdd() since makePod() does + // NOTE: This is required for TestStaticPolicyAdd() since makePod() does
@ -424,16 +530,28 @@ index 1fdb49b52ad..49f63dd9efd 100644
+ } + }
+ } + }
+ return cpuSet + return cpuSet
+} }
+
// isHintSocketAligned function return true if numa nodes in hint are socket aligned. // isHintSocketAligned function return true if numa nodes in hint are socket aligned.
func (p *staticPolicy) isHintSocketAligned(hint topologymanager.TopologyHint, minAffinitySize int) bool {
numaNodesBitMask := hint.NUMANodeAffinity.GetBits()
diff --git a/pkg/kubelet/cm/cpumanager/policy_static_test.go b/pkg/kubelet/cm/cpumanager/policy_static_test.go diff --git a/pkg/kubelet/cm/cpumanager/policy_static_test.go b/pkg/kubelet/cm/cpumanager/policy_static_test.go
index 63f31486d19..c25ee484a94 100644 index 63f31486d19..b0ce9d497d9 100644
--- a/pkg/kubelet/cm/cpumanager/policy_static_test.go --- a/pkg/kubelet/cm/cpumanager/policy_static_test.go
+++ b/pkg/kubelet/cm/cpumanager/policy_static_test.go +++ b/pkg/kubelet/cm/cpumanager/policy_static_test.go
@@ -28,6 +28,7 @@ import ( @@ -17,10 +17,13 @@ limitations under the License.
package cpumanager
import (
+ "errors"
"fmt"
"reflect"
"testing"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ restclient "k8s.io/client-go/rest"
v1 "k8s.io/api/core/v1"
utilfeature "k8s.io/apiserver/pkg/util/feature"
featuregatetesting "k8s.io/component-base/featuregate/testing"
@@ -28,6 +31,7 @@ import (
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state"
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
"k8s.io/kubernetes/pkg/kubelet/cm/cpuset" "k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
@ -441,7 +559,7 @@ index 63f31486d19..c25ee484a94 100644
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager" "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager"
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask" "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask"
) )
@@ -70,8 +71,9 @@ func (spt staticPolicyTest) PseudoClone() staticPolicyTest { @@ -70,8 +74,9 @@ func (spt staticPolicyTest) PseudoClone() staticPolicyTest {
} }
func TestStaticPolicyName(t *testing.T) { func TestStaticPolicyName(t *testing.T) {
@ -452,7 +570,7 @@ index 63f31486d19..c25ee484a94 100644
policyName := policy.Name() policyName := policy.Name()
if policyName != "static" { if policyName != "static" {
@@ -81,6 +83,7 @@ func TestStaticPolicyName(t *testing.T) { @@ -81,6 +86,7 @@ func TestStaticPolicyName(t *testing.T) {
} }
func TestStaticPolicyStart(t *testing.T) { func TestStaticPolicyStart(t *testing.T) {
@ -460,7 +578,7 @@ index 63f31486d19..c25ee484a94 100644
testCases := []staticPolicyTest{ testCases := []staticPolicyTest{
{ {
description: "non-corrupted state", description: "non-corrupted state",
@@ -156,7 +159,7 @@ func TestStaticPolicyStart(t *testing.T) { @@ -156,7 +162,7 @@ func TestStaticPolicyStart(t *testing.T) {
} }
for _, testCase := range testCases { for _, testCase := range testCases {
t.Run(testCase.description, func(t *testing.T) { t.Run(testCase.description, func(t *testing.T) {
@ -469,7 +587,7 @@ index 63f31486d19..c25ee484a94 100644
policy := p.(*staticPolicy) policy := p.(*staticPolicy)
st := &mockState{ st := &mockState{
@@ -204,7 +207,6 @@ func TestStaticPolicyAdd(t *testing.T) { @@ -204,7 +210,6 @@ func TestStaticPolicyAdd(t *testing.T) {
largeTopoCPUSet := cpuset.New(largeTopoCPUids...) largeTopoCPUSet := cpuset.New(largeTopoCPUids...)
largeTopoSock0CPUSet := cpuset.New(largeTopoSock0CPUids...) largeTopoSock0CPUSet := cpuset.New(largeTopoSock0CPUids...)
largeTopoSock1CPUSet := cpuset.New(largeTopoSock1CPUids...) largeTopoSock1CPUSet := cpuset.New(largeTopoSock1CPUids...)
@ -477,7 +595,7 @@ index 63f31486d19..c25ee484a94 100644
// these are the cases which must behave the same regardless the policy options. // these are the cases which must behave the same regardless the policy options.
// So we will permutate the options to ensure this holds true. // So we will permutate the options to ensure this holds true.
@@ -627,7 +629,9 @@ func runStaticPolicyTestCase(t *testing.T, testCase staticPolicyTest) { @@ -627,7 +632,9 @@ func runStaticPolicyTestCase(t *testing.T, testCase staticPolicyTest) {
cpus = testCase.reservedCPUs.Clone() cpus = testCase.reservedCPUs.Clone()
} }
testExcl := false testExcl := false
@ -488,7 +606,7 @@ index 63f31486d19..c25ee484a94 100644
st := &mockState{ st := &mockState{
assignments: testCase.stAssignments, assignments: testCase.stAssignments,
@@ -674,6 +678,8 @@ func runStaticPolicyTestCaseWithFeatureGate(t *testing.T, testCase staticPolicyT @@ -674,6 +681,8 @@ func runStaticPolicyTestCaseWithFeatureGate(t *testing.T, testCase staticPolicyT
} }
func TestStaticPolicyReuseCPUs(t *testing.T) { func TestStaticPolicyReuseCPUs(t *testing.T) {
@ -497,7 +615,7 @@ index 63f31486d19..c25ee484a94 100644
testCases := []struct { testCases := []struct {
staticPolicyTest staticPolicyTest
expCSetAfterAlloc cpuset.CPUSet expCSetAfterAlloc cpuset.CPUSet
@@ -698,7 +704,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) { @@ -698,7 +707,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) {
} }
for _, testCase := range testCases { for _, testCase := range testCases {
@ -506,7 +624,7 @@ index 63f31486d19..c25ee484a94 100644
st := &mockState{ st := &mockState{
assignments: testCase.stAssignments, assignments: testCase.stAssignments,
@@ -731,6 +737,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) { @@ -731,6 +740,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) {
func TestStaticPolicyRemove(t *testing.T) { func TestStaticPolicyRemove(t *testing.T) {
excludeReserved := false excludeReserved := false
@ -514,7 +632,7 @@ index 63f31486d19..c25ee484a94 100644
testCases := []staticPolicyTest{ testCases := []staticPolicyTest{
{ {
description: "SingleSocketHT, DeAllocOneContainer", description: "SingleSocketHT, DeAllocOneContainer",
@@ -789,7 +796,7 @@ func TestStaticPolicyRemove(t *testing.T) { @@ -789,7 +799,7 @@ func TestStaticPolicyRemove(t *testing.T) {
} }
for _, testCase := range testCases { for _, testCase := range testCases {
@ -523,7 +641,7 @@ index 63f31486d19..c25ee484a94 100644
st := &mockState{ st := &mockState{
assignments: testCase.stAssignments, assignments: testCase.stAssignments,
@@ -812,6 +819,7 @@ func TestStaticPolicyRemove(t *testing.T) { @@ -812,6 +822,7 @@ func TestStaticPolicyRemove(t *testing.T) {
func TestTopologyAwareAllocateCPUs(t *testing.T) { func TestTopologyAwareAllocateCPUs(t *testing.T) {
excludeReserved := false excludeReserved := false
@ -531,7 +649,7 @@ index 63f31486d19..c25ee484a94 100644
testCases := []struct { testCases := []struct {
description string description string
topo *topology.CPUTopology topo *topology.CPUTopology
@@ -880,7 +888,8 @@ func TestTopologyAwareAllocateCPUs(t *testing.T) { @@ -880,7 +891,8 @@ func TestTopologyAwareAllocateCPUs(t *testing.T) {
}, },
} }
for _, tc := range testCases { for _, tc := range testCases {
@ -541,7 +659,7 @@ index 63f31486d19..c25ee484a94 100644
policy := p.(*staticPolicy) policy := p.(*staticPolicy)
st := &mockState{ st := &mockState{
assignments: tc.stAssignments, assignments: tc.stAssignments,
@@ -913,6 +922,7 @@ type staticPolicyTestWithResvList struct { @@ -913,6 +925,7 @@ type staticPolicyTestWithResvList struct {
topo *topology.CPUTopology topo *topology.CPUTopology
numReservedCPUs int numReservedCPUs int
reserved cpuset.CPUSet reserved cpuset.CPUSet
@ -549,7 +667,7 @@ index 63f31486d19..c25ee484a94 100644
stAssignments state.ContainerCPUAssignments stAssignments state.ContainerCPUAssignments
stDefaultCPUSet cpuset.CPUSet stDefaultCPUSet cpuset.CPUSet
pod *v1.Pod pod *v1.Pod
@@ -923,6 +933,8 @@ type staticPolicyTestWithResvList struct { @@ -923,6 +936,8 @@ type staticPolicyTestWithResvList struct {
} }
func TestStaticPolicyStartWithResvList(t *testing.T) { func TestStaticPolicyStartWithResvList(t *testing.T) {
@ -558,7 +676,7 @@ index 63f31486d19..c25ee484a94 100644
testCases := []staticPolicyTestWithResvList{ testCases := []staticPolicyTestWithResvList{
{ {
description: "empty cpuset", description: "empty cpuset",
@@ -952,10 +964,9 @@ func TestStaticPolicyStartWithResvList(t *testing.T) { @@ -952,10 +967,9 @@ func TestStaticPolicyStartWithResvList(t *testing.T) {
expNewErr: fmt.Errorf("[cpumanager] unable to reserve the required amount of CPUs (size of 0-1 did not equal 1)"), expNewErr: fmt.Errorf("[cpumanager] unable to reserve the required amount of CPUs (size of 0-1 did not equal 1)"),
}, },
} }
@ -570,7 +688,7 @@ index 63f31486d19..c25ee484a94 100644
if !reflect.DeepEqual(err, testCase.expNewErr) { if !reflect.DeepEqual(err, testCase.expNewErr) {
t.Errorf("StaticPolicy Start() error (%v). expected error: %v but got: %v", t.Errorf("StaticPolicy Start() error (%v). expected error: %v but got: %v",
@@ -996,6 +1007,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { @@ -996,6 +1010,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
topo: topoSingleSocketHT, topo: topoSingleSocketHT,
numReservedCPUs: 1, numReservedCPUs: 1,
reserved: cpuset.New(0), reserved: cpuset.New(0),
@ -578,7 +696,7 @@ index 63f31486d19..c25ee484a94 100644
stAssignments: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{},
stDefaultCPUSet: cpuset.New(1, 2, 3, 4, 5, 6, 7), stDefaultCPUSet: cpuset.New(1, 2, 3, 4, 5, 6, 7),
pod: makePod("fakePod", "fakeContainer2", "8000m", "8000m"), pod: makePod("fakePod", "fakeContainer2", "8000m", "8000m"),
@@ -1008,6 +1020,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { @@ -1008,6 +1023,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
topo: topoSingleSocketHT, topo: topoSingleSocketHT,
numReservedCPUs: 2, numReservedCPUs: 2,
reserved: cpuset.New(0, 1), reserved: cpuset.New(0, 1),
@ -586,7 +704,7 @@ index 63f31486d19..c25ee484a94 100644
stAssignments: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{},
stDefaultCPUSet: cpuset.New(2, 3, 4, 5, 6, 7), stDefaultCPUSet: cpuset.New(2, 3, 4, 5, 6, 7),
pod: makePod("fakePod", "fakeContainer2", "1000m", "1000m"), pod: makePod("fakePod", "fakeContainer2", "1000m", "1000m"),
@@ -1020,6 +1033,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { @@ -1020,6 +1036,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
topo: topoSingleSocketHT, topo: topoSingleSocketHT,
numReservedCPUs: 2, numReservedCPUs: 2,
reserved: cpuset.New(0, 1), reserved: cpuset.New(0, 1),
@ -594,7 +712,7 @@ index 63f31486d19..c25ee484a94 100644
stAssignments: state.ContainerCPUAssignments{ stAssignments: state.ContainerCPUAssignments{
"fakePod": map[string]cpuset.CPUSet{ "fakePod": map[string]cpuset.CPUSet{
"fakeContainer100": cpuset.New(2, 3, 6, 7), "fakeContainer100": cpuset.New(2, 3, 6, 7),
@@ -1036,6 +1050,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { @@ -1036,6 +1053,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
topo: topoSingleSocketHT, topo: topoSingleSocketHT,
numReservedCPUs: 2, numReservedCPUs: 2,
reserved: cpuset.New(0, 1), reserved: cpuset.New(0, 1),
@ -602,7 +720,7 @@ index 63f31486d19..c25ee484a94 100644
stAssignments: state.ContainerCPUAssignments{ stAssignments: state.ContainerCPUAssignments{
"fakePod": map[string]cpuset.CPUSet{ "fakePod": map[string]cpuset.CPUSet{
"fakeContainer100": cpuset.New(2, 3, 6, 7), "fakeContainer100": cpuset.New(2, 3, 6, 7),
@@ -1047,11 +1062,29 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { @@ -1047,11 +1065,29 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
expCPUAlloc: true, expCPUAlloc: true,
expCSet: cpuset.New(0, 1), expCSet: cpuset.New(0, 1),
}, },
@ -633,6 +751,131 @@ index 63f31486d19..c25ee484a94 100644
st := &mockState{ st := &mockState{
assignments: testCase.stAssignments, assignments: testCase.stAssignments,
@@ -1182,3 +1218,124 @@ func newCPUSetPtr(cpus ...int) *cpuset.CPUSet {
ret := cpuset.New(cpus...)
return &ret
}
+
+func makePodWithLabels(podLabels map[string]string) *v1.Pod {
+ return &v1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-pod",
+ Namespace: "test-namespace",
+ Labels: podLabels,
+ },
+ }
+}
+
+func fakeBuildConfigFromFlags(masterUrl string, kubeconfigPath string) (*restclient.Config, error) {
+
+ return &restclient.Config{}, nil
+}
+
+func fakeBuildConfigFromFlagsError(masterUrl string, kubeconfigPath string) (*restclient.Config, error) {
+
+ errString := fmt.Sprintf("%s file not found", kubeconfigPath)
+ return nil, errors.New(errString)
+
+}
+
+func getFakeInfraPodNamespace(_ string) (*v1.Namespace, error) {
+
+ return &v1.Namespace{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-namespace",
+ Labels: map[string]string{
+ "app.starlingx.io/component": "platform",
+ },
+ }}, nil
+}
+
+func getFakeNonInfraPodNamespace(_ string) (*v1.Namespace, error) {
+
+ return &v1.Namespace{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-namespace",
+ Labels: map[string]string{
+ "fake": "label",
+ }}}, nil
+
+}
+
+type kubeInfraPodTestCase struct {
+ description string
+ pod *v1.Pod
+ namespaceFunc getPodNamespace
+ expectedValue bool
+}
+
+func TestKubeInfraPod(t *testing.T) {
+ testCases := []kubeInfraPodTestCase{
+ {
+ description: "Pod with platform label and namespace without platform label",
+ pod: makePodWithLabels(map[string]string{
+ "app.starlingx.io/component": "platform",
+ }),
+ namespaceFunc: getFakeNonInfraPodNamespace,
+ expectedValue: true,
+
+ },
+ {
+ description: "Pod without platform label and namespace with platform label",
+ pod: makePodWithLabels(map[string]string{
+ "test": "label",
+ }),
+ namespaceFunc: getFakeInfraPodNamespace,
+ expectedValue: true,
+ },
+ {
+ description: "Pod without platform label and namespace without platform label",
+ pod: makePodWithLabels(map[string]string{
+ "test": "namespace",
+ }),
+ namespaceFunc: getFakeNonInfraPodNamespace,
+ expectedValue: false,
+ },
+
+ }
+
+ for _, testCase := range testCases {
+ t.Run(testCase.description, func(t *testing.T) {
+
+ varGetNamespaceObject = testCase.namespaceFunc
+ varBuildConfigFromFlags = fakeBuildConfigFromFlags
+ gotValue := isKubeInfra(testCase.pod)
+
+ if gotValue != testCase.expectedValue {
+ t.Errorf("StaticPolicy isKubeInfraPod() error %v. expected value %v actual value %v",
+ testCase.description, testCase.expectedValue, gotValue)
+ } else {
+ fmt.Printf("StaticPolicy isKubeInfraPod() test successful. : %v ", testCase.description)
+ }
+
+ })
+ }
+
+ test := kubeInfraPodTestCase{
+ description: "Failure reading kubeconfig file",
+ pod: makePodWithLabels(map[string]string{
+ "test": "namespace",
+ }),
+ namespaceFunc: getFakeNonInfraPodNamespace,
+ expectedValue: false,
+ }
+
+ varGetNamespaceObject = getPodNamespaceObject
+ varBuildConfigFromFlags = fakeBuildConfigFromFlagsError
+
+ gotValue := isKubeInfra(test.pod)
+
+ if gotValue != test.expectedValue {
+ t.Errorf("StaticPolicy isKubeInfraPod() error %v. expected value %v actual value %v",
+ test.description, test.expectedValue, gotValue)
+ } else {
+ fmt.Printf("StaticPolicy isKubeInfraPod() test successful. : %v ", test.description)
+ }
+
+}
diff --git a/pkg/kubelet/cm/devicemanager/manager_stub.go b/pkg/kubelet/cm/devicemanager/manager_stub.go diff --git a/pkg/kubelet/cm/devicemanager/manager_stub.go b/pkg/kubelet/cm/devicemanager/manager_stub.go
new file mode 100644 new file mode 100644
index 00000000000..e6874f88d8a index 00000000000..e6874f88d8a

View File

@ -1,4 +1,4 @@
From 856dfbe0960418618262998ebce65d0ae070c1bb Mon Sep 17 00:00:00 2001 From 4f74e4f9bbfd2909a3c740cb6a1b5233af277f72 Mon Sep 17 00:00:00 2001
From: Saba Touheed Mujawar <sabatouheed.mujawar@windriver.com> From: Saba Touheed Mujawar <sabatouheed.mujawar@windriver.com>
Date: Fri, 1 Dec 2023 07:42:14 -0500 Date: Fri, 1 Dec 2023 07:42:14 -0500
Subject: [PATCH] kubelet cpumanager introduce concept of isolated CPUs Subject: [PATCH] kubelet cpumanager introduce concept of isolated CPUs
@ -48,14 +48,15 @@ Signed-off-by: Ramesh Kumar Sivanandam <rameshkumar.sivanandam@windriver.com>
Signed-off-by: Sachin Gopala Krishna <saching.krishna@windriver.com> Signed-off-by: Sachin Gopala Krishna <saching.krishna@windriver.com>
Signed-off-by: Boovan Rajendran <boovan.rajendran@windriver.com> Signed-off-by: Boovan Rajendran <boovan.rajendran@windriver.com>
Signed-off-by: Saba Touheed Mujawar <sabatouheed.mujawar@windriver.com> Signed-off-by: Saba Touheed Mujawar <sabatouheed.mujawar@windriver.com>
Signed-off-by: Kaustubh Dhokte <kaustubh.dhokte@windriver.com>
--- ---
pkg/kubelet/cm/container_manager_linux.go | 1 + pkg/kubelet/cm/container_manager_linux.go | 1 +
pkg/kubelet/cm/cpumanager/cpu_manager.go | 35 +++++- pkg/kubelet/cm/cpumanager/cpu_manager.go | 35 +++-
pkg/kubelet/cm/cpumanager/cpu_manager_test.go | 23 +++- pkg/kubelet/cm/cpumanager/cpu_manager_test.go | 23 ++-
pkg/kubelet/cm/cpumanager/policy_static.go | 83 ++++++++++++- pkg/kubelet/cm/cpumanager/policy_static.go | 157 +++++++++++++--
.../cm/cpumanager/policy_static_test.go | 53 +++++++-- .../cm/cpumanager/policy_static_test.go | 178 +++++++++++++++++-
pkg/kubelet/cm/devicemanager/manager_stub.go | 110 ++++++++++++++++++ pkg/kubelet/cm/devicemanager/manager_stub.go | 110 +++++++++++
6 files changed, 284 insertions(+), 21 deletions(-) 6 files changed, 474 insertions(+), 30 deletions(-)
create mode 100644 pkg/kubelet/cm/devicemanager/manager_stub.go create mode 100644 pkg/kubelet/cm/devicemanager/manager_stub.go
diff --git a/pkg/kubelet/cm/container_manager_linux.go b/pkg/kubelet/cm/container_manager_linux.go diff --git a/pkg/kubelet/cm/container_manager_linux.go b/pkg/kubelet/cm/container_manager_linux.go
@ -266,18 +267,26 @@ index daecd35f67b..2298cc037fe 100644
testCases := []struct { testCases := []struct {
diff --git a/pkg/kubelet/cm/cpumanager/policy_static.go b/pkg/kubelet/cm/cpumanager/policy_static.go diff --git a/pkg/kubelet/cm/cpumanager/policy_static.go b/pkg/kubelet/cm/cpumanager/policy_static.go
index 9b7545c2207..e9a2defd848 100644 index 9b7545c2207..e32803306c0 100644
--- a/pkg/kubelet/cm/cpumanager/policy_static.go --- a/pkg/kubelet/cm/cpumanager/policy_static.go
+++ b/pkg/kubelet/cm/cpumanager/policy_static.go +++ b/pkg/kubelet/cm/cpumanager/policy_static.go
@@ -18,6 +18,7 @@ package cpumanager @@ -17,9 +17,15 @@ limitations under the License.
package cpumanager
import ( import (
+ "context"
"fmt" "fmt"
+ "strconv" + "strconv"
+ k8sclient "k8s.io/client-go/kubernetes"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ restclient "k8s.io/client-go/rest"
v1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1"
+ "k8s.io/client-go/tools/clientcmd"
utilfeature "k8s.io/apiserver/pkg/util/feature" utilfeature "k8s.io/apiserver/pkg/util/feature"
@@ -27,6 +28,7 @@ import ( "k8s.io/klog/v2"
podutil "k8s.io/kubernetes/pkg/api/v1/pod"
@@ -27,6 +33,7 @@ import (
"k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state"
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
@ -285,7 +294,32 @@ index 9b7545c2207..e9a2defd848 100644
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager" "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager"
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask" "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask"
"k8s.io/kubernetes/pkg/kubelet/metrics" "k8s.io/kubernetes/pkg/kubelet/metrics"
@@ -110,6 +112,10 @@ type staticPolicy struct { @@ -43,6 +50,12 @@ const (
ErrorSMTAlignment = "SMTAlignmentError"
)
+type getPodNamespace func(string) (*v1.Namespace, error)
+type buildFromConfigFlag func(masterUrl string, kubeconfigPath string) (*restclient.Config, error)
+
+var varGetNamespaceObject getPodNamespace
+var varBuildConfigFromFlags buildFromConfigFlag
+
// SMTAlignmentError represents an error due to SMT alignment
type SMTAlignmentError struct {
RequestedCPUs int
@@ -62,11 +75,6 @@ func (e SMTAlignmentError) Type() string {
return ErrorSMTAlignment
}
-// Define namespaces used by platform infrastructure pods
-var infraNamespaces = [...]string{
- "kube-system", "armada", "cert-manager", "platform-deployment-manager", "portieris", "vault", "notification", "flux-helm", "metrics-server",
-}
-
// staticPolicy is a CPU manager policy that does not change CPU
// assignments for exclusively pinned guaranteed containers after the main
// container process starts.
@@ -110,6 +118,10 @@ type staticPolicy struct {
topology *topology.CPUTopology topology *topology.CPUTopology
// set of CPUs that is not available for exclusive assignment // set of CPUs that is not available for exclusive assignment
reservedCPUs cpuset.CPUSet reservedCPUs cpuset.CPUSet
@ -296,7 +330,7 @@ index 9b7545c2207..e9a2defd848 100644
// If true, default CPUSet should exclude reserved CPUs // If true, default CPUSet should exclude reserved CPUs
excludeReserved bool excludeReserved bool
// Superset of reservedCPUs. It includes not just the reservedCPUs themselves, // Superset of reservedCPUs. It includes not just the reservedCPUs themselves,
@@ -132,7 +138,8 @@ var _ Policy = &staticPolicy{} @@ -132,7 +144,8 @@ var _ Policy = &staticPolicy{}
// NewStaticPolicy returns a CPU manager policy that does not change CPU // NewStaticPolicy returns a CPU manager policy that does not change CPU
// assignments for exclusively pinned guaranteed containers after the main // assignments for exclusively pinned guaranteed containers after the main
// container process starts. // container process starts.
@ -306,7 +340,7 @@ index 9b7545c2207..e9a2defd848 100644
opts, err := NewStaticPolicyOptions(cpuPolicyOptions) opts, err := NewStaticPolicyOptions(cpuPolicyOptions)
if err != nil { if err != nil {
return nil, err return nil, err
@@ -147,6 +154,8 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv @@ -147,6 +160,8 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv
policy := &staticPolicy{ policy := &staticPolicy{
topology: topology, topology: topology,
affinity: affinity, affinity: affinity,
@ -315,7 +349,7 @@ index 9b7545c2207..e9a2defd848 100644
excludeReserved: excludeReserved, excludeReserved: excludeReserved,
cpusToReuse: make(map[string]cpuset.CPUSet), cpusToReuse: make(map[string]cpuset.CPUSet),
options: opts, options: opts,
@@ -183,6 +192,12 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv @@ -183,6 +198,12 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv
policy.reservedCPUs = reserved policy.reservedCPUs = reserved
policy.reservedPhysicalCPUs = reservedPhysicalCPUs policy.reservedPhysicalCPUs = reservedPhysicalCPUs
@ -328,7 +362,7 @@ index 9b7545c2207..e9a2defd848 100644
return policy, nil return policy, nil
} }
@@ -216,8 +231,9 @@ func (p *staticPolicy) validateState(s state.State) error { @@ -216,8 +237,9 @@ func (p *staticPolicy) validateState(s state.State) error {
} else { } else {
s.SetDefaultCPUSet(allCPUs) s.SetDefaultCPUSet(allCPUs)
} }
@ -340,7 +374,17 @@ index 9b7545c2207..e9a2defd848 100644
return nil return nil
} }
@@ -316,16 +332,39 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai @@ -307,6 +329,9 @@ func (p *staticPolicy) updateCPUsToReuse(pod *v1.Pod, container *v1.Container, c
}
func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Container) (rerr error) {
+
+ varGetNamespaceObject = getPodNamespaceObject
+ varBuildConfigFromFlags = clientcmd.BuildConfigFromFlags
// Process infra pods before guaranteed pods
if isKubeInfra(pod) {
// Container belongs in reserved pool.
@@ -316,16 +341,39 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai
return nil return nil
} }
@ -382,7 +426,7 @@ index 9b7545c2207..e9a2defd848 100644
numCPUs := p.guaranteedCPUs(pod, container) numCPUs := p.guaranteedCPUs(pod, container)
if numCPUs == 0 { if numCPUs == 0 {
// container belongs in the shared pool (nothing to do; use default cpuset) // container belongs in the shared pool (nothing to do; use default cpuset)
@@ -391,7 +430,9 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai @@ -391,7 +439,9 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai
} }
s.SetCPUSet(string(pod.UID), container.Name, cpuset) s.SetCPUSet(string(pod.UID), container.Name, cpuset)
p.updateCPUsToReuse(pod, container, cpuset) p.updateCPUsToReuse(pod, container, cpuset)
@ -393,10 +437,71 @@ index 9b7545c2207..e9a2defd848 100644
return nil return nil
} }
@@ -709,6 +750,36 @@ func isKubeInfra(pod *v1.Pod) bool { @@ -699,14 +749,91 @@ func (p *staticPolicy) generateCPUTopologyHints(availableCPUs cpuset.CPUSet, reu
return false return hints
} }
+func getPodNamespaceObject(podNamespaceName string) (*v1.Namespace, error) {
+
+ cfg, err := varBuildConfigFromFlags("", "/etc/kubernetes/kubelet.conf")
+ if err != nil {
+ klog.Error("Failed to build client config from /etc/kubernetes/kubelet.conf: ", err.Error())
+ return nil, err
+ }
+
+ clientset, err := k8sclient.NewForConfig(cfg)
+ if err != nil {
+ klog.Error("Failed to get clientset for KUBECONFIG /etc/kubernetes/kubelet.conf: ", err.Error())
+ return nil, err
+ }
+
+ namespaceObj, err := clientset.CoreV1().Namespaces().Get(context.TODO(), podNamespaceName, metav1.GetOptions{})
+ if err != nil {
+ klog.Error("Error getting namespace object:", err.Error())
+ return nil, err
+ }
+
+ return namespaceObj, nil
+
+}
+
// check if a given pod is in a platform infrastructure namespace
func isKubeInfra(pod *v1.Pod) bool {
- for _, namespace := range infraNamespaces {
- if namespace == pod.Namespace {
- return true
- }
+
+ podName := pod.GetName()
+ podNamespaceName := pod.GetNamespace()
+
+ klog.InfoS("Checking pod ", podName , " for label 'app.starlingx.io/component=platform'.")
+ podLabels := pod.GetLabels()
+ val, ok := podLabels["app.starlingx.io/component"]
+ if (ok && val == "platform") {
+ klog.InfoS("Pod ", podName, " has 'app.starlingx.io/component=platform' label. Assigning platform CPUs.")
+ return true
}
+
+ klog.InfoS("Pod ", pod.GetName(), " does not have 'app.starlingx.io/component=platform' label. Checking its namespace information...")
+
+ namespaceObj, err := varGetNamespaceObject(podNamespaceName)
+ if err != nil {
+ return false
+ }
+
+ namespaceLabels := namespaceObj.GetLabels()
+ val, ok = namespaceLabels["app.starlingx.io/component"]
+ if ok && val == "platform" {
+ klog.InfoS("For pod: ", podName, ", its Namespace ", podNamespaceName, " has 'app.starlingx.io/component=platform' label. Assigning platform CPUs.")
+ return true
+ }
+
+ klog.InfoS("Neither pod ", podName, " nor its namespace ", podNamespaceName, " has 'app.starlingx.io/component=platform' label. Not assigning platform CPUs.")
return false
+
+}
+
+// get the isolated CPUs (if any) from the devices associated with a specific container +// get the isolated CPUs (if any) from the devices associated with a specific container
+func (p *staticPolicy) podIsolCPUs(pod *v1.Pod, container *v1.Container) cpuset.CPUSet { +func (p *staticPolicy) podIsolCPUs(pod *v1.Pod, container *v1.Container) cpuset.CPUSet {
+ // NOTE: This is required for TestStaticPolicyAdd() since makePod() does + // NOTE: This is required for TestStaticPolicyAdd() since makePod() does
@ -425,16 +530,27 @@ index 9b7545c2207..e9a2defd848 100644
+ } + }
+ } + }
+ return cpuSet + return cpuSet
+} }
+
// isHintSocketAligned function return true if numa nodes in hint are socket aligned. // isHintSocketAligned function return true if numa nodes in hint are socket aligned.
func (p *staticPolicy) isHintSocketAligned(hint topologymanager.TopologyHint, minAffinitySize int) bool {
numaNodesBitMask := hint.NUMANodeAffinity.GetBits()
diff --git a/pkg/kubelet/cm/cpumanager/policy_static_test.go b/pkg/kubelet/cm/cpumanager/policy_static_test.go diff --git a/pkg/kubelet/cm/cpumanager/policy_static_test.go b/pkg/kubelet/cm/cpumanager/policy_static_test.go
index b864c6c57c6..cb363bb29ab 100644 index b864c6c57c6..d94f8fdac14 100644
--- a/pkg/kubelet/cm/cpumanager/policy_static_test.go --- a/pkg/kubelet/cm/cpumanager/policy_static_test.go
+++ b/pkg/kubelet/cm/cpumanager/policy_static_test.go +++ b/pkg/kubelet/cm/cpumanager/policy_static_test.go
@@ -27,6 +27,7 @@ import ( @@ -17,16 +17,20 @@ limitations under the License.
package cpumanager
import (
+ "errors"
"fmt"
"reflect"
"testing"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ restclient "k8s.io/client-go/rest"
v1 "k8s.io/api/core/v1"
utilfeature "k8s.io/apiserver/pkg/util/feature"
featuregatetesting "k8s.io/component-base/featuregate/testing"
pkgfeatures "k8s.io/kubernetes/pkg/features" pkgfeatures "k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state"
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
@ -442,7 +558,7 @@ index b864c6c57c6..cb363bb29ab 100644
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager" "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager"
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask" "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask"
"k8s.io/utils/cpuset" "k8s.io/utils/cpuset"
@@ -70,8 +71,9 @@ func (spt staticPolicyTest) PseudoClone() staticPolicyTest { @@ -70,8 +74,9 @@ func (spt staticPolicyTest) PseudoClone() staticPolicyTest {
} }
func TestStaticPolicyName(t *testing.T) { func TestStaticPolicyName(t *testing.T) {
@ -453,7 +569,7 @@ index b864c6c57c6..cb363bb29ab 100644
policyName := policy.Name() policyName := policy.Name()
if policyName != "static" { if policyName != "static" {
@@ -81,6 +83,7 @@ func TestStaticPolicyName(t *testing.T) { @@ -81,6 +86,7 @@ func TestStaticPolicyName(t *testing.T) {
} }
func TestStaticPolicyStart(t *testing.T) { func TestStaticPolicyStart(t *testing.T) {
@ -461,7 +577,7 @@ index b864c6c57c6..cb363bb29ab 100644
testCases := []staticPolicyTest{ testCases := []staticPolicyTest{
{ {
description: "non-corrupted state", description: "non-corrupted state",
@@ -156,7 +159,7 @@ func TestStaticPolicyStart(t *testing.T) { @@ -156,7 +162,7 @@ func TestStaticPolicyStart(t *testing.T) {
} }
for _, testCase := range testCases { for _, testCase := range testCases {
t.Run(testCase.description, func(t *testing.T) { t.Run(testCase.description, func(t *testing.T) {
@ -470,7 +586,7 @@ index b864c6c57c6..cb363bb29ab 100644
policy := p.(*staticPolicy) policy := p.(*staticPolicy)
st := &mockState{ st := &mockState{
@@ -204,7 +207,6 @@ func TestStaticPolicyAdd(t *testing.T) { @@ -204,7 +210,6 @@ func TestStaticPolicyAdd(t *testing.T) {
largeTopoCPUSet := cpuset.New(largeTopoCPUids...) largeTopoCPUSet := cpuset.New(largeTopoCPUids...)
largeTopoSock0CPUSet := cpuset.New(largeTopoSock0CPUids...) largeTopoSock0CPUSet := cpuset.New(largeTopoSock0CPUids...)
largeTopoSock1CPUSet := cpuset.New(largeTopoSock1CPUids...) largeTopoSock1CPUSet := cpuset.New(largeTopoSock1CPUids...)
@ -478,7 +594,7 @@ index b864c6c57c6..cb363bb29ab 100644
// these are the cases which must behave the same regardless the policy options. // these are the cases which must behave the same regardless the policy options.
// So we will permutate the options to ensure this holds true. // So we will permutate the options to ensure this holds true.
@@ -627,7 +629,9 @@ func runStaticPolicyTestCase(t *testing.T, testCase staticPolicyTest) { @@ -627,7 +632,9 @@ func runStaticPolicyTestCase(t *testing.T, testCase staticPolicyTest) {
cpus = testCase.reservedCPUs.Clone() cpus = testCase.reservedCPUs.Clone()
} }
testExcl := false testExcl := false
@ -489,7 +605,7 @@ index b864c6c57c6..cb363bb29ab 100644
st := &mockState{ st := &mockState{
assignments: testCase.stAssignments, assignments: testCase.stAssignments,
@@ -674,6 +678,8 @@ func runStaticPolicyTestCaseWithFeatureGate(t *testing.T, testCase staticPolicyT @@ -674,6 +681,8 @@ func runStaticPolicyTestCaseWithFeatureGate(t *testing.T, testCase staticPolicyT
} }
func TestStaticPolicyReuseCPUs(t *testing.T) { func TestStaticPolicyReuseCPUs(t *testing.T) {
@ -498,7 +614,7 @@ index b864c6c57c6..cb363bb29ab 100644
testCases := []struct { testCases := []struct {
staticPolicyTest staticPolicyTest
expCSetAfterAlloc cpuset.CPUSet expCSetAfterAlloc cpuset.CPUSet
@@ -698,7 +704,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) { @@ -698,7 +707,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) {
} }
for _, testCase := range testCases { for _, testCase := range testCases {
@ -507,7 +623,7 @@ index b864c6c57c6..cb363bb29ab 100644
st := &mockState{ st := &mockState{
assignments: testCase.stAssignments, assignments: testCase.stAssignments,
@@ -731,6 +737,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) { @@ -731,6 +740,7 @@ func TestStaticPolicyReuseCPUs(t *testing.T) {
func TestStaticPolicyRemove(t *testing.T) { func TestStaticPolicyRemove(t *testing.T) {
excludeReserved := false excludeReserved := false
@ -515,7 +631,7 @@ index b864c6c57c6..cb363bb29ab 100644
testCases := []staticPolicyTest{ testCases := []staticPolicyTest{
{ {
description: "SingleSocketHT, DeAllocOneContainer", description: "SingleSocketHT, DeAllocOneContainer",
@@ -789,7 +796,7 @@ func TestStaticPolicyRemove(t *testing.T) { @@ -789,7 +799,7 @@ func TestStaticPolicyRemove(t *testing.T) {
} }
for _, testCase := range testCases { for _, testCase := range testCases {
@ -524,7 +640,7 @@ index b864c6c57c6..cb363bb29ab 100644
st := &mockState{ st := &mockState{
assignments: testCase.stAssignments, assignments: testCase.stAssignments,
@@ -812,6 +819,7 @@ func TestStaticPolicyRemove(t *testing.T) { @@ -812,6 +822,7 @@ func TestStaticPolicyRemove(t *testing.T) {
func TestTopologyAwareAllocateCPUs(t *testing.T) { func TestTopologyAwareAllocateCPUs(t *testing.T) {
excludeReserved := false excludeReserved := false
@ -532,7 +648,7 @@ index b864c6c57c6..cb363bb29ab 100644
testCases := []struct { testCases := []struct {
description string description string
topo *topology.CPUTopology topo *topology.CPUTopology
@@ -880,7 +888,8 @@ func TestTopologyAwareAllocateCPUs(t *testing.T) { @@ -880,7 +891,8 @@ func TestTopologyAwareAllocateCPUs(t *testing.T) {
}, },
} }
for _, tc := range testCases { for _, tc := range testCases {
@ -542,7 +658,7 @@ index b864c6c57c6..cb363bb29ab 100644
policy := p.(*staticPolicy) policy := p.(*staticPolicy)
st := &mockState{ st := &mockState{
assignments: tc.stAssignments, assignments: tc.stAssignments,
@@ -913,6 +922,7 @@ type staticPolicyTestWithResvList struct { @@ -913,6 +925,7 @@ type staticPolicyTestWithResvList struct {
topo *topology.CPUTopology topo *topology.CPUTopology
numReservedCPUs int numReservedCPUs int
reserved cpuset.CPUSet reserved cpuset.CPUSet
@ -550,7 +666,7 @@ index b864c6c57c6..cb363bb29ab 100644
stAssignments state.ContainerCPUAssignments stAssignments state.ContainerCPUAssignments
stDefaultCPUSet cpuset.CPUSet stDefaultCPUSet cpuset.CPUSet
pod *v1.Pod pod *v1.Pod
@@ -923,6 +933,8 @@ type staticPolicyTestWithResvList struct { @@ -923,6 +936,8 @@ type staticPolicyTestWithResvList struct {
} }
func TestStaticPolicyStartWithResvList(t *testing.T) { func TestStaticPolicyStartWithResvList(t *testing.T) {
@ -559,7 +675,7 @@ index b864c6c57c6..cb363bb29ab 100644
testCases := []staticPolicyTestWithResvList{ testCases := []staticPolicyTestWithResvList{
{ {
description: "empty cpuset", description: "empty cpuset",
@@ -952,10 +964,9 @@ func TestStaticPolicyStartWithResvList(t *testing.T) { @@ -952,10 +967,9 @@ func TestStaticPolicyStartWithResvList(t *testing.T) {
expNewErr: fmt.Errorf("[cpumanager] unable to reserve the required amount of CPUs (size of 0-1 did not equal 1)"), expNewErr: fmt.Errorf("[cpumanager] unable to reserve the required amount of CPUs (size of 0-1 did not equal 1)"),
}, },
} }
@ -571,7 +687,7 @@ index b864c6c57c6..cb363bb29ab 100644
if !reflect.DeepEqual(err, testCase.expNewErr) { if !reflect.DeepEqual(err, testCase.expNewErr) {
t.Errorf("StaticPolicy Start() error (%v). expected error: %v but got: %v", t.Errorf("StaticPolicy Start() error (%v). expected error: %v but got: %v",
@@ -996,6 +1007,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { @@ -996,6 +1010,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
topo: topoSingleSocketHT, topo: topoSingleSocketHT,
numReservedCPUs: 1, numReservedCPUs: 1,
reserved: cpuset.New(0), reserved: cpuset.New(0),
@ -579,7 +695,7 @@ index b864c6c57c6..cb363bb29ab 100644
stAssignments: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{},
stDefaultCPUSet: cpuset.New(1, 2, 3, 4, 5, 6, 7), stDefaultCPUSet: cpuset.New(1, 2, 3, 4, 5, 6, 7),
pod: makePod("fakePod", "fakeContainer2", "8000m", "8000m"), pod: makePod("fakePod", "fakeContainer2", "8000m", "8000m"),
@@ -1008,6 +1020,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { @@ -1008,6 +1023,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
topo: topoSingleSocketHT, topo: topoSingleSocketHT,
numReservedCPUs: 2, numReservedCPUs: 2,
reserved: cpuset.New(0, 1), reserved: cpuset.New(0, 1),
@ -587,7 +703,7 @@ index b864c6c57c6..cb363bb29ab 100644
stAssignments: state.ContainerCPUAssignments{}, stAssignments: state.ContainerCPUAssignments{},
stDefaultCPUSet: cpuset.New(2, 3, 4, 5, 6, 7), stDefaultCPUSet: cpuset.New(2, 3, 4, 5, 6, 7),
pod: makePod("fakePod", "fakeContainer2", "1000m", "1000m"), pod: makePod("fakePod", "fakeContainer2", "1000m", "1000m"),
@@ -1020,6 +1033,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { @@ -1020,6 +1036,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
topo: topoSingleSocketHT, topo: topoSingleSocketHT,
numReservedCPUs: 2, numReservedCPUs: 2,
reserved: cpuset.New(0, 1), reserved: cpuset.New(0, 1),
@ -595,7 +711,7 @@ index b864c6c57c6..cb363bb29ab 100644
stAssignments: state.ContainerCPUAssignments{ stAssignments: state.ContainerCPUAssignments{
"fakePod": map[string]cpuset.CPUSet{ "fakePod": map[string]cpuset.CPUSet{
"fakeContainer100": cpuset.New(2, 3, 6, 7), "fakeContainer100": cpuset.New(2, 3, 6, 7),
@@ -1036,6 +1050,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { @@ -1036,6 +1053,7 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
topo: topoSingleSocketHT, topo: topoSingleSocketHT,
numReservedCPUs: 2, numReservedCPUs: 2,
reserved: cpuset.New(0, 1), reserved: cpuset.New(0, 1),
@ -603,7 +719,7 @@ index b864c6c57c6..cb363bb29ab 100644
stAssignments: state.ContainerCPUAssignments{ stAssignments: state.ContainerCPUAssignments{
"fakePod": map[string]cpuset.CPUSet{ "fakePod": map[string]cpuset.CPUSet{
"fakeContainer100": cpuset.New(2, 3, 6, 7), "fakeContainer100": cpuset.New(2, 3, 6, 7),
@@ -1047,11 +1062,29 @@ func TestStaticPolicyAddWithResvList(t *testing.T) { @@ -1047,11 +1065,29 @@ func TestStaticPolicyAddWithResvList(t *testing.T) {
expCPUAlloc: true, expCPUAlloc: true,
expCSet: cpuset.New(0, 1), expCSet: cpuset.New(0, 1),
}, },
@ -634,6 +750,132 @@ index b864c6c57c6..cb363bb29ab 100644
st := &mockState{ st := &mockState{
assignments: testCase.stAssignments, assignments: testCase.stAssignments,
@@ -1182,3 +1218,125 @@ func newCPUSetPtr(cpus ...int) *cpuset.CPUSet {
ret := cpuset.New(cpus...)
return &ret
}
+
+func makePodWithLabels(podLabels map[string]string) *v1.Pod {
+ return &v1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-pod",
+ Namespace: "test-namespace",
+ Labels: podLabels,
+ },
+ }
+}
+
+func fakeBuildConfigFromFlags(masterUrl string, kubeconfigPath string) (*restclient.Config, error) {
+
+ return &restclient.Config{}, nil
+}
+
+func fakeBuildConfigFromFlagsError(masterUrl string, kubeconfigPath string) (*restclient.Config, error) {
+
+ errString := fmt.Sprintf("%s file not found", kubeconfigPath)
+ return nil, errors.New(errString)
+
+}
+
+func getFakeInfraPodNamespace(_ string) (*v1.Namespace, error) {
+
+ return &v1.Namespace{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-namespace",
+ Labels: map[string]string{
+ "app.starlingx.io/component": "platform",
+ },
+ }}, nil
+}
+
+func getFakeNonInfraPodNamespace(_ string) (*v1.Namespace, error) {
+
+ return &v1.Namespace{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-namespace",
+ Labels: map[string]string{
+ "fake": "label",
+ }}}, nil
+
+}
+
+type kubeInfraPodTestCase struct {
+ description string
+ pod *v1.Pod
+ namespaceFunc getPodNamespace
+ expectedValue bool
+}
+
+func TestKubeInfraPod(t *testing.T) {
+ testCases := []kubeInfraPodTestCase{
+ {
+ description: "Pod with platform label and namespace without platform label",
+ pod: makePodWithLabels(map[string]string{
+ "app.starlingx.io/component": "platform",
+ }),
+ namespaceFunc: getFakeNonInfraPodNamespace,
+ expectedValue: true,
+
+ },
+ {
+ description: "Pod without platform label and namespace with platform label",
+ pod: makePodWithLabels(map[string]string{
+ "test": "label",
+ }),
+ namespaceFunc: getFakeInfraPodNamespace,
+ expectedValue: true,
+ },
+ {
+ description: "Pod without platform label and namespace without platform label",
+ pod: makePodWithLabels(map[string]string{
+ "test": "namespace",
+ }),
+ namespaceFunc: getFakeNonInfraPodNamespace,
+ expectedValue: false,
+ },
+
+ }
+
+ for _, testCase := range testCases {
+ t.Run(testCase.description, func(t *testing.T) {
+
+ varGetNamespaceObject = testCase.namespaceFunc
+ varBuildConfigFromFlags = fakeBuildConfigFromFlags
+ gotValue := isKubeInfra(testCase.pod)
+
+ if gotValue != testCase.expectedValue {
+ t.Errorf("StaticPolicy isKubeInfraPod() error %v. expected value %v actual value %v",
+ testCase.description, testCase.expectedValue, gotValue)
+ } else {
+ fmt.Printf("StaticPolicy isKubeInfraPod() test successful. : %v ", testCase.description)
+ }
+
+ })
+ }
+
+ test := kubeInfraPodTestCase{
+ description: "Failure reading kubeconfig file",
+ pod: makePodWithLabels(map[string]string{
+ "test": "namespace",
+ }),
+ namespaceFunc: getFakeNonInfraPodNamespace,
+ expectedValue: false,
+ }
+
+ varGetNamespaceObject = getPodNamespaceObject
+ varBuildConfigFromFlags = fakeBuildConfigFromFlagsError
+
+ gotValue := isKubeInfra(test.pod)
+
+ if gotValue != test.expectedValue {
+ t.Errorf("StaticPolicy isKubeInfraPod() error %v. expected value %v actual value %v",
+ test.description, test.expectedValue, gotValue)
+ } else {
+ fmt.Printf("StaticPolicy isKubeInfraPod() test successful. : %v ", test.description)
+ }
+
+}
+
diff --git a/pkg/kubelet/cm/devicemanager/manager_stub.go b/pkg/kubelet/cm/devicemanager/manager_stub.go diff --git a/pkg/kubelet/cm/devicemanager/manager_stub.go b/pkg/kubelet/cm/devicemanager/manager_stub.go
new file mode 100644 new file mode 100644
index 00000000000..98abcde2519 index 00000000000..98abcde2519