Skip to content

Commit 83b7f04

Browse files
marioferhmarioferh
authored
Fix default hard eviction threshold when PCC is applied (openshift#505)
After PCC is applied, hard evicition is different to the kubelet config default Before: memory.available<100Mi nodefs.available<10% imagefs.available<15% nodefs.inodesFree<5% After PCC: memory.available<100Mi After fix, hard eviction tresholds are default ones Signed-off-by: Mario Fernandez <mariofer@redhat.com> Signed-off-by: Mario Fernandez <mariofer@redhat.com>
1 parent f4bb04d commit 83b7f04

File tree

94 files changed

+33945
-10
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

94 files changed

+33945
-10
lines changed

‎go.mod

+3
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ require (
5656
github.com/coreos/go-systemd/v22 v22.3.2 // indirect
5757
github.com/coreos/vcontext v0.0.0-20201120045928-b0e13dab675c // indirect
5858
github.com/davecgh/go-spew v1.1.1 // indirect
59+
github.com/docker/distribution v2.8.1+incompatible // indirect
5960
github.com/emicklei/go-restful v2.10.0+incompatible // indirect
6061
github.com/evanphx/json-patch v4.12.0+incompatible // indirect
6162
github.com/fatih/color v1.12.0 // indirect
@@ -96,6 +97,7 @@ require (
9697
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
9798
github.com/nxadm/tail v1.4.8 // indirect
9899
github.com/oklog/ulid v1.3.1 // indirect
100+
github.com/opencontainers/go-digest v1.0.0 // indirect
99101
github.com/prometheus/client_model v0.2.0 // indirect
100102
github.com/prometheus/common v0.32.1 // indirect
101103
github.com/prometheus/procfs v0.7.3 // indirect
@@ -118,6 +120,7 @@ require (
118120
gopkg.in/yaml.v2 v2.4.0 // indirect
119121
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
120122
howett.net/plist v0.0.0-20181124034731-591f970eefbb // indirect
123+
k8s.io/apiserver v0.24.2 // indirect
121124
k8s.io/component-base v0.24.2 // indirect
122125
k8s.io/gengo v0.0.0-20211129171323-c02415ce4185 // indirect
123126
k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42 // indirect

‎go.sum

+3
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,7 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm
187187
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
188188
github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
189189
github.com/docker/distribution v2.8.0+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
190+
github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68=
190191
github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
191192
github.com/docker/docker v20.10.12+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
192193
github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
@@ -602,6 +603,7 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y
602603
github.com/onsi/gomega v1.14.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0=
603604
github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE=
604605
github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
606+
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
605607
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
606608
github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
607609
github.com/opencontainers/runc v1.1.0/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc=
@@ -1344,6 +1346,7 @@ k8s.io/apiextensions-apiserver v0.24.2 h1:/4NEQHKlEz1MlaK/wHT5KMKC9UKYz6NZz6JE6o
13441346
k8s.io/apiextensions-apiserver v0.24.2/go.mod h1:e5t2GMFVngUEHUd0wuCJzw8YDwZoqZfJiGOW6mm2hLQ=
13451347
k8s.io/apimachinery v0.24.2 h1:5QlH9SL2C8KMcrNJPor+LbXVTaZRReml7svPEh4OKDM=
13461348
k8s.io/apimachinery v0.24.2/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM=
1349+
k8s.io/apiserver v0.24.2 h1:orxipm5elPJSkkFNlwH9ClqaKEDJJA3yR2cAAlCnyj4=
13471350
k8s.io/apiserver v0.24.2/go.mod h1:pSuKzr3zV+L+MWqsEo0kHHYwCo77AT5qXbFXP2jbvFI=
13481351
k8s.io/cli-runtime v0.24.2/go.mod h1:1LIhKL2RblkhfG4v5lZEt7FtgFG5mVb8wqv5lE9m5qY=
13491352
k8s.io/client-go v0.24.2 h1:CoXFSf8if+bLEbinDqN9ePIDGzcLtqhfd6jpfnwGOFA=

‎pkg/performanceprofile/controller/performanceprofile/components/kubeletconfig/kubeletconfig.go

+24-9
Original file line numberDiff line numberDiff line change
@@ -26,14 +26,20 @@ const (
2626
// 4. Reserved CPUs
2727
// 5. Memory manager policy
2828
// Please avoid specifying them and use the relevant API to configure these parameters.
29-
experimentalKubeletSnippetAnnotation = "kubeletconfig.experimental"
30-
cpuManagerPolicyStatic = "static"
31-
cpuManagerPolicyOptionFullPCPUsOnly = "full-pcpus-only"
32-
memoryManagerPolicyStatic = "Static"
33-
defaultKubeReservedMemory = "500Mi"
34-
defaultSystemReservedMemory = "500Mi"
35-
defaultHardEvictionThreshold = "100Mi"
36-
evictionHardMemoryAvailable = "memory.available"
29+
experimentalKubeletSnippetAnnotation = "kubeletconfig.experimental"
30+
cpuManagerPolicyStatic = "static"
31+
cpuManagerPolicyOptionFullPCPUsOnly = "full-pcpus-only"
32+
memoryManagerPolicyStatic = "Static"
33+
defaultKubeReservedMemory = "500Mi"
34+
defaultSystemReservedMemory = "500Mi"
35+
defaultHardEvictionThresholdMemory = "100Mi"
36+
defaultHardEvictionThresholdNodefs = "10%"
37+
defaultHardEvictionThresholdImagefs = "15%"
38+
defaultHardEvictionThresholdNodefsInodesFree = "5%"
39+
evictionHardMemoryAvailable = "memory.available"
40+
evictionHardNodefsAvaialble = "nodefs.available"
41+
evictionHardImagefsAvailable = "imagefs.available"
42+
evictionHardNodefsInodesFree = "nodefs.inodesFree"
3743
)
3844

3945
// New returns new KubeletConfig object for performance sensetive workflows
@@ -60,7 +66,16 @@ func New(profile *performancev2.PerformanceProfile, profileMCPLabels map[string]
6066
kubeletConfig.EvictionHard = map[string]string{}
6167
}
6268
if _, ok := kubeletConfig.EvictionHard[evictionHardMemoryAvailable]; !ok {
63-
kubeletConfig.EvictionHard[evictionHardMemoryAvailable] = defaultHardEvictionThreshold
69+
kubeletConfig.EvictionHard[evictionHardMemoryAvailable] = defaultHardEvictionThresholdMemory
70+
}
71+
if _, ok := kubeletConfig.EvictionHard[evictionHardNodefsAvaialble]; !ok {
72+
kubeletConfig.EvictionHard[evictionHardNodefsAvaialble] = defaultHardEvictionThresholdNodefs
73+
}
74+
if _, ok := kubeletConfig.EvictionHard[evictionHardImagefsAvailable]; !ok {
75+
kubeletConfig.EvictionHard[evictionHardImagefsAvailable] = defaultHardEvictionThresholdImagefs
76+
}
77+
if _, ok := kubeletConfig.EvictionHard[evictionHardNodefsInodesFree]; !ok {
78+
kubeletConfig.EvictionHard[evictionHardNodefsInodesFree] = defaultHardEvictionThresholdNodefsInodesFree
6479
}
6580

6681
// set the default memory kube-reserved

‎pkg/performanceprofile/controller/performanceprofile/components/kubeletconfig/kubeletconfig_test.go

+26-1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
. "github.com/onsi/ginkgo"
88
. "github.com/onsi/gomega"
99
kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1"
10+
kubeletv1beta1 "k8s.io/kubernetes/pkg/kubelet/apis/config/v1beta1"
1011
"k8s.io/utils/pointer"
1112

1213
"github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/components"
@@ -138,7 +139,8 @@ var _ = Describe("Kubelet Config", func() {
138139
It("should set the kubelet config accordingly", func() {
139140
profile := testutils.NewPerformanceProfile("test")
140141
profile.Annotations = map[string]string{
141-
experimentalKubeletSnippetAnnotation: `{"allowedUnsafeSysctls": ["net.core.somaxconn"], "evictionHard": {"memory.available": "200Mi"}}`,
142+
experimentalKubeletSnippetAnnotation: `{"allowedUnsafeSysctls": ["net.core.somaxconn"], "evictionHard": {"memory.available": "200Mi",
143+
"nodefs.available": "20%", "imagefs.available": "25%", "nodefs.inodesFree": "10%"}}`,
142144
}
143145
selectorKey, selectorValue := components.GetFirstKeyAndValue(profile.Spec.MachineConfigPoolSelector)
144146
kc, err := New(profile, map[string]string{selectorKey: selectorValue})
@@ -148,6 +150,29 @@ var _ = Describe("Kubelet Config", func() {
148150
manifest := string(y)
149151
Expect(manifest).To(ContainSubstring("net.core.somaxconn"))
150152
Expect(manifest).To(ContainSubstring("memory.available: 200Mi"))
153+
Expect(manifest).To(ContainSubstring("nodefs.available: 20%"))
154+
Expect(manifest).To(ContainSubstring("imagefs.available: 25%"))
155+
Expect(manifest).To(ContainSubstring("nodefs.inodesFree: 10%"))
156+
})
157+
158+
It("should set the default kubelet config", func() {
159+
profile := testutils.NewPerformanceProfile("test")
160+
selectorKey, selectorValue := components.GetFirstKeyAndValue(profile.Spec.MachineConfigPoolSelector)
161+
kc, err := New(profile, map[string]string{selectorKey: selectorValue})
162+
y, err := yaml.Marshal(kc)
163+
Expect(err).ToNot(HaveOccurred())
164+
165+
manifest := string(y)
166+
167+
memoryAvaialable := "memory.available: " + string(kubeletv1beta1.DefaultEvictionHard[evictionHardMemoryAvailable])
168+
nodefsAvailable := "nodefs.available: " + string(kubeletv1beta1.DefaultEvictionHard[evictionHardNodefsAvaialble])
169+
imagefsAvailable := "imagefs.available: " + string(kubeletv1beta1.DefaultEvictionHard[evictionHardImagefsAvailable])
170+
nodefsInodesFree := "nodefs.inodesFree: " + string(kubeletv1beta1.DefaultEvictionHard[evictionHardNodefsInodesFree])
171+
172+
Expect(manifest).To(ContainSubstring(memoryAvaialable))
173+
Expect(manifest).To(ContainSubstring(nodefsAvailable))
174+
Expect(manifest).To(ContainSubstring(imagefsAvailable))
175+
Expect(manifest).To(ContainSubstring(nodefsInodesFree))
151176
})
152177

153178
It("should allow to override the cpumanager policy options and update the kubelet config accordingly", func() {

‎test/e2e/performanceprofile/testdata/must-gather/must-gather.e2e/quay-io-openshift-kni-performance-addon-operator-must-gather-sha256-e0807f2ff1ff1754cdb7475e665c6c60fb1faa1f14eb0f2f7a791388ac8b33e8/cluster-scoped-resources/machineconfiguration.openshift.io/kubeletconfigs/performance-performance.yaml

+3
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,10 @@ spec:
132132
full-pcpus-only: "true"
133133
cpuManagerReconcilePeriod: 5s
134134
evictionHard:
135+
nodefs.available: 15%
135136
memory.available: 100Mi
137+
imagefs.available: 10%
138+
nodefs.inodesFree: 5%
136139
evictionPressureTransitionPeriod: 0s
137140
fileCheckFrequency: 0s
138141
httpCheckFrequency: 0s

‎test/e2e/performanceprofile/testdata/render-expected-output/manual_kubeletconfig.yaml

+3
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,9 @@ spec:
2626
cpuManagerReconcilePeriod: 5s
2727
evictionHard:
2828
memory.available: 100Mi
29+
imagefs.available: 15%
30+
nodefs.available: 10%
31+
nodefs.inodesFree: 5%
2932
evictionPressureTransitionPeriod: 0s
3033
fileCheckFrequency: 0s
3134
httpCheckFrequency: 0s

0 commit comments

Comments
 (0)