Add performanceProfile API offline cores (openshift#345)

Signed-off-by: Mario Fernandez <mariofer@redhat.com>

Author: marioferh

assets/performanceprofile/scripts/set-cpus-offline.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/bash
+
+set -euo pipefail
+
+for cpu in ${OFFLINE_CPUS//,/ };
+  do
+    online_cpu_file="/sys/devices/system/cpu/cpu$cpu/online"
+    if [ ! -f "${online_cpu_file}" ]; then
+      echo "ERROR: ${online_cpu_file} does not exist, abort script execution"
+      exit 1
+    fi
+  done
+
+echo "All cpus offlined exists, set them offline"
+
+for cpu in ${OFFLINE_CPUS//,/ };
+  do
+    online_cpu_file="/sys/devices/system/cpu/cpu$cpu/online"
+    echo 0 > "${online_cpu_file}"
+    echo "offline cpu num $cpu"
+  done
+

docs/performanceprofile/performance_profile.md

@@ -31,6 +31,7 @@ CPU defines a set of CPU related features.
 | reserved | Reserved defines a set of CPUs that will not be used for any container workloads initiated by kubelet. | *[CPUSet](#cpuset) | true |
 | isolated | Isolated defines a set of CPUs that will be used to give to application threads the most execution time possible, which means removing as many extraneous tasks off a CPU as possible. It is important to notice the CPU manager can choose any CPU to run the workload except the reserved CPUs. In order to guarantee that your workload will run on the isolated CPU:\n  1. The union of reserved CPUs and isolated CPUs should include all online CPUs\n  2. The isolated CPUs field should be the complementary to reserved CPUs field | *[CPUSet](#cpuset) | true |
 | balanceIsolated | BalanceIsolated toggles whether or not the Isolated CPU set is eligible for load balancing work loads. When this option is set to \"false\", the Isolated CPU set will be static, meaning workloads have to explicitly assign each thread to a specific cpu in order to work across multiple CPUs. Setting this to \"true\" allows workloads to be balanced across CPUs. Setting this to \"false\" offers the most predictable performance for guaranteed workloads, but it offloads the complexity of cpu load balancing to the application. Defaults to \"true\" | *bool | false |
+| offlined | Offline defines a set of CPUs that will be unused and set offline | *[CPUSet](#cpuset) | true |
 
 [Back to TOC](#table-of-contents)
 

examples/performanceprofile/samples/performance_v2_performanceprofile.yaml

@@ -11,8 +11,9 @@ spec:
     - "idle=poll"
     - "intel_idle.max_cstate=0"
   cpu:
-    isolated: "2-3"
-    reserved: "0-1"
+      isolated: "2"
+      reserved: "0-1"
+      offlined: "3"
   hugepages:
     defaultHugepagesSize: "1G"
     pages:

manifests/20-performance-profile.crd.yaml

@@ -370,6 +370,9 @@ spec:
                     isolated:
                       description: 'Isolated defines a set of CPUs that will be used to give to application threads the most execution time possible, which means removing as many extraneous tasks off a CPU as possible. It is important to notice the CPU manager can choose any CPU to run the workload except the reserved CPUs. In order to guarantee that your workload will run on the isolated CPU:   1. The union of reserved CPUs and isolated CPUs should include all online CPUs   2. The isolated CPUs field should be the complementary to reserved CPUs field'
                       type: string
+                    offlined:
+                      description: Offline defines a set of CPUs that will be unused and set offline
+                      type: string
                     reserved:
                       description: Reserved defines a set of CPUs that will not be used for any container workloads initiated by kubelet.
                       type: string

pkg/apis/performanceprofile/v2/performanceprofile_types.go

@@ -98,6 +98,9 @@ type CPU struct {
 	// Defaults to "true"
 	// +optional
 	BalanceIsolated *bool `json:"balanceIsolated,omitempty"`
+	// Offline defines a set of CPUs that will be unused and set offline
+	// +optional
+	Offlined *CPUSet `json:"offlined,omitempty"`
 }
 
 // HugePageSize defines size of huge pages, can be 2M or 1G.

pkg/apis/performanceprofile/v2/performanceprofile_validation.go

@@ -127,7 +127,12 @@ func (r *PerformanceProfile) validateCPUs() field.ErrorList {
 		}
 
 		if r.Spec.CPU.Isolated != nil && r.Spec.CPU.Reserved != nil {
-			cpuLists, err := components.NewCPULists(string(*r.Spec.CPU.Reserved), string(*r.Spec.CPU.Isolated))
+			var offlined string
+			if r.Spec.CPU.Offlined != nil {
+				offlined = string(*r.Spec.CPU.Offlined)
+			}
+
+			cpuLists, err := components.NewCPULists(string(*r.Spec.CPU.Reserved), string(*r.Spec.CPU.Isolated), offlined)
 			if err != nil {
 				allErrs = append(allErrs, field.InternalError(field.NewPath("spec.cpu"), err))
 			}
@@ -141,10 +146,16 @@ func (r *PerformanceProfile) validateCPUs() field.ErrorList {
 					allErrs = append(allErrs, field.Invalid(field.NewPath("spec.cpu.isolated"), r.Spec.CPU.Isolated, "isolated CPUs can not be empty"))
 				}
 
-				if overlap := cpuLists.Intersect(); len(overlap) != 0 {
+				if overlap := components.Intersect(cpuLists.GetIsolated(), cpuLists.GetReserved()); len(overlap) != 0 {
 					allErrs = append(allErrs, field.Invalid(field.NewPath("spec.cpu"), r.Spec.CPU, fmt.Sprintf("reserved and isolated cpus overlap: %v", overlap)))
 				}
 			}
+
+			if r.Spec.CPU.Offlined != nil {
+				if overlap := components.Intersect(cpuLists.GetReserved(), cpuLists.GetOfflined()); len(overlap) != 0 {
+					allErrs = append(allErrs, field.Invalid(field.NewPath("spec.cpu"), r.Spec.CPU, fmt.Sprintf("reserved and offlined cpus overlap: %v", overlap)))
+				}
+			}
 		}
 	}
 

pkg/apis/performanceprofile/v2/performanceprofile_validation_test.go

@@ -20,9 +20,11 @@ const (
 	// HugePagesCount defines the huge page count used for tests
 	HugePagesCount = 4
 	// IsolatedCPUs defines the isolated CPU set used for tests
-	IsolatedCPUs = CPUSet("4-7")
+	IsolatedCPUs = CPUSet("4-6")
 	// ReservedCPUs defines the reserved CPU set used for tests
 	ReservedCPUs = CPUSet("0-3")
+	// ReservedCPUs defines the reserved CPU set used for tests
+	OfflinedCPUs = CPUSet("7")
 	// SingleNUMAPolicy defines the topologyManager policy used for tests
 	SingleNUMAPolicy = "single-numa-node"
 
@@ -48,6 +50,7 @@ func NewPerformanceProfile(name string) *PerformanceProfile {
 	size := HugePageSize1G
 	isolatedCPUs := IsolatedCPUs
 	reservedCPUs := ReservedCPUs
+	offlinedCPUs := OfflinedCPUs
 	numaPolicy := SingleNUMAPolicy
 
 	netDeviceName := NetDeviceName
@@ -64,6 +67,7 @@ func NewPerformanceProfile(name string) *PerformanceProfile {
 			CPU: &CPU{
 				Isolated: &isolatedCPUs,
 				Reserved: &reservedCPUs,
+				Offlined: &offlinedCPUs,
 			},
 			HugePages: &HugePages{
 				DefaultHugePagesSize: &size,
@@ -140,9 +144,11 @@ var _ = Describe("PerformanceProfile", func() {
 
 		It("should reject cpus allocation with no reserved CPUs", func() {
 			reservedCPUs := CPUSet("")
-			isolatedCPUs := CPUSet("0-7")
-			profile.Spec.CPU.Isolated = &isolatedCPUs
+			isolatedCPUs := CPUSet("0-6")
+			offlinedCPUs := CPUSet("7")
 			profile.Spec.CPU.Reserved = &reservedCPUs
+			profile.Spec.CPU.Isolated = &isolatedCPUs
+			profile.Spec.CPU.Offlined = &offlinedCPUs
 			errors := profile.validateCPUs()
 			Expect(errors[0].Error()).To(ContainSubstring("reserved CPUs can not be empty"))
 		})
@@ -157,7 +163,17 @@ var _ = Describe("PerformanceProfile", func() {
 			Expect(errors[0].Error()).To(ContainSubstring("isolated CPUs can not be empty"))
 		})
 
-		It("should reject cpus allocation with overlapping sets", func() {
+		It("should allow cpus allocation with no offlined CPUs", func() {
+			cpusIsolaled := CPUSet("0")
+			cpusReserved := CPUSet("1")
+			profile.Spec.CPU.Isolated = &cpusIsolaled
+			profile.Spec.CPU.Reserved = &cpusReserved
+			profile.Spec.CPU.Offlined = nil
+			errors := profile.validateCPUs()
+			Expect(errors).To(BeEmpty())
+		})
+
+		It("should reject cpus allocation with overlapping sets between reserved and isolated", func() {
 			reservedCPUs := CPUSet("0-7")
 			isolatedCPUs := CPUSet("0-15")
 			profile.Spec.CPU.Reserved = &reservedCPUs
@@ -166,6 +182,18 @@ var _ = Describe("PerformanceProfile", func() {
 			Expect(errors).NotTo(BeEmpty(), "should have validation error when reserved and isolation CPUs have overlap")
 			Expect(errors[0].Error()).To(ContainSubstring("reserved and isolated cpus overlap"))
 		})
+
+		It("should reject cpus allocation with overlapping sets between reserved and offlined", func() {
+			reservedCPUs := CPUSet("0-7")
+			isolatedCPUs := CPUSet("8-11")
+			offlinedCPUs := CPUSet("0,12-15")
+			profile.Spec.CPU.Reserved = &reservedCPUs
+			profile.Spec.CPU.Isolated = &isolatedCPUs
+			profile.Spec.CPU.Offlined = &offlinedCPUs
+			errors := profile.validateCPUs()
+			Expect(errors).NotTo(BeEmpty(), "should have validation error when reserved and isolation CPUs have overlap")
+			Expect(errors[0].Error()).To(ContainSubstring("reserved and offlined cpus overlap"))
+		})
 	})
 
 	Describe("Label selectors validation", func() {

pkg/apis/performanceprofile/v2/zz_generated.deepcopy.go

@@ -15,6 +15,7 @@ import (
 	igntypes "github.com/coreos/ignition/v2/config/v3_2/types"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
 	"k8s.io/utils/pointer"
 
 	performancev2 "github.com/openshift/cluster-node-tuning-operator/pkg/apis/performanceprofile/v2"
@@ -48,6 +49,7 @@ const (
 	udevRpsRules       = "99-netdev-rps.rules"
 	// scripts
 	hugepagesAllocation = "hugepages-allocation"
+	setCPUsOffline      = "set-cpus-offline"
 	ociHooks            = "low-latency-hooks"
 	setRPSMask          = "set-rps-mask"
 )
@@ -76,6 +78,7 @@ const (
 	environmentHugepagesSize  = "HUGEPAGES_SIZE"
 	environmentHugepagesCount = "HUGEPAGES_COUNT"
 	environmentNUMANode       = "NUMA_NODE"
+	environmentOfflineCpus    = "OFFLINE_CPUS"
 )
 
 const (
@@ -139,7 +142,7 @@ func getIgnitionConfig(profile *performancev2.PerformanceProfile) (*igntypes.Con
 
 	// add script files under the node /usr/local/bin directory
 	mode := 0700
-	for _, script := range []string{hugepagesAllocation, ociHooks, setRPSMask} {
+	for _, script := range []string{hugepagesAllocation, ociHooks, setRPSMask, setCPUsOffline} {
 		dst := getBashScriptPath(script)
 		content, err := assets.Scripts.ReadFile(fmt.Sprintf("scripts/%s.sh", script))
 		if err != nil {
@@ -221,6 +224,24 @@ func getIgnitionConfig(profile *performancev2.PerformanceProfile) (*igntypes.Con
 		})
 	}
 
+	if profile.Spec.CPU.Offlined != nil {
+		offlinedCPUSList, err := cpuset.Parse(string(*profile.Spec.CPU.Offlined))
+		if err != nil {
+			return nil, err
+		}
+		offlinedCPUSstring := components.ListToString(offlinedCPUSList.ToSlice())
+		offlineCPUsService, err := getSystemdContent(getOfflineCPUs(offlinedCPUSstring))
+		if err != nil {
+			return nil, err
+		}
+
+		ignitionConfig.Systemd.Units = append(ignitionConfig.Systemd.Units, igntypes.Unit{
+			Contents: &offlineCPUsService,
+			Enabled:  pointer.BoolPtr(true),
+			Name:     getSystemdService(setCPUsOffline),
+		})
+	}
+
 	return ignitionConfig, nil
 }
 
@@ -305,6 +326,27 @@ func getHugepagesAllocationUnitOptions(hugepagesSize string, hugepagesCount int3
 	}
 }
 
+func getOfflineCPUs(offlineCpus string) []*unit.UnitOption {
+	return []*unit.UnitOption{
+		// [Unit]
+		// Description
+		unit.NewUnitOption(systemdSectionUnit, systemdDescription, fmt.Sprintf("Set cpus offline: %s", offlineCpus)),
+		// Before
+		unit.NewUnitOption(systemdSectionUnit, systemdBefore, systemdServiceKubelet),
+		// Environment
+		unit.NewUnitOption(systemdSectionService, systemdEnvironment, getSystemdEnvironment(environmentOfflineCpus, offlineCpus)),
+		// Type
+		unit.NewUnitOption(systemdSectionService, systemdType, systemdServiceTypeOneshot),
+		// RemainAfterExit
+		unit.NewUnitOption(systemdSectionService, systemdRemainAfterExit, systemdTrue),
+		// ExecStart
+		unit.NewUnitOption(systemdSectionService, systemdExecStart, getBashScriptPath(setCPUsOffline)),
+		// [Install]
+		// WantedBy
+		unit.NewUnitOption(systemdSectionInstall, systemdWantedBy, systemdTargetMultiUser),
+	}
+}
+
 func getRPSUnitOptions(rpsMask string) []*unit.UnitOption {
 	cmd := fmt.Sprintf("%s %%i %s", getBashScriptPath(setRPSMask), rpsMask)
 	return []*unit.UnitOption{

pkg/performanceprofile/controller/performanceprofile/components/machineconfig/machineconfig.go

@@ -33,6 +33,27 @@ const hugepagesAllocationService = `
         name: hugepages-allocation-1048576kB-NUMA0.service
 `
 
+const offlineCPUS = `
+      - contents: |
+          [Unit]
+          Description=Set cpus offline: 6,7
+          Before=kubelet.service
+
+          [Service]
+          Environment=OFFLINE_CPUS=6,7
+          Type=oneshot
+          RemainAfterExit=true
+          ExecStart=/usr/local/bin/set-cpus-offline.sh
+
+          [Install]
+          WantedBy=multi-user.target
+        enabled: true
+        name: set-cpus-offline.service
+`
+
+var CPUs = []int{1, 2, 3, 4, 5, 6, 7, 8, 9}
+var CPUstring = "1,2,3,4,5,6,7,8,9"
+
 var _ = Describe("Machine Config", func() {
 
 	Context("machine config creation ", func() {
@@ -45,7 +66,7 @@ var _ = Describe("Machine Config", func() {
 		})
 	})
 
-	Context("with hugepages with specified NUMA node", func() {
+	Context("with hugepages with specified NUMA node and offlinedCPUs", func() {
 		var manifest string
 
 		BeforeEach(func() {
@@ -73,5 +94,15 @@ var _ = Describe("Machine Config", func() {
 			Expect(manifest).To(ContainSubstring(hugepagesAllocationService))
 		})
 
+		It("should add systemd unit to offlineCPUs", func() {
+			Expect(manifest).To(ContainSubstring(offlineCPUS))
+		})
+	})
+
+	Context("check listToString ", func() {
+		It("should create string from CPUSet", func() {
+			res := components.ListToString(CPUs)
+			Expect(res).To(Equal(CPUstring))
+		})
 	})
 })

pkg/performanceprofile/controller/performanceprofile/components/machineconfig/machineconfig_test.go

@@ -56,7 +56,7 @@ var _ = Describe("Tuned", func() {
 			manifest := getTunedManifest(profile)
 
 			Expect(manifest).To(ContainSubstring(expectedMatchSelector))
-			Expect(manifest).To(ContainSubstring("isolated_cores=4-7"))
+			Expect(manifest).To(ContainSubstring("isolated_cores=4-5"))
 			Expect(manifest).To(ContainSubstring("governor=performance"))
 			Expect(manifest).To(ContainSubstring("service.stalld=start,enable"))
 			Expect(manifest).To(ContainSubstring("sched_rt_runtime_us=-1"))

pkg/performanceprofile/controller/performanceprofile/components/tuned/tuned_test.go

@@ -79,11 +79,12 @@ func CPUListToMaskList(cpulist string) (hexMask string, err error) {
 type CPULists struct {
 	reserved cpuset.CPUSet
 	isolated cpuset.CPUSet
+	offlined cpuset.CPUSet
 }
 
 // Intersect returns cpu ids found in both the provided cpuLists, if any
-func (c *CPULists) Intersect() []int {
-	commonSet := c.reserved.Intersection(c.isolated)
+func Intersect(firstSet cpuset.CPUSet, secondSet cpuset.CPUSet) []int {
+	commonSet := firstSet.Intersection(secondSet)
 	return commonSet.ToSlice()
 }
 
@@ -95,8 +96,12 @@ func (c *CPULists) GetReserved() cpuset.CPUSet {
 	return c.reserved
 }
 
+func (c *CPULists) GetOfflined() cpuset.CPUSet {
+	return c.offlined
+}
+
 // NewCPULists parse text representations of reserved and isolated cpusets definiton and returns a CPULists object
-func NewCPULists(reservedList, isolatedList string) (*CPULists, error) {
+func NewCPULists(reservedList, isolatedList, offlinedList string) (*CPULists, error) {
 	var err error
 	reserved, err := cpuset.Parse(reservedList)
 	if err != nil {
@@ -106,9 +111,14 @@ func NewCPULists(reservedList, isolatedList string) (*CPULists, error) {
 	if err != nil {
 		return nil, err
 	}
+	offlined, err := cpuset.Parse(offlinedList)
+	if err != nil {
+		return nil, err
+	}
 	return &CPULists{
 		reserved: reserved,
 		isolated: isolated,
+		offlined: offlined,
 	}, nil
 }
 
@@ -142,3 +152,11 @@ func CPUMaskToCPUSet(cpuMask string) (cpuset.CPUSet, error) {
 
 	return builder.Result(), nil
 }
+
+func ListToString(cpus []int) string {
+	items := make([]string, len(cpus))
+	for idx, cpu := range cpus {
+		items[idx] = strconv.FormatInt(int64(cpu), 10)
+	}
+	return strings.Join(items, ",")
+}