refactor: Improve slurm collector organization

* Remove unnecessary job props and labels.

* Use slice of metrics to avoid using sync lock.

* Move regex patterns to a separate file to be used in different collectors.

* Update e2e test outputs and docs.

Signed-off-by: Mahendra Paipuri <mahendra.paipuri@gmail.com>

Author: mahendrapaipuri

Makefile

@@ -139,6 +139,7 @@ ifeq ($(CGO_BUILD), 0)
 test-e2e: build pkg/collector/testdata/sys/.unpacked pkg/collector/testdata/proc/.unpacked
 	@echo ">> running end-to-end tests"
 	./scripts/e2e-test.sh -s exporter-cgroups-v1
+	./scripts/e2e-test.sh -s exporter-cgroups-v1-memory-subsystem
 	./scripts/e2e-test.sh -s exporter-cgroups-v2-nvidia-ipmiutil
 	./scripts/e2e-test.sh -s exporter-cgroups-v2-amd-ipmitool
 	./scripts/e2e-test.sh -s exporter-cgroups-v2-nogpu
@@ -187,6 +188,7 @@ ifeq ($(CGO_BUILD), 0)
 test-e2e-update: build pkg/collector/testdata/sys/.unpacked pkg/collector/testdata/proc/.unpacked
 	@echo ">> updating end-to-end tests outputs"
 	./scripts/e2e-test.sh -s exporter-cgroups-v1 -u || true
+	./scripts/e2e-test.sh -s exporter-cgroups-v1-memory-subsystem -u || true
 	./scripts/e2e-test.sh -s exporter-cgroups-v2-nvidia-ipmiutil -u || true
 	./scripts/e2e-test.sh -s exporter-cgroups-v2-amd-ipmitool -u || true
 	./scripts/e2e-test.sh -s exporter-cgroups-v2-nogpu -u || true

etc/slurm/README.md

@@ -13,8 +13,8 @@ An example [systemd service file](https://github.com/mahendrapaipuri/ceems/blob/
 is also provided in the repo that can be used along with these prolog and epilog scripts.
 
 > [!IMPORTANT]
-> The CLI arguments `--collector.slurm.job-props-path` and `--collector.slurm.gpu-job-map-path`
-are hidden and cannot be seen in `ceems_exporter --help` output. However, these arguments
+> The CLI argument `--collector.slurm.gpu-job-map-path`
+is hidden and cannot be seen in `ceems_exporter --help` output. However, this argument
 exists in the exporter and can be used.
 
 Even with such prolog and epilog scripts, operators should grant the user running CEEMS

etc/slurm/epilog.d/slurmjobprops.sh

@@ -7,8 +7,6 @@ Type=simple
 User=ceems
 Group=ceems
 ExecStart=/usr/local/bin/ceems_exporter \
-    --collector.slurm.gpu.type=nvidia \
-    --collector.slurm.job.props.path="/run/slurmjobprops" \
     --collector.slurm.gpu.job.map.path="/run/gpujobmap" \
     --collector.ipmi.dcmi.cmd="sudo /usr/sbin/ipmi-dcmi --get-system-power-statistics" \
     --log.level=debug

etc/slurm/prolog.d/slurmjobprops.sh

@@ -25,7 +25,7 @@ var (
 	reParens        = regexp.MustCompile(`\((.*)\)`)
 )
 
-// Check if file exists.
+// fileExists checks if given file exists or not.
 func fileExists(filename string) bool {
 	info, err := os.Stat(filename)
 	if os.IsNotExist(err) {
@@ -35,17 +35,17 @@ func fileExists(filename string) bool {
 	return !info.IsDir()
 }
 
-// Find named matches in regex groups and return a map.
-func findNamedMatches(regex *regexp.Regexp, str string) map[string]string {
-	match := regex.FindStringSubmatch(str)
+// // Find named matches in regex groups and return a map.
+// func findNamedMatches(regex *regexp.Regexp, str string) map[string]string {
+// 	match := regex.FindStringSubmatch(str)
 
-	results := map[string]string{}
-	for i, name := range match {
-		results[regex.SubexpNames()[i]] = name
-	}
+// 	results := map[string]string{}
+// 	for i, name := range match {
+// 		results[regex.SubexpNames()[i]] = name
+// 	}
 
-	return results
-}
+// 	return results
+// }
 
 // SanitizeMetricName sanitize the given metric name by replacing invalid characters by underscores.
 //

init/systemd/ceems_exporter_no_privs.service

@@ -21,12 +21,6 @@ import (
 
 const perfCollectorSubsystem = "perf"
 
-// slurm related regexes.
-var (
-	slurmCgroupIDRegex    = regexp.MustCompile("^.*/(?:.+?)job_([0-9]+)(?:.*$)")
-	slurmIgnoreProcsRegex = regexp.MustCompile("slurmstepd:(.*)|sleep ([0-9]+)|/bin/bash (.*)/slurm_script")
-)
-
 var (
 	perfHardwareProfilerMap = map[string]perf.HardwareProfilerType{
 		"CpuCycles":    perf.CpuCyclesProfiler,
@@ -173,7 +167,7 @@ func NewPerfCollector(logger log.Logger) (Collector, error) {
 
 	if *collectorState[slurmCollectorSubsystem] {
 		collector.manager = "slurm"
-		collector.cgroupIDRegex = slurmCgroupIDRegex
+		collector.cgroupIDRegex = slurmCgroupPathRegex
 		collector.filterProcCmdRegex = slurmIgnoreProcsRegex
 	}
 

pkg/collector/helper.go

@@ -70,7 +70,7 @@ func TestDiscoverProcess(t *testing.T) {
 	collector := perfCollector{
 		logger:                    log.NewNopLogger(),
 		envVar:                    "ENABLE_PROFILING",
-		cgroupIDRegex:             slurmCgroupIDRegex,
+		cgroupIDRegex:             slurmCgroupPathRegex,
 		filterProcCmdRegex:        slurmIgnoreProcsRegex,
 		perfHwProfilersEnabled:    true,
 		perfSwProfilersEnabled:    true,
@@ -121,7 +121,7 @@ func TestNewProfilers(t *testing.T) {
 
 	collector := perfCollector{
 		logger:                    log.NewNopLogger(),
-		cgroupIDRegex:             slurmCgroupIDRegex,
+		cgroupIDRegex:             slurmCgroupPathRegex,
 		filterProcCmdRegex:        slurmIgnoreProcsRegex,
 		perfHwProfilersEnabled:    true,
 		perfSwProfilersEnabled:    true,

pkg/collector/perf.go

@@ -0,0 +1,17 @@
+package collector
+
+import "regexp"
+
+// Regular expressions of cgroup paths for different resource managers
+/*
+	For v1 possibilities are /cpuacct/slurm/uid_1000/job_211
+							 /memory/slurm/uid_1000/job_211
+
+	For v2 possibilities are /system.slice/slurmstepd.scope/job_211
+							/system.slice/slurmstepd.scope/job_211/step_interactive
+							/system.slice/slurmstepd.scope/job_211/step_extern/user/task_0
+*/
+var (
+	slurmCgroupPathRegex  = regexp.MustCompile("^.*/slurm(?:.*?)/job_([0-9]+)(?:.*$)")
+	slurmIgnoreProcsRegex = regexp.MustCompile("slurmstepd:(.*)|sleep ([0-9]+)|/bin/bash (.*)/slurm_script")
+)