Handle cgroups v2 in OsProbe (#77128) (#77306)

* Handle cgroups v2 in `OsProbe` (#77128)

Closes #76812. Closes #77126.

OsProbe was only capable of handle cgroup data in the v1 format.
However, Debian 11 uses cgroups v2 by default, and Elasticsearch isn't
capable of reporting any cgroup information. Therefore, add support for
the v2 layout.

Note that we have to open access to all of /sys/fs/cgroup because with
cgroups v2, the files we need are in an unpredictably location.

* Handle a max memory value of 'max' (#77289)

* Handle a max memory value of 'max'

* Update docs/changelog/77289.yaml

* Delete 77289.yaml

* Fixes to backport

* Fix

Author: pugnascotia

docs/changelog/77128.yaml

@@ -0,0 +1,7 @@
+pr: 77128
+summary: Handle cgroups v2 in `OsProbe`
+area: Infra/Core
+type: enhancement
+issues:
+ - 77126
+ - 76812

qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java

@@ -838,7 +838,6 @@ public void test131InitProcessHasCorrectPID() {
     /**
      * Check that Elasticsearch reports per-node cgroup information.
      */
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/76812")
     public void test140CgroupOsStatsAreAvailable() throws Exception {
         waitForElasticsearch(installation);
 

qa/os/src/test/java/org/elasticsearch/packaging/util/Packages.java

@@ -252,12 +252,18 @@ private static void verifyDefaultInstallation(Installation es, Distribution dist
     /**
      * Starts Elasticsearch, without checking that startup is successful.
      */
-    public static Shell.Result runElasticsearchStartCommand(Shell sh) throws IOException {
+    public static Shell.Result runElasticsearchStartCommand(Shell sh) {
         if (isSystemd()) {
+            Packages.JournaldWrapper journald = new Packages.JournaldWrapper(sh);
             sh.run("systemctl daemon-reload");
             sh.run("systemctl enable elasticsearch.service");
             sh.run("systemctl is-enabled elasticsearch.service");
-            return sh.runIgnoreExitCode("systemctl start elasticsearch.service");
+            Result exitCode = sh.runIgnoreExitCode("systemctl start elasticsearch.service");
+            if (exitCode.isSuccess() == false) {
+                logger.warn(sh.runIgnoreExitCode("systemctl status elasticsearch.service").stdout);
+                logger.warn(journald.getLogs().stdout);
+            }
+            return exitCode;
         }
         return sh.runIgnoreExitCode("service elasticsearch start");
     }

server/src/main/java/org/elasticsearch/monitor/os/OsProbe.java

@@ -144,14 +144,11 @@ grant {
   permission java.io.FilePermission "/proc/self/mountinfo", "read";
   permission java.io.FilePermission "/proc/diskstats", "read";
 
-  // control group stats on Linux
+  // control group stats on Linux. cgroup v2 stats are in an unpredicable
+  // location under `/sys/fs/cgroup`, so unfortunately we have to allow
+  // read access to the entire directory hierarchy.
   permission java.io.FilePermission "/proc/self/cgroup", "read";
-  permission java.io.FilePermission "/sys/fs/cgroup/cpu", "read";
-  permission java.io.FilePermission "/sys/fs/cgroup/cpu/-", "read";
-  permission java.io.FilePermission "/sys/fs/cgroup/cpuacct", "read";
-  permission java.io.FilePermission "/sys/fs/cgroup/cpuacct/-", "read";
-  permission java.io.FilePermission "/sys/fs/cgroup/memory", "read";
-  permission java.io.FilePermission "/sys/fs/cgroup/memory/-", "read";
+  permission java.io.FilePermission "/sys/fs/cgroup/-", "read";
 
   // system memory on Linux systems affected by JDK bug (#66629)
   permission java.io.FilePermission "/proc/meminfo", "read";

server/src/main/resources/org/elasticsearch/bootstrap/security.policy

@@ -43,7 +43,7 @@ public void testOsInfo() throws IOException {
         final OsProbe osProbe = new OsProbe() {
 
             @Override
-            List<String> readOsRelease() throws IOException {
+            List<String> readOsRelease() {
                 assert Constants.LINUX : Constants.OS_NAME;
                 if (prettyName != null) {
                     final String quote = randomFrom("\"", "'", "");
@@ -78,8 +78,10 @@ public void testOsStats() {
         OsStats stats = osProbe.osStats();
         assertNotNull(stats);
         assertThat(stats.getTimestamp(), greaterThan(0L));
-        assertThat(stats.getCpu().getPercent(), anyOf(equalTo((short) -1),
-                is(both(greaterThanOrEqualTo((short) 0)).and(lessThanOrEqualTo((short) 100)))));
+        assertThat(
+            stats.getCpu().getPercent(),
+            anyOf(equalTo((short) -1), is(both(greaterThanOrEqualTo((short) 0)).and(lessThanOrEqualTo((short) 100))))
+        );
         double[] loadAverage = stats.getCpu().getLoadAverage();
         if (loadAverage != null) {
             assertThat(loadAverage.length, equalTo(3));
@@ -141,8 +143,11 @@ public void testOsStats() {
                 assertThat(stats.getCgroup().getCpuStat().getTimeThrottledNanos(), greaterThanOrEqualTo(0L));
                 // These could be null if transported from a node running an older version, but shouldn't be null on the current node
                 assertThat(stats.getCgroup().getMemoryControlGroup(), notNullValue());
-                assertThat(stats.getCgroup().getMemoryLimitInBytes(), notNullValue());
-                assertThat(new BigInteger(stats.getCgroup().getMemoryLimitInBytes()), greaterThan(BigInteger.ZERO));
+                String memoryLimitInBytes = stats.getCgroup().getMemoryLimitInBytes();
+                assertThat(memoryLimitInBytes, notNullValue());
+                if (memoryLimitInBytes.equals("max") == false) {
+                    assertThat(new BigInteger(memoryLimitInBytes), greaterThan(BigInteger.ZERO));
+                }
                 assertThat(stats.getCgroup().getMemoryUsageInBytes(), notNullValue());
                 assertThat(new BigInteger(stats.getCgroup().getMemoryUsageInBytes()), greaterThan(BigInteger.ZERO));
             }
@@ -173,16 +178,14 @@ String readProcLoadavg() {
     }
 
     public void testCgroupProbe() {
-        assumeTrue("test runs on Linux only", Constants.LINUX);
-
-        final boolean areCgroupStatsAvailable = randomBoolean();
+        final int availableCgroupsVersion = randomFrom(0, 1, 2);
         final String hierarchy = randomAlphaOfLength(16);
 
-        final OsProbe probe = buildStubOsProbe(areCgroupStatsAvailable, hierarchy);
+        final OsProbe probe = buildStubOsProbe(availableCgroupsVersion, hierarchy);
 
         final OsStats.Cgroup cgroup = probe.osStats().getCgroup();
 
-        if (areCgroupStatsAvailable) {
+        if (availableCgroupsVersion > 0) {
             assertNotNull(cgroup);
             assertThat(cgroup.getCpuAcctControlGroup(), equalTo("/" + hierarchy));
             assertThat(cgroup.getCpuAcctUsageNanos(), equalTo(364869866063112L));
@@ -200,61 +203,53 @@ public void testCgroupProbe() {
     }
 
     public void testCgroupProbeWithMissingCpuAcct() {
-        assumeTrue("test runs on Linux only", Constants.LINUX);
-
         final String hierarchy = randomAlphaOfLength(16);
 
         // This cgroup data is missing a line about cpuacct
-        List<String> procSelfCgroupLines = getProcSelfGroupLines(hierarchy)
-            .stream()
+        List<String> procSelfCgroupLines = getProcSelfGroupLines(1, hierarchy).stream()
             .map(line -> line.replaceFirst(",cpuacct", ""))
             .collect(Collectors.toList());
 
-        final OsProbe probe = buildStubOsProbe(true, hierarchy, procSelfCgroupLines);
+        final OsProbe probe = buildStubOsProbe(1, hierarchy, procSelfCgroupLines);
 
         final OsStats.Cgroup cgroup = probe.osStats().getCgroup();
 
         assertNull(cgroup);
     }
 
     public void testCgroupProbeWithMissingCpu() {
-        assumeTrue("test runs on Linux only", Constants.LINUX);
-
         final String hierarchy = randomAlphaOfLength(16);
 
         // This cgroup data is missing a line about cpu
-        List<String> procSelfCgroupLines = getProcSelfGroupLines(hierarchy)
-            .stream()
+        List<String> procSelfCgroupLines = getProcSelfGroupLines(1, hierarchy).stream()
             .map(line -> line.replaceFirst(":cpu,", ":"))
             .collect(Collectors.toList());
 
-
-        final OsProbe probe = buildStubOsProbe(true, hierarchy, procSelfCgroupLines);
+        final OsProbe probe = buildStubOsProbe(1, hierarchy, procSelfCgroupLines);
 
         final OsStats.Cgroup cgroup = probe.osStats().getCgroup();
 
         assertNull(cgroup);
     }
 
     public void testCgroupProbeWithMissingMemory() {
-        assumeTrue("test runs on Linux only", Constants.LINUX);
-
         final String hierarchy = randomAlphaOfLength(16);
 
         // This cgroup data is missing a line about memory
-        List<String> procSelfCgroupLines = getProcSelfGroupLines(hierarchy)
-            .stream()
+        List<String> procSelfCgroupLines = getProcSelfGroupLines(1, hierarchy).stream()
             .filter(line -> line.contains(":memory:") == false)
             .collect(Collectors.toList());
 
-        final OsProbe probe = buildStubOsProbe(true, hierarchy, procSelfCgroupLines);
+        final OsProbe probe = buildStubOsProbe(1, hierarchy, procSelfCgroupLines);
 
         final OsStats.Cgroup cgroup = probe.osStats().getCgroup();
 
         assertNull(cgroup);
     }
 
     public void testGetTotalMemFromProcMeminfo() throws Exception {
+        int cgroupsVersion = randomFrom(1, 2);
+
         // missing MemTotal line
         List<String> meminfoLines = Arrays.asList(
             "MemFree:         8467692 kB",
@@ -265,7 +260,7 @@ public void testGetTotalMemFromProcMeminfo() throws Exception {
             "Active:         43637908 kB",
             "Inactive:        8130280 kB"
         );
-        OsProbe probe = buildStubOsProbe(true, "", org.elasticsearch.core.List.of(), meminfoLines);
+        OsProbe probe = buildStubOsProbe(cgroupsVersion, "", org.elasticsearch.core.List.of(), meminfoLines);
         assertThat(probe.getTotalMemFromProcMeminfo(), equalTo(0L));
 
         // MemTotal line with invalid value
@@ -279,7 +274,7 @@ public void testGetTotalMemFromProcMeminfo() throws Exception {
             "Active:         43637908 kB",
             "Inactive:        8130280 kB"
         );
-        probe = buildStubOsProbe(true, "", org.elasticsearch.core.List.of(), meminfoLines);
+        probe = buildStubOsProbe(cgroupsVersion, "", org.elasticsearch.core.List.of(), meminfoLines);
         assertThat(probe.getTotalMemFromProcMeminfo(), equalTo(0L));
 
         // MemTotal line with invalid unit
@@ -293,7 +288,7 @@ public void testGetTotalMemFromProcMeminfo() throws Exception {
             "Active:         43637908 kB",
             "Inactive:        8130280 kB"
         );
-        probe = buildStubOsProbe(true, "", org.elasticsearch.core.List.of(), meminfoLines);
+        probe = buildStubOsProbe(cgroupsVersion, "", org.elasticsearch.core.List.of(), meminfoLines);
         assertThat(probe.getTotalMemFromProcMeminfo(), equalTo(0L));
 
         // MemTotal line with random valid value
@@ -308,7 +303,7 @@ public void testGetTotalMemFromProcMeminfo() throws Exception {
             "Active:         43637908 kB",
             "Inactive:        8130280 kB"
         );
-        probe = buildStubOsProbe(true, "", org.elasticsearch.core.List.of(), meminfoLines);
+        probe = buildStubOsProbe(cgroupsVersion, "", org.elasticsearch.core.List.of(), meminfoLines);
         assertThat(probe.getTotalMemFromProcMeminfo(), equalTo(memTotalInKb * 1024L));
     }
 
@@ -319,7 +314,13 @@ public void testGetTotalMemoryOnDebian8() throws Exception {
         assertThat(osProbe.getTotalPhysicalMemorySize(), greaterThan(0L));
     }
 
-    private static List<String> getProcSelfGroupLines(String hierarchy) {
+    private static List<String> getProcSelfGroupLines(int cgroupsVersion, String hierarchy) {
+        // It doesn't really matter if cgroupsVersion == 0 here
+
+        if (cgroupsVersion == 2) {
+            return org.elasticsearch.core.List.of("0::/" + hierarchy);
+        }
+
         return Arrays.asList(
             "10:freezer:/",
             "9:net_cls,net_prio:/",
@@ -331,32 +332,40 @@ private static List<String> getProcSelfGroupLines(String hierarchy) {
             "3:perf_event:/",
             "2:cpu,cpuacct,cpuset:/" + hierarchy,
             "1:name=systemd:/user.slice/user-1000.slice/session-2359.scope",
-            "0::/cgroup2");
+            "0::/cgroup2"
+        );
     }
 
-    private static OsProbe buildStubOsProbe(final boolean areCgroupStatsAvailable, final String hierarchy) {
-        List<String> procSelfCgroupLines = getProcSelfGroupLines(hierarchy);
+    private static OsProbe buildStubOsProbe(final int availableCgroupsVersion, final String hierarchy) {
+        List<String> procSelfCgroupLines = getProcSelfGroupLines(availableCgroupsVersion, hierarchy);
 
-        return buildStubOsProbe(areCgroupStatsAvailable, hierarchy, procSelfCgroupLines);
+        return buildStubOsProbe(availableCgroupsVersion, hierarchy, procSelfCgroupLines);
     }
 
     /**
      * Builds a test instance of OsProbe. Methods that ordinarily read from the filesystem are overridden to return values based upon
      * the arguments to this method.
      *
-     * @param areCgroupStatsAvailable whether or not cgroup data is available. Normally OsProbe establishes this for itself.
+     * @param availableCgroupsVersion what version of cgroups are available, 1 or 2, or 0 for no cgroups. Normally OsProbe establishes this
+     *                                for itself.
      * @param hierarchy a mock value used to generate a cgroup hierarchy.
      * @param procSelfCgroupLines the lines that will be used as the content of <code>/proc/self/cgroup</code>
      * @param procMeminfoLines lines that will be used as the content of <code>/proc/meminfo</code>
      * @return a test instance
      */
     private static OsProbe buildStubOsProbe(
-        final boolean areCgroupStatsAvailable,
+        final int availableCgroupsVersion,
         final String hierarchy,
         List<String> procSelfCgroupLines,
         List<String> procMeminfoLines
     ) {
         return new OsProbe() {
+            @Override
+            OsStats.Cgroup getCgroup(boolean isLinux) {
+                // Pretend we're always on Linux so that we can run the cgroup tests
+                return super.getCgroup(true);
+            }
+
             @Override
             List<String> readProcSelfCgroup() {
                 return procSelfCgroupLines;
@@ -382,10 +391,7 @@ String readSysFsCgroupCpuAcctCpuAcctCfsQuota(String controlGroup) {
 
             @Override
             List<String> readSysFsCgroupCpuAcctCpuStat(String controlGroup) {
-                return Arrays.asList(
-                    "nr_periods 17992",
-                    "nr_throttled 1311",
-                    "throttled_time 139298645489");
+                return Arrays.asList("nr_periods 17992", "nr_throttled 1311", "throttled_time 139298645489");
             }
 
             @Override
@@ -403,22 +409,50 @@ String readSysFsCgroupMemoryUsageInBytes(String controlGroup) {
 
             @Override
             boolean areCgroupStatsAvailable() {
-                return areCgroupStatsAvailable;
+                return availableCgroupsVersion > 0;
             }
 
             @Override
-            List<String> readProcMeminfo() throws IOException {
+            List<String> readProcMeminfo() {
                 return procMeminfoLines;
             }
+
+            @Override
+            String readSysFsCgroupV2MemoryLimitInBytes(String controlGroup) {
+                assertThat(controlGroup, equalTo("/" + hierarchy));
+                // This is the highest value that can be stored in an unsigned 64 bit number, hence too big for long
+                return "18446744073709551615";
+            }
+
+            @Override
+            String readSysFsCgroupV2MemoryUsageInBytes(String controlGroup) {
+                assertThat(controlGroup, equalTo("/" + hierarchy));
+                return "4796416";
+            }
+
+            @Override
+            List<String> readCgroupV2CpuStats(String controlGroup) {
+                assertThat(controlGroup, equalTo("/" + hierarchy));
+                return org.elasticsearch.core.List.of(
+                    "usage_usec 364869866063112",
+                    "user_usec 34636",
+                    "system_usec 9896",
+                    "nr_periods 17992",
+                    "nr_throttled 1311",
+                    "throttled_usec 139298645489"
+                );
+            }
+
+            @Override
+            String readCgroupV2CpuLimit(String controlGroup) {
+                assertThat(controlGroup, equalTo("/" + hierarchy));
+                return "50000 100000";
+            }
         };
     }
 
-    private static OsProbe buildStubOsProbe(
-        final boolean areCgroupStatsAvailable,
-        final String hierarchy,
-        List<String> procSelfCgroupLines
-    ) {
-        return buildStubOsProbe(areCgroupStatsAvailable, hierarchy, procSelfCgroupLines, org.elasticsearch.core.List.of());
+    private static OsProbe buildStubOsProbe(final int availableCgroupsVersion, final String hierarchy, List<String> procSelfCgroupLines) {
+        return buildStubOsProbe(availableCgroupsVersion, hierarchy, procSelfCgroupLines, org.elasticsearch.core.List.of());
     }
 
 }

server/src/test/java/org/elasticsearch/monitor/os/OsProbeTests.java

@@ -1212,7 +1212,7 @@ static long machineMemoryFromStats(OsStats stats) {
         OsStats.Cgroup cgroup = stats.getCgroup();
         if (cgroup != null) {
             String containerLimitStr = cgroup.getMemoryLimitInBytes();
-            if (containerLimitStr != null) {
+            if (containerLimitStr != null && containerLimitStr.equals("max") == false) {
                 BigInteger containerLimit = new BigInteger(containerLimitStr);
                 if ((containerLimit.compareTo(BigInteger.valueOf(mem)) < 0 && containerLimit.compareTo(BigInteger.ZERO) > 0)
                         // mem <= 0 means the value couldn't be obtained for some reason