[release/9.4] Improve CPU metrics calculations for CgroupV2 (#6318)

* Getting metrics from path proc/self/cgroup

* Calculate usage by considering actual time elapsed since last usage calculation

* Decoupling actual slices read logic in V2 Parser.

* Fixing Broken Tests

* Mark Experimental

* A few test cases

* Nanoseconds for time interval

* Remove Min to reflect actual usages

* Resolving comments and adding tests.

* Resolving comments and test fixes

* Correctness Warning minor fix

* Fixing conflicting correctness warnings

* Minor change to retrigger pipelines

* New log line and minor fixes

* Acceptance test

* Increasing patch version for 9.4.1

---------

Co-authored-by: Nidhi Verma <nive@microsoft.com>
Co-authored-by: Jose Perez Rodriguez <joperezr@microsoft.com>

Author: 3 people

eng/Versions.props

@@ -2,7 +2,7 @@
   <PropertyGroup Label="Version settings">
     <MajorVersion>9</MajorVersion>
     <MinorVersion>4</MinorVersion>
-    <PatchVersion>0</PatchVersion>
+    <PatchVersion>1</PatchVersion>
     <PreReleaseVersionLabel>preview</PreReleaseVersionLabel>
     <PreReleaseVersionIteration>1</PreReleaseVersionIteration>
     <VersionPrefix>$(MajorVersion).$(MinorVersion).$(PatchVersion)</VersionPrefix>

src/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring/Linux/ILinuxUtilizationParser.cs

@@ -23,6 +23,13 @@ internal interface ILinuxUtilizationParser
     /// <returns>nanoseconds.</returns>
     long GetCgroupCpuUsageInNanoseconds();
 
+    /// <summary>
+    /// For CgroupV2 only and experimental. Reads the file cpu.stat based on /proc/self/cgroup, which is part of the cgroup v2 CPU controller.
+    /// It provides statistics about the CPU usage of a cgroup from its actual slice.
+    /// </summary>
+    /// <returns>nanoseconds.</returns>
+    long GetCgroupCpuUsageInNanosecondsV2();
+
     /// <summary>
     /// Reads the file /sys/fs/cgroup/cpu.max, which is part of the cgroup v2 CPU controller.
     /// It is used to set the maximum amount of CPU time that can be used by a cgroup.
@@ -33,6 +40,16 @@ internal interface ILinuxUtilizationParser
     /// <returns>cpuUnits.</returns>
     float GetCgroupLimitedCpus();
 
+    /// <summary>
+    /// For CgroupV2 only and experimental. Reads the file cpu.max based on /proc/self/cgroup, which is part of the cgroup v2 CPU controller.
+    /// It is used to set the maximum amount of CPU time that can be used by a cgroup from actual slice.
+    /// The file contains two fields, separated by a space.
+    /// The first field is the quota, which specifies the maximum amount of CPU time (in microseconds) that can be used by the cgroup during one period.
+    /// The second value is the period, which specifies the length of a period in microseconds.
+    /// </summary>
+    /// <returns>cpuUnits.</returns>
+    float GetCgroupLimitV2();
+
     /// <summary>
     /// Reads the file /proc/stat, which  provides information about the system’s memory usage.
     /// It contains information about the total amount of installed memory, the amount of free and used memory, and the amount of memory used by the kernel and buffers/cache.
@@ -66,4 +83,10 @@ internal interface ILinuxUtilizationParser
     /// </summary>
     /// <returns>cpuPodRequest.</returns>
     float GetCgroupRequestCpu();
+
+    /// <summary>
+    /// For CgroupV2 only and experimental. Reads the file cpu.weight based on /proc/self/cgroup. And calculates the Pod CPU Request in millicores based on actual slice.
+    /// </summary>
+    /// <returns>cpuPodRequest.</returns>
+    float GetCgroupRequestCpuV2();
 }

src/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring/Linux/LinuxUtilizationParserCgroupV1.cs

@@ -94,6 +94,10 @@ public LinuxUtilizationParserCgroupV1(IFileSystem fileSystem, IUserHz userHz)
         _userHz = userHz.Value;
     }
 
+    public float GetCgroupLimitV2() => throw new NotSupportedException();
+    public float GetCgroupRequestCpuV2() => throw new NotSupportedException();
+    public long GetCgroupCpuUsageInNanosecondsV2() => throw new NotSupportedException();
+
     public long GetCgroupCpuUsageInNanoseconds()
     {
         using ReturnableBufferWriter<char> bufferWriter = new(_sharedBufferWriterPool);

src/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring/Linux/LinuxUtilizationParserCgroupV2.cs

@@ -21,6 +21,9 @@ internal sealed class LinuxUtilizationParserCgroupV2 : ILinuxUtilizationParser
 {
     private const int Thousand = 1000;
     private const int CpuShares = 1024;
+    private const string CpuStat = "cpu.stat"; // File containing CPU usage in nanoseconds.
+    private const string CpuLimit = "cpu.max"; // File with amount of CPU time available to the group along with the accounting period in microseconds.
+    private const string CpuRequest = "cpu.weight"; // CPU weights, also known as shares in cgroup v1, is used for resource allocation.
     private static readonly ObjectPool<BufferWriter<char>> _sharedBufferWriterPool = BufferWriterPool.CreateBufferWriterPool<char>();
 
     /// <remarks>
@@ -86,47 +89,76 @@ internal sealed class LinuxUtilizationParserCgroupV2 : ILinuxUtilizationParser
     /// </summary>
     private static readonly FileInfo _cpuPodWeight = new("/sys/fs/cgroup/cpu.weight");
 
+    private static readonly FileInfo _cpuCgroupInfoFile = new("/proc/self/cgroup");
+
     private readonly IFileSystem _fileSystem;
     private readonly long _userHz;
 
+    // Cache for the trimmed path string to avoid repeated file reads and processing
+    private string? _cachedCgroupPath;
+
     public LinuxUtilizationParserCgroupV2(IFileSystem fileSystem, IUserHz userHz)
     {
         _fileSystem = fileSystem;
         _userHz = userHz.Value;
     }
 
-    public long GetCgroupCpuUsageInNanoseconds()
+    public string GetCgroupPath(string filename)
     {
-        // The value we are interested in starts with this. We just want to make sure it is true.
-        const string Usage_usec = "usage_usec";
-
-        // If the file doesn't exist, we assume that the system is a Host and we read the CPU usage from /proc/stat.
-        if (!_fileSystem.Exists(_cpuacctUsage))
+        // If we've already parsed the path, use the cached value
+        if (_cachedCgroupPath != null)
         {
-            return GetHostCpuUsageInNanoseconds();
+            return $"{_cachedCgroupPath}{filename}";
         }
 
         using ReturnableBufferWriter<char> bufferWriter = new(_sharedBufferWriterPool);
-        _fileSystem.ReadAll(_cpuacctUsage, bufferWriter.Buffer);
-        ReadOnlySpan<char> usage = bufferWriter.Buffer.WrittenSpan;
 
-        if (!usage.StartsWith(Usage_usec))
+        // Read the content of the file
+        _fileSystem.ReadFirstLine(_cpuCgroupInfoFile, bufferWriter.Buffer);
+        ReadOnlySpan<char> fileContent = bufferWriter.Buffer.WrittenSpan;
+
+        // Ensure the file content is not empty
+        if (fileContent.IsEmpty)
         {
-            Throw.InvalidOperationException($"Could not parse '{_cpuacctUsage}'. We expected first line of the file to start with '{Usage_usec}' but it was '{new string(usage)}' instead.");
+            Throw.InvalidOperationException($"The file '{_cpuCgroupInfoFile}' is empty or could not be read.");
         }
 
-        ReadOnlySpan<char> cpuUsage = usage.Slice(Usage_usec.Length, usage.Length - Usage_usec.Length);
+        // Find the index of the first colon (:)
+        int colonIndex = fileContent.LastIndexOf(':');
+        if (colonIndex == -1 || colonIndex + 1 >= fileContent.Length)
+        {
+            Throw.InvalidOperationException($"Invalid format in file '{_cpuCgroupInfoFile}'. Expected content with ':' separator.");
+        }
 
-        int next = GetNextNumber(cpuUsage, out long microseconds);
+        // Extract the part after the last colon and cache it for future use
+        ReadOnlySpan<char> trimmedPath = fileContent.Slice(colonIndex + 1);
+        _cachedCgroupPath = "/sys/fs/cgroup" + trimmedPath.ToString().TrimEnd('/') + "/";
 
-        if (microseconds == -1)
+        return $"{_cachedCgroupPath}{filename}";
+    }
+
+    public long GetCgroupCpuUsageInNanoseconds()
+    {
+        // If the file doesn't exist, we assume that the system is a Host and we read the CPU usage from /proc/stat.
+        if (!_fileSystem.Exists(_cpuacctUsage))
         {
-            Throw.InvalidOperationException($"Could not get cpu usage from '{_cpuacctUsage}'. Expected positive number, but got '{new string(usage)}'.");
+            return GetHostCpuUsageInNanoseconds();
         }
 
-        // In cgroup v2, the Units are microseconds for usage_usec.
-        // We multiply by 1000 to convert to nanoseconds to keep the common calculation logic.
-        return microseconds * Thousand;
+        return ParseCpuUsageFromFile(_fileSystem, _cpuacctUsage);
+    }
+
+    public long GetCgroupCpuUsageInNanosecondsV2()
+    {
+        FileInfo cpuUsageFile = new(GetCgroupPath(CpuStat));
+
+        // If the file doesn't exist, we assume that the system is a Host and we read the CPU usage from /proc/stat.
+        if (!_fileSystem.Exists(cpuUsageFile))
+        {
+            return GetHostCpuUsageInNanoseconds();
+        }
+
+        return ParseCpuUsageFromFile(_fileSystem, cpuUsageFile);
     }
 
     public long GetHostCpuUsageInNanoseconds()
@@ -184,6 +216,22 @@ public float GetCgroupLimitedCpus()
         return GetHostCpuCount();
     }
 
+    /// <remarks>
+    /// When CGroup limits are set, we can calculate number of cores based on the file settings.
+    /// It should be 99% of the cases when app is hosted in the container environment.
+    /// Otherwise, we assume that all host's CPUs are available, which we read from proc/stat file.
+    /// </remarks>
+    public float GetCgroupLimitV2()
+    {
+        FileInfo cpuLimitsFile = new(GetCgroupPath(CpuLimit));
+        if (LinuxUtilizationParserCgroupV2.TryGetCpuLimitFromCgroupsV2(_fileSystem, cpuLimitsFile, out float cpus))
+        {
+            return cpus;
+        }
+
+        return GetHostCpuCount();
+    }
+
     /// <remarks>
     /// If we are able to read the CPU share, we calculate the CPU request based on the weight by dividing it by 1024.
     /// If we can't read the CPU weight, we assume that the pod/vm cpu request is 1 core by default.
@@ -198,6 +246,21 @@ public float GetCgroupRequestCpu()
         return GetHostCpuCount();
     }
 
+    /// <remarks>
+    /// If we are able to read the CPU share, we calculate the CPU request based on the weight by dividing it by 1024.
+    /// If we can't read the CPU weight, we assume that the pod/vm cpu request is 1 core by default.
+    /// </remarks>
+    public float GetCgroupRequestCpuV2()
+    {
+        FileInfo cpuRequestsFile = new(GetCgroupPath(CpuRequest));
+        if (TryGetCgroupRequestCpuV2(_fileSystem, cpuRequestsFile, out float cpuPodRequest))
+        {
+            return cpuPodRequest / CpuShares;
+        }
+
+        return GetHostCpuCount();
+    }
+
     /// <remarks>
     /// If the file doesn't exist, we assume that the system is a Host and we read the memory from /proc/meminfo.
     /// </remarks>
@@ -447,6 +510,34 @@ static void ThrowException(ReadOnlySpan<char> content) =>
                 $"Could not parse '{_cpuSetCpus}'. Expected comma-separated list of integers, with dashes (\"-\") based ranges (\"0\", \"2-6,12\") but got '{new string(content)}'.");
     }
 
+    private static long ParseCpuUsageFromFile(IFileSystem fileSystem, FileInfo cpuUsageFile)
+    {
+        // The value we are interested in starts with this. We just want to make sure it is true.
+        const string Usage_usec = "usage_usec";
+
+        using ReturnableBufferWriter<char> bufferWriter = new(_sharedBufferWriterPool);
+        fileSystem.ReadAll(cpuUsageFile, bufferWriter.Buffer);
+        ReadOnlySpan<char> usage = bufferWriter.Buffer.WrittenSpan;
+
+        if (!usage.StartsWith(Usage_usec))
+        {
+            Throw.InvalidOperationException($"Could not parse '{cpuUsageFile}'. We expected first line of the file to start with '{Usage_usec}' but it was '{new string(usage)}' instead.");
+        }
+
+        ReadOnlySpan<char> cpuUsage = usage.Slice(Usage_usec.Length, usage.Length - Usage_usec.Length);
+
+        int next = GetNextNumber(cpuUsage, out long microseconds);
+
+        if (microseconds == -1)
+        {
+            Throw.InvalidOperationException($"Could not get cpu usage from '{cpuUsageFile}'. Expected positive number, but got '{new string(usage)}'.");
+        }
+
+        // In cgroup v2, the Units are microseconds for usage_usec.
+        // We multiply by 1000 to convert to nanoseconds to keep the common calculation logic.
+        return microseconds * Thousand;
+    }
+
     /// <remarks>
     /// The input must contain only number. If there is something more than whitespace before the number, it will return failure (-1).
     /// </remarks>
@@ -492,8 +583,27 @@ private static bool TryGetCpuUnitsFromCgroups(IFileSystem fileSystem, out float
             return false;
         }
 
+        return TryParseCpuQuotaAndPeriodFromFile(fileSystem, _cpuCfsQuaotaPeriodUs, out cpuUnits);
+    }
+
+    /// <remarks>
+    /// If the file doesn't exist, we assume that the system is a Host and we read the CPU usage from /proc/stat.
+    /// </remarks>
+    private static bool TryGetCpuLimitFromCgroupsV2(IFileSystem fileSystem, FileInfo cpuLimitsFile, out float cpuUnits)
+    {
+        if (!fileSystem.Exists(cpuLimitsFile))
+        {
+            cpuUnits = 0;
+            return false;
+        }
+
+        return TryParseCpuQuotaAndPeriodFromFile(fileSystem, cpuLimitsFile, out cpuUnits);
+    }
+
+    private static bool TryParseCpuQuotaAndPeriodFromFile(IFileSystem fileSystem, FileInfo cpuLimitsFile, out float cpuUnits)
+    {
         using ReturnableBufferWriter<char> bufferWriter = new(_sharedBufferWriterPool);
-        fileSystem.ReadFirstLine(_cpuCfsQuaotaPeriodUs, bufferWriter.Buffer);
+        fileSystem.ReadFirstLine(cpuLimitsFile, bufferWriter.Buffer);
 
         ReadOnlySpan<char> quotaBuffer = bufferWriter.Buffer.WrittenSpan;
 
@@ -513,7 +623,7 @@ private static bool TryGetCpuUnitsFromCgroups(IFileSystem fileSystem, out float
 
         if (quota == -1)
         {
-            Throw.InvalidOperationException($"Could not parse '{_cpuCfsQuaotaPeriodUs}'. Expected an integer but got: '{new string(quotaBuffer)}'.");
+            Throw.InvalidOperationException($"Could not parse '{cpuLimitsFile}'. Expected an integer but got: '{new string(quotaBuffer)}'.");
         }
 
         string quotaString = quota.ToString(CultureInfo.CurrentCulture);
@@ -523,7 +633,7 @@ private static bool TryGetCpuUnitsFromCgroups(IFileSystem fileSystem, out float
 
         if (period == -1)
         {
-            Throw.InvalidOperationException($"Could not parse '{_cpuCfsQuaotaPeriodUs}'. Expected to get an integer but got: '{new string(cpuPeriodSlice)}'.");
+            Throw.InvalidOperationException($"Could not parse '{cpuLimitsFile}'. Expected to get an integer but got: '{new string(cpuPeriodSlice)}'.");
         }
 
         cpuUnits = (float)quota / period;
@@ -533,37 +643,53 @@ private static bool TryGetCpuUnitsFromCgroups(IFileSystem fileSystem, out float
 
     private static bool TryGetCgroupRequestCpu(IFileSystem fileSystem, out float cpuUnits)
     {
-        const long CpuPodWeightPossibleMax = 10_000;
-        const long CpuPodWeightPossibleMin = 1;
-
         if (!fileSystem.Exists(_cpuPodWeight))
         {
             cpuUnits = 0;
             return false;
         }
 
+        return TryParseCpuWeightFromFile(fileSystem, _cpuPodWeight, out cpuUnits);
+    }
+
+    private static bool TryGetCgroupRequestCpuV2(IFileSystem fileSystem, FileInfo cpuRequestsFile, out float cpuUnits)
+    {
+        if (!fileSystem.Exists(cpuRequestsFile))
+        {
+            cpuUnits = 0;
+            return false;
+        }
+
+        return TryParseCpuWeightFromFile(fileSystem, cpuRequestsFile, out cpuUnits);
+    }
+
+    private static bool TryParseCpuWeightFromFile(IFileSystem fileSystem, FileInfo cpuWeightFile, out float cpuUnits)
+    {
+        const long CpuPodWeightPossibleMax = 10_000;
+        const long CpuPodWeightPossibleMin = 1;
+
         using ReturnableBufferWriter<char> bufferWriter = new(_sharedBufferWriterPool);
-        fileSystem.ReadFirstLine(_cpuPodWeight, bufferWriter.Buffer);
+        fileSystem.ReadFirstLine(cpuWeightFile, bufferWriter.Buffer);
         ReadOnlySpan<char> cpuPodWeightBuffer = bufferWriter.Buffer.WrittenSpan;
 
         if (cpuPodWeightBuffer.IsEmpty || (cpuPodWeightBuffer.Length == 2 && cpuPodWeightBuffer[0] == '-' && cpuPodWeightBuffer[1] == '1'))
         {
             Throw.InvalidOperationException(
-                $"Could not parse '{_cpuPodWeight}' content. Expected to find CPU weight but got '{new string(cpuPodWeightBuffer)}' instead.");
+                $"Could not parse '{cpuWeightFile}' content. Expected to find CPU weight but got '{new string(cpuPodWeightBuffer)}' instead.");
         }
 
         _ = GetNextNumber(cpuPodWeightBuffer, out long cpuPodWeight);
 
         if (cpuPodWeight == -1)
         {
             Throw.InvalidOperationException(
-                $"Could not parse '{_cpuPodWeight}' content. Expected to get an integer but got: '{cpuPodWeightBuffer}'.");
+                $"Could not parse '{cpuWeightFile}' content. Expected to get an integer but got: '{cpuPodWeightBuffer}'.");
         }
 
         if (cpuPodWeight < CpuPodWeightPossibleMin || cpuPodWeight > CpuPodWeightPossibleMax)
         {
             Throw.ArgumentOutOfRangeException("CPU weight",
-                $"Expected to find CPU weight in range [{CpuPodWeightPossibleMin}-{CpuPodWeightPossibleMax}] in '{_cpuPodWeight}', but got '{cpuPodWeight}' instead.");
+                $"Expected to find CPU weight in range [{CpuPodWeightPossibleMin}-{CpuPodWeightPossibleMax}] in '{cpuWeightFile}', but got '{cpuPodWeight}' instead.");
         }
 
         // The formula to calculate CPU pod weight (measured in millicores) from CPU share: