KVM Agent config to reserve dom0 CPUs (#7987)

This PR allows an admin to reserve some hypervisor host CPUs for system use. Another way to think of it is limiting the number of CPUs allocatable to VMs. This can be useful if the admin wants to do other things with the hypervisor's CPU, for example reserve some cores for running hyperconverged storage processes.

Co-authored-by: Marcus Sorensen <mls@apple.com>

Author: Marcus Sorensen

agent/conf/agent.properties

@@ -279,6 +279,11 @@ hypervisor.type=kvm
 # If this parameter is used, property host.overcommit.mem.mb must be set to 0.
 #host.reserved.mem.mb=1024
 
+# Number of CPU cores to subtract from advertised available cores.
+# These are reserved for system activity, or otherwise share host CPU resources with
+# CloudStack VM allocation.
+# host.reserved.cpu.count = 0
+
 # The model of Watchdog timer to present to the Guest.
 # For all models refer to the libvirt documentation.
 #vm.watchdog.model=i6300esb

agent/src/main/java/com/cloud/agent/properties/AgentProperties.java

@@ -502,6 +502,15 @@ public class AgentProperties{
      */
     public static final Property<Integer> HOST_RESERVED_MEM_MB = new Property<>("host.reserved.mem.mb", 1024);
 
+    /**
+     * How many host CPUs to reserve for non-allocation.<br>
+     * This can be used to set aside CPU cores on the host for other tasks, such as running hyperconverged storage<br>
+     * processes, etc.
+     * Data type: Integer.<br>
+     * Default value: <code>0</code>
+     */
+    public static final Property<Integer> HOST_RESERVED_CPU_CORE_COUNT = new Property<>("host.reserved.cpu.count", 0);
+
     /**
      * The model of Watchdog timer to present to the Guest.<br>
      * For all models refer to the libvirt documentation.<br>

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java

@@ -460,6 +460,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
 
     private long dom0OvercommitMem;
 
+    private int dom0MinCpuCores;
+
     protected int cmdsTimeout;
     protected int stopTimeout;
     protected CPUStat cpuStat = new CPUStat();
@@ -1063,6 +1065,8 @@ public boolean configure(final String name, final Map<String, Object> params) th
         // Reserve 1GB unless admin overrides
         dom0MinMem = ByteScaleUtils.mebibytesToBytes(AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HOST_RESERVED_MEM_MB));
 
+        dom0MinCpuCores = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HOST_RESERVED_CPU_CORE_COUNT);
+
         // Support overcommit memory for host if host uses ZSWAP, KSM and other memory
         // compressing technologies
         dom0OvercommitMem = ByteScaleUtils.mebibytesToBytes(AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HOST_OVERCOMMIT_MEM_MB));
@@ -3540,15 +3544,15 @@ private Map<String, String> getVersionStrings() {
     @Override
     public StartupCommand[] initialize() {
 
-        final KVMHostInfo info = new KVMHostInfo(dom0MinMem, dom0OvercommitMem, manualCpuSpeed);
+        final KVMHostInfo info = new KVMHostInfo(dom0MinMem, dom0OvercommitMem, manualCpuSpeed, dom0MinCpuCores);
 
         String capabilities = String.join(",", info.getCapabilities());
         if (dpdkSupport) {
             capabilities += ",dpdk";
         }
 
         final StartupRoutingCommand cmd =
-                new StartupRoutingCommand(info.getCpus(), info.getCpuSpeed(), info.getTotalMemory(), info.getReservedMemory(), capabilities, hypervisorType,
+                new StartupRoutingCommand(info.getAllocatableCpus(), info.getCpuSpeed(), info.getTotalMemory(), info.getReservedMemory(), capabilities, hypervisorType,
                         RouterPrivateIpStrategy.HostLocal);
         cmd.setCpuSockets(info.getCpuSockets());
         fillNetworkInformation(cmd);

plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/linux/KVMHostInfo.java

@@ -48,7 +48,8 @@ public class KVMHostInfo {
 
     private static final Logger LOGGER = Logger.getLogger(KVMHostInfo.class);
 
-    private int cpus;
+    private int totalCpus;
+    private int allocatableCpus;
     private int cpusockets;
     private long cpuSpeed;
     private long totalMemory;
@@ -58,16 +59,25 @@ public class KVMHostInfo {
 
     private static String cpuInfoFreqFileName = "/sys/devices/system/cpu/cpu0/cpufreq/base_frequency";
 
-    public KVMHostInfo(long reservedMemory, long overCommitMemory, long manualSpeed) {
+    public KVMHostInfo(long reservedMemory, long overCommitMemory, long manualSpeed, int reservedCpus) {
         this.cpuSpeed = manualSpeed;
         this.reservedMemory = reservedMemory;
         this.overCommitMemory = overCommitMemory;
         this.getHostInfoFromLibvirt();
         this.totalMemory = new MemStat(this.getReservedMemory(), this.getOverCommitMemory()).getTotal();
+        this.allocatableCpus = totalCpus - reservedCpus;
+        if (allocatableCpus < 1) {
+            LOGGER.warn(String.format("Aggressive reserved CPU config leaves no usable CPUs for VMs! Total system CPUs: %d, Reserved: %d, Allocatable: %d", totalCpus, reservedCpus, allocatableCpus));
+            allocatableCpus = 0;
+        }
+    }
+
+    public int getTotalCpus() {
+        return this.totalCpus;
     }
 
-    public int getCpus() {
-        return this.cpus;
+    public int getAllocatableCpus() {
+        return this.allocatableCpus;
     }
 
     public int getCpuSockets() {
@@ -189,7 +199,7 @@ private void getHostInfoFromLibvirt() {
             if (hosts.nodes > 0) {
                 this.cpusockets = hosts.sockets * hosts.nodes;
             }
-            this.cpus = hosts.cpus;
+            this.totalCpus = hosts.cpus;
 
             final LibvirtCapXMLParser parser = new LibvirtCapXMLParser();
             parser.parseCapabilitiesXML(capabilities);

plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/linux/KVMHostInfoTest.java

@@ -73,8 +73,36 @@ public void manualCpuSpeedTest() throws Exception {
             Mockito.when(conn.close()).thenReturn(0);
             int manualSpeed = 500;
 
-            KVMHostInfo kvmHostInfo = new KVMHostInfo(10, 10, manualSpeed);
+            KVMHostInfo kvmHostInfo = new KVMHostInfo(10, 10, manualSpeed, 0);
             Assert.assertEquals(kvmHostInfo.getCpuSpeed(), manualSpeed);
         }
     }
+
+    @Test
+    public void reservedCpuCoresTest() throws Exception {
+        if (!System.getProperty("os.name").equals("Linux")) {
+            return;
+        }
+        try (MockedStatic<LibvirtConnection> ignored = Mockito.mockStatic(LibvirtConnection.class)) {
+            Connect conn = Mockito.mock(Connect.class);
+            NodeInfo nodeInfo = Mockito.mock(NodeInfo.class);
+            nodeInfo.cpus = 10;
+            String capabilitiesXml = "<capabilities></capabilities>";
+
+            Mockito.when(LibvirtConnection.getConnection()).thenReturn(conn);
+            Mockito.when(conn.nodeInfo()).thenReturn(nodeInfo);
+            Mockito.when(conn.getCapabilities()).thenReturn(capabilitiesXml);
+            Mockito.when(conn.close()).thenReturn(0);
+            int manualSpeed = 500;
+
+            KVMHostInfo kvmHostInfo = new KVMHostInfo(10, 10, 100, 2);
+            Assert.assertEquals("reserve two CPU cores", 8, kvmHostInfo.getAllocatableCpus());
+
+            kvmHostInfo = new KVMHostInfo(10, 10, 100, 0);
+            Assert.assertEquals("no reserve CPU core setting", 10, kvmHostInfo.getAllocatableCpus());
+
+            kvmHostInfo = new KVMHostInfo(10, 10, 100, 12);
+            Assert.assertEquals("Misconfigured/too large CPU reserve", 0, kvmHostInfo.getAllocatableCpus());
+        }
+    }
 }