Give a system VM one NIC queue per CPU

server/src/main/java/com/cloud/hypervisor/HypervisorGuruBase.java

@@ -16,6 +16,7 @@
 // under the License.
 package com.cloud.hypervisor;
 
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -73,6 +74,7 @@
 import com.cloud.storage.Volume;
 import com.cloud.utils.Pair;
 import com.cloud.utils.component.AdapterBase;
+import com.cloud.vm.VmDetailConstants;
 import com.cloud.vm.NicProfile;
 import com.cloud.vm.NicVO;
 import com.cloud.vm.UserVmManager;
@@ -86,6 +88,8 @@
 
 public abstract class HypervisorGuruBase extends AdapterBase implements HypervisorGuru, Configurable {
 
+    private static final int MAX_TAP_QUEUES = 256;
+
     @Inject
     protected
     NicDao nicDao;
@@ -265,6 +269,34 @@ protected void addServiceOfferingExtraConfiguration(ServiceOffering offering, Vi
         }
     }
 
+    /**
+     * Gives a system VM one NIC queue per CPU. The guest only ever uses as many queues as it has
+     * CPUs, so this is what the CPU count already implies, and a system VM is only given more CPUs
+     * to move more packets.
+     *
+     * A user VM sets this per VM with deployVirtualMachine or updateVirtualMachine. A system VM
+     * goes through neither, so there is otherwise no way for it to get a queue at all.
+     *
+     * The default offering is a single CPU, so nothing changes until an operator resizes.
+     */
+    protected void addDefaultNicQueuesForSystemVm(VirtualMachineTO to) {
+        if (to.getType() == null || !to.getType().isUsedBySystem()) {
+            return;
+        }
+        Map<String, String> details = to.getDetails();
+        if (details != null && details.containsKey(VmDetailConstants.NIC_MULTIQUEUE_NUMBER)) {
+            return;
+        }
+        // A tap device stops at 256 queues, and the host refuses the VM rather than trimming.
+        int queues = Math.min(to.getCpus(), MAX_TAP_QUEUES);
+        if (queues < 2) {
+            return;
+        }
+        Map<String, String> updated = details == null ? new HashMap<>() : new HashMap<>(details);
+        updated.put(VmDetailConstants.NIC_MULTIQUEUE_NUMBER, String.valueOf(queues));
+        to.setDetails(updated);
+    }
+
     protected VirtualMachineTO toVirtualMachineTO(VirtualMachineProfile vmProfile) {
         ServiceOffering offering = serviceOfferingDao.findById(vmProfile.getId(), vmProfile.getServiceOfferingId());
         VirtualMachine vm = vmProfile.getVirtualMachine();
@@ -332,6 +364,8 @@ protected VirtualMachineTO toVirtualMachineTO(VirtualMachineProfile vmProfile) {
             addExtraConfig(detailsInVm, to, vm.getAccountId(), vm.getHypervisorType());
         }
 
+        addDefaultNicQueuesForSystemVm(to);
+
         addServiceOfferingExtraConfiguration(offering, to);
 
         // Set GPU details

server/src/test/java/com/cloud/hypervisor/KVMGuruTest.java

@@ -29,6 +29,7 @@
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
 import org.mockito.InjectMocks;
 import org.mockito.Mock;
 import org.mockito.Mockito;
@@ -51,6 +52,7 @@
 import com.cloud.storage.dao.GuestOSDao;
 import com.cloud.storage.dao.GuestOSHypervisorDao;
 import com.cloud.utils.Pair;
+import com.cloud.vm.VmDetailConstants;
 import com.cloud.vm.VirtualMachine;
 import com.cloud.vm.VirtualMachineProfile;
 
@@ -506,4 +508,80 @@ public void testGetNullWhenVMThereIsNoInformationOfUsedHosts() {
 
         Assert.assertNull(clusterId);
     }
+
+    private VirtualMachineTO systemVmTO(int cpus, Map<String, String> details) {
+        VirtualMachineTO to = Mockito.mock(VirtualMachineTO.class);
+        Mockito.when(to.getType()).thenReturn(VirtualMachine.Type.DomainRouter);
+        Mockito.when(to.getCpus()).thenReturn(cpus);
+        Mockito.when(to.getDetails()).thenReturn(details);
+        return to;
+    }
+
+    @SuppressWarnings("unchecked")
+    private Map<String, String> capturedDetails(VirtualMachineTO to) {
+        ArgumentCaptor<Map<String, String>> captor = ArgumentCaptor.forClass(Map.class);
+        Mockito.verify(to).setDetails(captor.capture());
+        return captor.getValue();
+    }
+
+    @Test
+    public void testSystemVmGetsAQueuePerCpu() {
+        VirtualMachineTO to = systemVmTO(4, null);
+
+        guru.addDefaultNicQueuesForSystemVm(to);
+
+        Assert.assertEquals("4", capturedDetails(to).get(VmDetailConstants.NIC_MULTIQUEUE_NUMBER));
+    }
+
+    @Test
+    public void testSystemVmQueuesStopAtTheTapLimit() {
+        VirtualMachineTO to = systemVmTO(512, null);
+
+        guru.addDefaultNicQueuesForSystemVm(to);
+
+        Assert.assertEquals("256", capturedDetails(to).get(VmDetailConstants.NIC_MULTIQUEUE_NUMBER));
+    }
+
+    @Test
+    public void testSingleCpuSystemVmIsLeftAlone() {
+        VirtualMachineTO to = systemVmTO(1, null);
+
+        guru.addDefaultNicQueuesForSystemVm(to);
+
+        Mockito.verify(to, Mockito.never()).setDetails(Mockito.anyMap());
+    }
+
+    @Test
+    public void testExistingDetailsAreKept() {
+        Map<String, String> existing = new HashMap<>();
+        existing.put(VmDetailConstants.ROOT_DISK_CONTROLLER, "virtio");
+        VirtualMachineTO to = systemVmTO(4, existing);
+
+        guru.addDefaultNicQueuesForSystemVm(to);
+
+        Map<String, String> details = capturedDetails(to);
+        Assert.assertEquals("4", details.get(VmDetailConstants.NIC_MULTIQUEUE_NUMBER));
+        Assert.assertEquals("virtio", details.get(VmDetailConstants.ROOT_DISK_CONTROLLER));
+    }
+
+    @Test
+    public void testAQueueNumberAlreadySetWins() {
+        Map<String, String> set = new HashMap<>();
+        set.put(VmDetailConstants.NIC_MULTIQUEUE_NUMBER, "2");
+        VirtualMachineTO to = systemVmTO(8, set);
+
+        guru.addDefaultNicQueuesForSystemVm(to);
+
+        Mockito.verify(to, Mockito.never()).setDetails(Mockito.anyMap());
+    }
+
+    @Test
+    public void testUserVmsAreNotTouched() {
+        VirtualMachineTO to = Mockito.mock(VirtualMachineTO.class);
+        Mockito.when(to.getType()).thenReturn(VirtualMachine.Type.User);
+
+        guru.addDefaultNicQueuesForSystemVm(to);
+
+        Mockito.verify(to, Mockito.never()).setDetails(Mockito.anyMap());
+    }
 }