cybozu-go · ysksuzuki · Nov 18, 2021 · Nov 11, 2021
diff --git a/docs/resource.md b/docs/resource.md
@@ -80,6 +80,7 @@ volumes:
 ignition: my-node.ign
 cpu: 2
 memory: 4G
+network-device-queue: 4
 smbios:
   manufacturer: cybozu
   product: mk2
@@ -99,6 +100,7 @@ The properties are:
 - `ignition`: [Ignition file](https://coreos.com/ignition/docs/latest/configuration-v2_1.html).
 - `cpu`: The amount of virtual CPUs.
 - `memory`: The amount of memory.
+- `network-device-queue`: The count of VM's network device queue. Placemat enables multi queue virtio-net if network-device-queue is greater than 1.
 - `smbios`: System Management BIOS (SMBIOS) values for `manufacturer`, `product`, and `serial`.  If `serial` is not set, a hash value of the node's name is used.
 - `uefi`: BIOS mode of the VM.
     - If false: The VM will load Qemu's default BIOS (SeaBIO) and enable iPXE boot by a net device.

diff --git a/v2/e2e/cluster.yml b/v2/e2e/cluster.yml
@@ -19,8 +19,9 @@ kind: Node
 name: node1
 interfaces:
 - ext-net
-cpu: 1
+cpu: 2
 memory: 1G
+network-device-queue: 4
 smbios:
   serial: 4ae0771764376e0184274e24e0b18abd8fa5f285
 volumes:
@@ -53,8 +54,9 @@ kind: Node
 name: node2
 interfaces:
 - ext-net
-cpu: 1
+cpu: 2
 memory: 1G
+network-device-queue: 4
 smbios:
   serial: 1145129a085ecf521ab524028302a6b4a4e49a00
 volumes:

diff --git a/v2/pkg/types/cluster.go b/v2/pkg/types/cluster.go
@@ -140,16 +140,17 @@ const (
 
 // NodeSpec represents a Node specification in YAML
 type NodeSpec struct {
-	Kind         string           `json:"kind"`
-	Name         string           `json:"name"`
-	Interfaces   []string         `json:"interfaces,omitempty"`
-	Volumes      []NodeVolumeSpec `json:"volumes,omitempty"`
-	IgnitionFile string           `json:"ignition,omitempty"`
-	CPU          int              `json:"cpu,omitempty"`
-	Memory       string           `json:"memory,omitempty"`
-	UEFI         bool             `json:"uefi,omitempty"`
-	TPM          bool             `json:"tpm,omitempty"`
-	SMBIOS       SMBIOSConfigSpec `json:"smbios,omitempty"`
+	Kind               string           `json:"kind"`
+	Name               string           `json:"name"`
+	Interfaces         []string         `json:"interfaces,omitempty"`
+	Volumes            []NodeVolumeSpec `json:"volumes,omitempty"`
+	IgnitionFile       string           `json:"ignition,omitempty"`
+	CPU                int              `json:"cpu,omitempty"`
+	Memory             string           `json:"memory,omitempty"`
+	NetworkDeviceQueue int              `json:"network-device-queue,omitempty"`
+	UEFI               bool             `json:"uefi,omitempty"`
+	TPM                bool             `json:"tpm,omitempty"`
+	SMBIOS             SMBIOSConfigSpec `json:"smbios,omitempty"`
 }
 
 func (n *NodeSpec) validate() error {

diff --git a/v2/pkg/types/cluster_test.go b/v2/pkg/types/cluster_test.go
@@ -52,6 +52,7 @@ interfaces:
 - r0-node2
 memory: 2G
 cpu: 8
+network-device-queue: 16
 smbios:
   manufacturer: cybozu
   product: mk2
@@ -159,11 +160,12 @@ file: cybozu-ubuntu-18.04-server-cloudimg-amd64.img
 							Path: "/var/foo/sabakan-data",
 						},
 					},
-					IgnitionFile: "my-node.ign",
-					CPU:          8,
-					Memory:       "2G",
-					UEFI:         false,
-					TPM:          true,
+					IgnitionFile:       "my-node.ign",
+					CPU:                8,
+					Memory:             "2G",
+					NetworkDeviceQueue: 16,
+					UEFI:               false,
+					TPM:                true,
 					SMBIOS: SMBIOSConfigSpec{
 						Manufacturer: "cybozu",
 						Product:      "mk2",

diff --git a/v2/pkg/vm/node.go b/v2/pkg/vm/node.go
@@ -32,15 +32,16 @@ type Node interface {
 }
 
 type node struct {
-	name         string
-	taps         []*tap
-	volumes      []nodeVolume
-	ignitionFile string
-	cpu          int
-	memory       string
-	uefi         bool
-	tpm          bool
-	smbios       smBIOSConfig
+	name               string
+	taps               []*tap
+	volumes            []nodeVolume
+	ignitionFile       string
+	cpu                int
+	memory             string
+	networkDeviceQueue int
+	uefi               bool
+	tpm                bool
+	smbios             smBIOSConfig
 }
 
 type smBIOSConfig struct {
@@ -52,12 +53,13 @@ type smBIOSConfig struct {
 // NewNode creates a Node from spec.
 func NewNode(spec *types.NodeSpec, imageSpecs []*types.ImageSpec) (Node, error) {
 	n := &node{
-		name:         spec.Name,
-		ignitionFile: spec.IgnitionFile,
-		cpu:          spec.CPU,
-		memory:       spec.Memory,
-		uefi:         spec.UEFI,
-		tpm:          spec.TPM,
+		name:               spec.Name,
+		ignitionFile:       spec.IgnitionFile,
+		cpu:                spec.CPU,
+		memory:             spec.Memory,
+		networkDeviceQueue: spec.NetworkDeviceQueue,
+		uefi:               spec.UEFI,
+		tpm:                spec.TPM,
 		smbios: smBIOSConfig{
 			manufacturer: spec.SMBIOS.Manufacturer,
 			product:      spec.SMBIOS.Product,
@@ -126,7 +128,7 @@ func (n *node) Setup(ctx context.Context, r *Runtime, mtu int, nodeCh chan<- BMC
 		}
 	}
 
-	qemu := newQemu(n.name, tapInfos, vArgs, n.ignitionFile, n.cpu, n.memory, n.uefi, n.tpm, n.smbios)
+	qemu := newQemu(n.name, tapInfos, vArgs, n.ignitionFile, n.cpu, n.memory, n.networkDeviceQueue, n.uefi, n.tpm, n.smbios)
 	c := qemu.command(r)
 	qemuCommand := well.CommandContext(ctx, c[0], c[1:]...)
 	qemuCommand.Stdout = util.NewColoredLogWriter("qemu", n.name, os.Stdout)
@@ -203,7 +205,7 @@ func (n *node) createVolumes(ctx context.Context, dataDir string) ([]volumeArgs,
 func (n *node) createTaps(mtu int) ([]*tapInfo, error) {
 	var tapInfos []*tapInfo
 	for _, tap := range n.taps {
-		tapInfo, err := tap.create(mtu)
+		tapInfo, err := tap.create(mtu, n.networkDeviceQueue)
 		if err != nil {
 			return nil, fmt.Errorf("failed to create the tap: %w", err)
 		}

diff --git a/v2/pkg/vm/qemu.go b/v2/pkg/vm/qemu.go
@@ -18,31 +18,33 @@ const (
 )
 
 type qemu struct {
-	name         string
-	taps         []*tapInfo
-	volumes      []volumeArgs
-	ignitionFile string
-	cpu          int
-	memory       string
-	uefi         bool
-	tpm          bool
-	smbios       smBIOSConfig
+	name               string
+	taps               []*tapInfo
+	volumes            []volumeArgs
+	ignitionFile       string
+	cpu                int
+	memory             string
+	networkDeviceQueue int
+	uefi               bool
+	tpm                bool
+	smbios             smBIOSConfig
 	macGenerator
 }
 
 func newQemu(nodeName string, taps []*tapInfo, volumes []volumeArgs, ignitionFile string, cpu int,
-	memory string, uefi bool, tpm bool, smbios smBIOSConfig) *qemu {
+	memory string, networkDeviceQueue int, uefi bool, tpm bool, smbios smBIOSConfig) *qemu {
 	return &qemu{
-		name:         nodeName,
-		taps:         taps,
-		volumes:      volumes,
-		ignitionFile: ignitionFile,
-		cpu:          cpu,
-		memory:       memory,
-		uefi:         uefi,
-		tpm:          tpm,
-		smbios:       smbios,
-		macGenerator: &macGeneratorForKVM{},
+		name:               nodeName,
+		taps:               taps,
+		volumes:            volumes,
+		ignitionFile:       ignitionFile,
+		cpu:                cpu,
+		memory:             memory,
+		networkDeviceQueue: networkDeviceQueue,
+		uefi:               uefi,
+		tpm:                tpm,
+		smbios:             smbios,
+		macGenerator:       &macGeneratorForKVM{},
 	}
 }
 
@@ -54,6 +56,9 @@ func (c *qemu) command(r *Runtime) []string {
 		if vhostNetSupported {
 			netdev += ",vhost=on"
 		}
+		if c.networkDeviceQueue > 1 {
+			netdev += fmt.Sprintf(",queues=%d", c.networkDeviceQueue)
+		}
 
 		params = append(params, "-netdev", netdev)
 
@@ -63,6 +68,9 @@ func (c *qemu) command(r *Runtime) []string {
 			fmt.Sprintf("netdev=%s", t.bridge),
 			fmt.Sprintf("mac=%s", c.generate()),
 		}
+		if c.networkDeviceQueue > 1 {
+			devParams = append(devParams, "mq=on", fmt.Sprintf("vectors=%d", 2*c.networkDeviceQueue+2))
+		}
 		if c.uefi {
 			// disable iPXE boot
 			devParams = append(devParams, "romfile=")

diff --git a/v2/pkg/vm/qemu_test.go b/v2/pkg/vm/qemu_test.go
@@ -50,6 +50,7 @@ kind: Node
 name: boot-0
 cpu: 8
 memory: 2G
+network-device-queue: 16
 interfaces:
 - r0-node1
 - r0-node2
@@ -121,7 +122,7 @@ use-nat: false
 			Expect(err).NotTo(HaveOccurred())
 			taps = append(taps, tap)
 
-			tapInfo, err := tap.create(1460)
+			tapInfo, err := tap.create(1460, nodeSpec.NetworkDeviceQueue)
 			Expect(err).NotTo(HaveOccurred())
 			tapInfos = append(tapInfos, tapInfo)
 		}
@@ -132,7 +133,7 @@ use-nat: false
 		}()
 
 		qemu := newQemu(nodeSpec.Name, tapInfos, volumeArgs, nodeSpec.IgnitionFile, nodeSpec.CPU, nodeSpec.Memory,
-			nodeSpec.UEFI, nodeSpec.TPM, smBIOSConfig{
+			nodeSpec.NetworkDeviceQueue, nodeSpec.UEFI, nodeSpec.TPM, smBIOSConfig{
 				manufacturer: nodeSpec.SMBIOS.Manufacturer,
 				product:      nodeSpec.SMBIOS.Product,
 				serial:       nodeSpec.SMBIOS.Serial,
@@ -148,10 +149,10 @@ qemu-system-x86_64
  -nographic
  -serial unix:%s/boot-0.socket,server,nowait
  -smbios type=1,serial=fb8f2417d0b4db30050719c31ce02a2e8141bbd8
- -netdev tap,id=r0-node1,ifname=%s,script=no,downscript=no,vhost=on
- -device virtio-net-pci,host_mtu=1460,netdev=r0-node1,mac=placemat
- -netdev tap,id=r0-node2,ifname=%s,script=no,downscript=no,vhost=on
- -device virtio-net-pci,host_mtu=1460,netdev=r0-node2,mac=placemat
+ -netdev tap,id=r0-node1,ifname=%s,script=no,downscript=no,vhost=on,queues=16
+ -device virtio-net-pci,host_mtu=1460,netdev=r0-node1,mac=placemat,mq=on,vectors=34
+ -netdev tap,id=r0-node2,ifname=%s,script=no,downscript=no,vhost=on,queues=16
+ -device virtio-net-pci,host_mtu=1460,netdev=r0-node2,mac=placemat,mq=on,vectors=34
  -drive if=virtio,cache=writeback,aio=threads,file=%s/root.img
  -drive if=virtio,cache=none,aio=native,format=qcow2,file=%s/seed.img
  -virtfs local,path=%s,mount_tag=sabakan,security_model=none,readonly
@@ -196,6 +197,7 @@ kind: Node
 name: boot-0
 cpu: 8
 memory: 2G
+network-device-queue: 16
 interfaces:
 - r0-node1
 - r0-node2
@@ -269,7 +271,7 @@ use-nat: false
 			Expect(err).NotTo(HaveOccurred())
 			taps = append(taps, tap)
 
-			tapInfo, err := tap.create(1460)
+			tapInfo, err := tap.create(1460, nodeSpec.NetworkDeviceQueue)
 			Expect(err).NotTo(HaveOccurred())
 			tapInfos = append(tapInfos, tapInfo)
 		}
@@ -280,7 +282,7 @@ use-nat: false
 		}()
 
 		qemu := newQemu(nodeSpec.Name, tapInfos, volumeArgs, nodeSpec.IgnitionFile, nodeSpec.CPU, nodeSpec.Memory,
-			nodeSpec.UEFI, nodeSpec.TPM, smBIOSConfig{
+			nodeSpec.NetworkDeviceQueue, nodeSpec.UEFI, nodeSpec.TPM, smBIOSConfig{
 				manufacturer: nodeSpec.SMBIOS.Manufacturer,
 				product:      nodeSpec.SMBIOS.Product,
 				serial:       nodeSpec.SMBIOS.Serial,
@@ -298,10 +300,10 @@ qemu-system-x86_64
  -drive if=pflash,file=/usr/share/OVMF/OVMF_CODE.fd,format=raw,readonly
  -drive if=pflash,file=%s/nvram/boot-0.fd,format=raw
  -smbios type=1,serial=fb8f2417d0b4db30050719c31ce02a2e8141bbd8
- -netdev tap,id=r0-node1,ifname=%s,script=no,downscript=no,vhost=on
- -device virtio-net-pci,host_mtu=1460,netdev=r0-node1,mac=placemat,romfile=
- -netdev tap,id=r0-node2,ifname=%s,script=no,downscript=no,vhost=on
- -device virtio-net-pci,host_mtu=1460,netdev=r0-node2,mac=placemat,romfile=
+ -netdev tap,id=r0-node1,ifname=%s,script=no,downscript=no,vhost=on,queues=16
+ -device virtio-net-pci,host_mtu=1460,netdev=r0-node1,mac=placemat,mq=on,vectors=34,romfile=
+ -netdev tap,id=r0-node2,ifname=%s,script=no,downscript=no,vhost=on,queues=16
+ -device virtio-net-pci,host_mtu=1460,netdev=r0-node2,mac=placemat,mq=on,vectors=34,romfile=
  -drive if=virtio,cache=writeback,aio=threads,file=%s/root.img
  -drive if=virtio,cache=none,aio=native,format=qcow2,file=%s/seed.img
  -virtfs local,path=%s,mount_tag=sabakan,security_model=none,readonly

diff --git a/v2/pkg/vm/tap.go b/v2/pkg/vm/tap.go
@@ -30,7 +30,7 @@ func newTap(bridgeName string) (*tap, error) {
 	}, nil
 }
 
-func (t *tap) create(mtu int) (*tapInfo, error) {
+func (t *tap) create(mtu, netDevQueue int) (*tapInfo, error) {
 	la := netlink.NewLinkAttrs()
 	name, err := dcnet.RandomLinkName(dcnet.LinkTypeTap)
 	if err != nil {
@@ -41,6 +41,10 @@ func (t *tap) create(mtu int) (*tapInfo, error) {
 		LinkAttrs: la,
 		Mode:      netlink.TUNTAP_MODE_TAP,
 	}
+	if netDevQueue > 1 {
+		tap.Flags = netlink.TUNTAP_MULTI_QUEUE_DEFAULTS | netlink.TUNTAP_VNET_HDR
+		tap.Queues = netDevQueue
+	}
 	if err := netlink.LinkAdd(tap); err != nil {
 		return nil, fmt.Errorf("failed to add the tap %s: %w", name, err)
 	}

diff --git a/v2/pkg/vm/tap_test.go b/v2/pkg/vm/tap_test.go
@@ -37,7 +37,7 @@ use-nat: false
 
 		tap, err := newTap("r0-node1")
 		Expect(err).NotTo(HaveOccurred())
-		tapInfo, err := tap.create(1460)
+		tapInfo, err := tap.create(1460, 4)
 		Expect(err).NotTo(HaveOccurred())
 		defer tap.Cleanup()
 
@@ -65,7 +65,7 @@ use-nat: false
 
 		tap, err := newTap("r0-node1")
 		Expect(err).NotTo(HaveOccurred())
-		tapInfo, err := tap.create(0)
+		tapInfo, err := tap.create(0, 0)
 		Expect(err).NotTo(HaveOccurred())
 		defer tap.Cleanup()