Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

add simple NUMA support #164

Merged
merged 2 commits into from
Apr 22, 2022
Merged

add simple NUMA support #164

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
38d13cc
add simple NUMA support
umezawatakeshi Apr 20, 2022
da719f6
add behavior in case of `numa` is omitted.
umezawatakeshi Apr 21, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions docs/resource.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,8 @@ The properties are:
- `maxcpus`: The amount of maximum hotpluggable CPUs. (CPU hotplug is not tested in placemat)
- `cpu`: The amount of virtual CPUs. Compatibility for older placemat and exclusive with `smp`.
- `memory`: The amount of memory.
- `numa`: The NUMA configuration. At present, only supports simple symmetric configuration: the amount of cpus and memory are same for all NUMA nodes and all the distances between NUMA nodes are same.
morimoto-cybozu marked this conversation as resolved.
Show resolved Hide resolved
- `nodes`: The number of NUMA nodes.
- `network-device-queue`: The count of VM's network device queue. Placemat enables multi queue virtio-net if network-device-queue is greater than 1.
- `smbios`: System Management BIOS (SMBIOS) values for `manufacturer`, `product`, and `serial`. If `serial` is not set, a hash value of the node's name is used.
- `uefi`: BIOS mode of the VM.
Expand Down
6 changes: 6 additions & 0 deletions v2/pkg/types/cluster.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,11 @@ type SMPSpec struct {
MaxCPUs int `json:"maxcpus,omitempty"`
}

type NUMASpec struct {
// Nodes represents the number of NUMA nodes. Used in case of simple symmetric NUMA configuration.
Nodes int `json:"nodes,omitempty"`
}

// NodeSpec represents a Node specification in YAML
type NodeSpec struct {
Kind string `json:"kind"`
Expand All @@ -182,6 +187,7 @@ type NodeSpec struct {
CPU int `json:"cpu,omitempty"` // compatibility use
SMP *SMPSpec `json:"smp,omitempty"`
Memory string `json:"memory,omitempty"`
NUMA NUMASpec `json:"numa,omitempty"`
NetworkDeviceQueue int `json:"network-device-queue,omitempty"`
UEFI bool `json:"uefi,omitempty"`
TPM bool `json:"tpm,omitempty"`
Expand Down
7 changes: 6 additions & 1 deletion v2/pkg/types/cluster_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,8 @@ smp:
dies: 6
sockets: 4
maxcpus: 100
numa:
nodes: 12
network-device-queue: 16
smbios:
manufacturer: cybozu
Expand Down Expand Up @@ -175,7 +177,10 @@ file: cybozu-ubuntu-18.04-server-cloudimg-amd64.img
Sockets: 4,
MaxCPUs: 100,
},
Memory: "2G",
Memory: "2G",
NUMA: NUMASpec{
Nodes: 12,
},
NetworkDeviceQueue: 16,
UEFI: false,
TPM: true,
Expand Down
12 changes: 10 additions & 2 deletions v2/pkg/vm/node.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ type node struct {
ignitionFile string
smp smpSpec
memory string
numa numaSpec
networkDeviceQueue int
uefi bool
tpm bool
Expand All @@ -59,6 +60,10 @@ type smpSpec struct {
maxCpus int
}

type numaSpec struct {
nodes int
}

// NewNode creates a Node from spec.
func NewNode(spec *types.NodeSpec, imageSpecs []*types.ImageSpec, deviceClassSpecs []*types.DeviceClassSpec) (Node, error) {
n := &node{
Expand All @@ -72,7 +77,10 @@ func NewNode(spec *types.NodeSpec, imageSpecs []*types.ImageSpec, deviceClassSpe
sockets: spec.SMP.Sockets,
maxCpus: spec.SMP.MaxCPUs,
},
memory: spec.Memory,
memory: spec.Memory,
numa: numaSpec{
nodes: spec.NUMA.Nodes,
},
networkDeviceQueue: spec.NetworkDeviceQueue,
uefi: spec.UEFI,
tpm: spec.TPM,
Expand Down Expand Up @@ -144,7 +152,7 @@ func (n *node) Setup(ctx context.Context, r *Runtime, mtu int, nodeCh chan<- BMC
}
}

qemu := newQemu(n.name, tapInfos, vArgs, n.ignitionFile, n.smp, n.memory, n.networkDeviceQueue, n.uefi, n.tpm, n.smbios)
qemu := newQemu(n.name, tapInfos, vArgs, n.ignitionFile, n.smp, n.memory, n.numa, n.networkDeviceQueue, n.uefi, n.tpm, n.smbios)
c := qemu.command(r)
qemuCommand := well.CommandContext(ctx, c[0], c[1:]...)
qemuCommand.Stdout = util.NewColoredLogWriter("qemu", n.name, os.Stdout)
Expand Down
10 changes: 9 additions & 1 deletion v2/pkg/vm/qemu.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ type qemu struct {
ignitionFile string
smp smpSpec
memory string
numa numaSpec
networkDeviceQueue int
uefi bool
tpm bool
Expand All @@ -32,14 +33,15 @@ type qemu struct {
}

func newQemu(nodeName string, taps []*tapInfo, volumes []volumeArgs, ignitionFile string, smp smpSpec,
memory string, networkDeviceQueue int, uefi bool, tpm bool, smbios smBIOSConfig) *qemu {
memory string, numa numaSpec, networkDeviceQueue int, uefi bool, tpm bool, smbios smBIOSConfig) *qemu {
return &qemu{
name: nodeName,
taps: taps,
volumes: volumes,
ignitionFile: ignitionFile,
smp: smp,
memory: memory,
numa: numa,
networkDeviceQueue: networkDeviceQueue,
uefi: uefi,
tpm: tpm,
Expand Down Expand Up @@ -139,6 +141,12 @@ func (c *qemu) qemuParams(r *Runtime) []string {
if c.memory != "" {
params = append(params, "-m", c.memory)
}
if c.numa.nodes != 0 {
cpuPerNode := c.smp.cpus / c.numa.nodes
for i := 0; i < c.numa.nodes; i++ {
params = append(params, "-numa", fmt.Sprintf("node,cpus=%d-%d", cpuPerNode*i, cpuPerNode*(i+1)-1))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure whether this is valid for the latest QEMU.
It would be OK to care only after a user has complained about this.
https://www.qemu.org/docs/master/about/removed-features.html#numa-node-without-memory-specified-removed-in-5-2

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm... it may seem a problem. Ubuntu 20.04 uses QEMU 4.2 but Ubuntu 22.04 uses QEMU 6.2.

}
}
if !r.Graphic {
p := r.socketPath(c.name)
params = append(params, "-nographic")
Expand Down
20 changes: 16 additions & 4 deletions v2/pkg/vm/qemu_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,13 +49,15 @@ var _ = Describe("QEMU command builder", func() {
kind: Node
name: boot-0
smp:
cpus: 8
cpus: 72
cores: 3
threads: 2
dies: 6
sockets: 4
maxcpus: 100
memory: 2G
numa:
nodes: 6
network-device-queue: 16
interfaces:
- r0-node1
Expand Down Expand Up @@ -145,7 +147,9 @@ use-nat: false
dies: nodeSpec.SMP.Dies,
sockets: nodeSpec.SMP.Sockets,
maxCpus: nodeSpec.SMP.MaxCPUs,
}, nodeSpec.Memory, nodeSpec.NetworkDeviceQueue, nodeSpec.UEFI, nodeSpec.TPM, smBIOSConfig{
}, nodeSpec.Memory, numaSpec{
nodes: nodeSpec.NUMA.Nodes,
}, nodeSpec.NetworkDeviceQueue, nodeSpec.UEFI, nodeSpec.TPM, smBIOSConfig{
manufacturer: nodeSpec.SMBIOS.Manufacturer,
product: nodeSpec.SMBIOS.Product,
serial: nodeSpec.SMBIOS.Serial,
Expand All @@ -156,8 +160,14 @@ use-nat: false
expected := strings.ReplaceAll(fmt.Sprintf(`
qemu-system-x86_64
-enable-kvm
-smp 8,cores=3,threads=2,dies=6,sockets=4,maxcpus=100
-smp 72,cores=3,threads=2,dies=6,sockets=4,maxcpus=100
-m 2G
-numa node,cpus=0-11
-numa node,cpus=12-23
-numa node,cpus=24-35
-numa node,cpus=36-47
-numa node,cpus=48-59
-numa node,cpus=60-71
-nographic
-serial unix:%s/boot-0.socket,server,nowait
-smbios type=1,serial=fb8f2417d0b4db30050719c31ce02a2e8141bbd8
Expand Down Expand Up @@ -300,7 +310,9 @@ use-nat: false
dies: nodeSpec.SMP.Dies,
sockets: nodeSpec.SMP.Sockets,
maxCpus: nodeSpec.SMP.MaxCPUs,
}, nodeSpec.Memory, nodeSpec.NetworkDeviceQueue, nodeSpec.UEFI, nodeSpec.TPM, smBIOSConfig{
}, nodeSpec.Memory, numaSpec{
nodes: nodeSpec.NUMA.Nodes,
}, nodeSpec.NetworkDeviceQueue, nodeSpec.UEFI, nodeSpec.TPM, smBIOSConfig{
manufacturer: nodeSpec.SMBIOS.Manufacturer,
product: nodeSpec.SMBIOS.Product,
serial: nodeSpec.SMBIOS.Serial,
Expand Down