From 284ad62394d18597db95c0d9688895090e90474d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sun, 20 Feb 2022 10:15:48 +0100 Subject: [PATCH 01/18] Add the qemu driver to the minikube registry Since the machine drivers are hardcoded in minikube, drivers need to be added to the registry to be tested. Add a basic sanity check for the qemu-system binary, and set up the basic configuration such as cpus/memory. --- go.mod | 1 + go.sum | 2 + pkg/minikube/driver/driver.go | 2 + pkg/minikube/driver/driver_darwin.go | 2 + pkg/minikube/driver/driver_linux.go | 1 + pkg/minikube/driver/driver_test.go | 1 + pkg/minikube/driver/driver_windows.go | 1 + pkg/minikube/registry/drvs/init.go | 1 + pkg/minikube/registry/drvs/qemu/doc.go | 17 +++++ pkg/minikube/registry/drvs/qemu/qemu.go | 83 +++++++++++++++++++++++++ site/content/en/docs/drivers/_index.md | 3 + site/content/en/docs/drivers/qemu.md | 22 +++++++ 12 files changed, 136 insertions(+) create mode 100644 pkg/minikube/registry/drvs/qemu/doc.go create mode 100644 pkg/minikube/registry/drvs/qemu/qemu.go create mode 100644 site/content/en/docs/drivers/qemu.md diff --git a/go.mod b/go.mod index 074379731926..0e35693fa30c 100644 --- a/go.mod +++ b/go.mod @@ -155,6 +155,7 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.13.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/machine-drivers/docker-machine-driver-qemu v0.1.0 github.com/magiconair/properties v1.8.5 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect diff --git a/go.sum b/go.sum index a3972ba0e29b..c2fbcf795d47 100644 --- a/go.sum +++ b/go.sum @@ -756,6 +756,8 @@ github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9 github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/machine-drivers/docker-machine-driver-qemu v0.1.0 h1:r5bMzOqca0dZuCkn/8vCM8QNpdsLPCVZ6VP5z/IU0HM= +github.com/machine-drivers/docker-machine-driver-qemu v0.1.0/go.mod h1:1gevpWYcs2Gmvt9G9T6AbeRw9tGnQdQiKt56B7n2X7g= github.com/machine-drivers/docker-machine-driver-vmware v0.1.5 h1:51GqJ84u9EBATnn8rWsHNavcuRPlCLnDmvjzZVuliwY= github.com/machine-drivers/docker-machine-driver-vmware v0.1.5/go.mod h1:dTnTzUH3uzhMo0ddV1zRjGYWcVhQWwqiHPxz5l+HPd0= github.com/machine-drivers/machine v0.7.1-0.20211105063445-78a84df85426 h1:gVDPCmqwvHQ4ox/9svvnkomYJAAiV59smbPdTK4DIm4= diff --git a/pkg/minikube/driver/driver.go b/pkg/minikube/driver/driver.go index 0b2e317c93f6..9bde2a04040f 100644 --- a/pkg/minikube/driver/driver.go +++ b/pkg/minikube/driver/driver.go @@ -46,6 +46,8 @@ const ( SSH = "ssh" // KVM2 driver KVM2 = "kvm2" + // QEMU driver + QEMU = "qemu" // VirtualBox driver VirtualBox = "virtualbox" // HyperKit driver diff --git a/pkg/minikube/driver/driver_darwin.go b/pkg/minikube/driver/driver_darwin.go index 195f9841c0af..505b97ff0bbd 100644 --- a/pkg/minikube/driver/driver_darwin.go +++ b/pkg/minikube/driver/driver_darwin.go @@ -27,6 +27,7 @@ var supportedDrivers = func() []string { if runtime.GOARCH == "arm64" { // on darwin/arm64 only docker and ssh are supported yet return []string{ + QEMU, Docker, Podman, SSH, @@ -50,6 +51,7 @@ var supportedDrivers = func() []string { VMwareFusion, HyperKit, VMware, + QEMU, Docker, Podman, SSH, diff --git a/pkg/minikube/driver/driver_linux.go b/pkg/minikube/driver/driver_linux.go index a428a9a2bfe0..9a5b4695ab72 100644 --- a/pkg/minikube/driver/driver_linux.go +++ b/pkg/minikube/driver/driver_linux.go @@ -25,6 +25,7 @@ var supportedDrivers = []string{ VirtualBox, VMwareFusion, KVM2, + QEMU, VMware, None, Docker, diff --git a/pkg/minikube/driver/driver_test.go b/pkg/minikube/driver/driver_test.go index 021f78c74c8e..52b0be0f711e 100644 --- a/pkg/minikube/driver/driver_test.go +++ b/pkg/minikube/driver/driver_test.go @@ -66,6 +66,7 @@ func TestMachineType(t *testing.T) { None: "bare metal machine", SSH: "bare metal machine", KVM2: "VM", + QEMU: "VM", VirtualBox: "VM", HyperKit: "VM", VMware: "VM", diff --git a/pkg/minikube/driver/driver_windows.go b/pkg/minikube/driver/driver_windows.go index 8352366ae29b..473ee45ae9ca 100644 --- a/pkg/minikube/driver/driver_windows.go +++ b/pkg/minikube/driver/driver_windows.go @@ -32,6 +32,7 @@ var supportedDrivers = []string{ VMwareFusion, HyperV, VMware, + QEMU, Docker, Podman, SSH, diff --git a/pkg/minikube/registry/drvs/init.go b/pkg/minikube/registry/drvs/init.go index 06b11158273f..40c9ff18a5f4 100644 --- a/pkg/minikube/registry/drvs/init.go +++ b/pkg/minikube/registry/drvs/init.go @@ -25,6 +25,7 @@ import ( _ "k8s.io/minikube/pkg/minikube/registry/drvs/none" _ "k8s.io/minikube/pkg/minikube/registry/drvs/parallels" _ "k8s.io/minikube/pkg/minikube/registry/drvs/podman" + _ "k8s.io/minikube/pkg/minikube/registry/drvs/qemu" _ "k8s.io/minikube/pkg/minikube/registry/drvs/ssh" _ "k8s.io/minikube/pkg/minikube/registry/drvs/virtualbox" _ "k8s.io/minikube/pkg/minikube/registry/drvs/vmware" diff --git a/pkg/minikube/registry/drvs/qemu/doc.go b/pkg/minikube/registry/drvs/qemu/doc.go new file mode 100644 index 000000000000..ea4425ce8d91 --- /dev/null +++ b/pkg/minikube/registry/drvs/qemu/doc.go @@ -0,0 +1,17 @@ +/* +Copyright 2018 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package qemu diff --git a/pkg/minikube/registry/drvs/qemu/qemu.go b/pkg/minikube/registry/drvs/qemu/qemu.go new file mode 100644 index 000000000000..bd4b9e7eca34 --- /dev/null +++ b/pkg/minikube/registry/drvs/qemu/qemu.go @@ -0,0 +1,83 @@ +/* +Copyright 2018 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package qemu + +import ( + "fmt" + "os/exec" + "runtime" + + "github.com/docker/machine/libmachine/drivers" + drvqemu "github.com/machine-drivers/docker-machine-driver-qemu" + + "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/download" + "k8s.io/minikube/pkg/minikube/driver" + "k8s.io/minikube/pkg/minikube/localpath" + "k8s.io/minikube/pkg/minikube/registry" +) + +const ( + docURL = "https://minikube.sigs.k8s.io/docs/reference/drivers/qemu/" +) + +func init() { + if err := registry.Register(registry.DriverDef{ + Name: driver.QEMU, + Config: configure, + Status: status, + Default: true, + Priority: registry.Experimental, + }); err != nil { + panic(fmt.Sprintf("register failed: %v", err)) + } +} + +func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) { + name := config.MachineName(cc, n) + return drvqemu.Driver{ + BaseDriver: &drivers.BaseDriver{ + MachineName: name, + StorePath: localpath.MiniPath(), + SSHUser: "docker", + }, + Boot2DockerURL: download.LocalISOResource(cc.MinikubeISO), + DiskSize: cc.DiskSize, + Memory: cc.Memory, + CPU: cc.CPUs, + }, nil +} + +func status() registry.State { + var qemuSystem string + arch := runtime.GOARCH + switch arch { + case "amd64": + qemuSystem = "qemu-system-x86_64" + case "arm64": + qemuSystem = "qemu-system-aarch64" + default: + return registry.State{Error: fmt.Errorf("unknown arch: %s", arch), Doc: docURL} + } + + _, err := exec.LookPath(qemuSystem) + if err != nil { + return registry.State{Error: err, Fix: "Install qemu-system", Doc: docURL} + } + + return registry.State{Installed: true, Healthy: true, Running: true} +} diff --git a/site/content/en/docs/drivers/_index.md b/site/content/en/docs/drivers/_index.md index daf996e84fba..6744c03ff377 100644 --- a/site/content/en/docs/drivers/_index.md +++ b/site/content/en/docs/drivers/_index.md @@ -17,6 +17,7 @@ To do so, we use the [Docker Machine](https://github.com/docker/machine) library * [Docker]({{}}) - container-based (preferred) * [KVM2]({{}}) - VM-based (preferred) * [VirtualBox]({{}}) - VM +* [QEMU]({{}}) - VM (experimental) * [None]({{}}) - bare-metal * [Podman]({{}}) - container (experimental) * [SSH]({{}}) - remote ssh @@ -29,6 +30,7 @@ To do so, we use the [Docker Machine](https://github.com/docker/machine) library * [VirtualBox]({{}}) - VM * [Parallels]({{}}) - VM * [VMware Fusion]({{}}) - VM +* [QEMU]({{}}) - VM (experimental) * [SSH]({{}}) - remote ssh ## Windows @@ -37,4 +39,5 @@ To do so, we use the [Docker Machine](https://github.com/docker/machine) library * [Docker]({{}}) - VM + Container (preferred) * [VirtualBox]({{}}) - VM * [VMware Workstation]({{}}) - VM +* [QEMU]({{}}) - VM (experimental) * [SSH]({{}}) - remote ssh diff --git a/site/content/en/docs/drivers/qemu.md b/site/content/en/docs/drivers/qemu.md new file mode 100644 index 000000000000..23167b1e5b32 --- /dev/null +++ b/site/content/en/docs/drivers/qemu.md @@ -0,0 +1,22 @@ +--- +title: "qemu" +weight: 3 +description: > + QEMU driver +aliases: + - /docs/reference/drivers/qemu +--- + +## Overview + +The `qemu` driver users QEMU (system) for VM creation. + + + +## Issues + +* [Full list of open 'qemu' driver issues](https://github.com/kubernetes/minikube/labels/co%2Fqemu-driver) + +## Troubleshooting + +* Run `minikube start --alsologtostderr -v=4` to debug crashes From 98b8b6a0b41c14fdccead8aadb7eab2f189e5605 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sun, 20 Feb 2022 12:41:03 +0100 Subject: [PATCH 02/18] Upgrade to latest docker-machine-driver-qemu Copy some of the default values from the regular setup, that is not used when not using docker-machine config. --- go.mod | 2 +- go.sum | 2 ++ pkg/minikube/registry/drvs/qemu/qemu.go | 4 ++++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 0e35693fa30c..d617d5a16e37 100644 --- a/go.mod +++ b/go.mod @@ -155,7 +155,7 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.13.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect - github.com/machine-drivers/docker-machine-driver-qemu v0.1.0 + github.com/machine-drivers/docker-machine-driver-qemu v0.1.1-0.20180730160045-1d6b42f755d0 github.com/magiconair/properties v1.8.5 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect diff --git a/go.sum b/go.sum index c2fbcf795d47..a8c1134e5fbe 100644 --- a/go.sum +++ b/go.sum @@ -758,6 +758,8 @@ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/machine-drivers/docker-machine-driver-qemu v0.1.0 h1:r5bMzOqca0dZuCkn/8vCM8QNpdsLPCVZ6VP5z/IU0HM= github.com/machine-drivers/docker-machine-driver-qemu v0.1.0/go.mod h1:1gevpWYcs2Gmvt9G9T6AbeRw9tGnQdQiKt56B7n2X7g= +github.com/machine-drivers/docker-machine-driver-qemu v0.1.1-0.20180730160045-1d6b42f755d0 h1:1Yo+YdvX8hKRqAwFPLcR58DJgOhbZ1Sh2LPP1xtIMVA= +github.com/machine-drivers/docker-machine-driver-qemu v0.1.1-0.20180730160045-1d6b42f755d0/go.mod h1:1gevpWYcs2Gmvt9G9T6AbeRw9tGnQdQiKt56B7n2X7g= github.com/machine-drivers/docker-machine-driver-vmware v0.1.5 h1:51GqJ84u9EBATnn8rWsHNavcuRPlCLnDmvjzZVuliwY= github.com/machine-drivers/docker-machine-driver-vmware v0.1.5/go.mod h1:dTnTzUH3uzhMo0ddV1zRjGYWcVhQWwqiHPxz5l+HPd0= github.com/machine-drivers/machine v0.7.1-0.20211105063445-78a84df85426 h1:gVDPCmqwvHQ4ox/9svvnkomYJAAiV59smbPdTK4DIm4= diff --git a/pkg/minikube/registry/drvs/qemu/qemu.go b/pkg/minikube/registry/drvs/qemu/qemu.go index bd4b9e7eca34..e9f13016603b 100644 --- a/pkg/minikube/registry/drvs/qemu/qemu.go +++ b/pkg/minikube/registry/drvs/qemu/qemu.go @@ -19,6 +19,7 @@ package qemu import ( "fmt" "os/exec" + "path/filepath" "runtime" "github.com/docker/machine/libmachine/drivers" @@ -59,6 +60,9 @@ func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) { DiskSize: cc.DiskSize, Memory: cc.Memory, CPU: cc.CPUs, + EnginePort: 2376, + FirstQuery: true, + DiskPath: filepath.Join(localpath.MiniPath(), "machines", name, fmt.Sprintf("%s.img", name)), }, nil } From d9c303e6423ba2334af00b62c338f1f2fa94a256 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Tue, 22 Feb 2022 08:05:41 +0100 Subject: [PATCH 03/18] Use portforwarding for qemu with user network Only the SSH and the Docker port (2376) are available with the slirp network, everything else needs tunneling. --- pkg/minikube/driver/driver.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/minikube/driver/driver.go b/pkg/minikube/driver/driver.go index 9bde2a04040f..40bf67f554a4 100644 --- a/pkg/minikube/driver/driver.go +++ b/pkg/minikube/driver/driver.go @@ -183,6 +183,9 @@ func AllowsPreload(driverName string) bool { // NeedsPortForward returns true if driver is unable provide direct IP connectivity func NeedsPortForward(name string) bool { + if name == QEMU { + return true + } if !IsKIC(name) { return false } From 927d7d5596408756317021249015303b4b3f8bb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sat, 2 Apr 2022 10:18:42 +0200 Subject: [PATCH 04/18] Upgrade to latest docker-machine-driver-qemu It is still ancient, so there is bound to be issues with the vendored code from libmachine and drivers. --- go.mod | 2 +- go.sum | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index d617d5a16e37..ed2a36fc236d 100644 --- a/go.mod +++ b/go.mod @@ -155,7 +155,7 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.13.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect - github.com/machine-drivers/docker-machine-driver-qemu v0.1.1-0.20180730160045-1d6b42f755d0 + github.com/machine-drivers/docker-machine-driver-qemu v0.1.1-0.20220331133007-0324171328f7 github.com/magiconair/properties v1.8.5 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect diff --git a/go.sum b/go.sum index a8c1134e5fbe..2a345375d417 100644 --- a/go.sum +++ b/go.sum @@ -364,6 +364,7 @@ github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TT github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v0.0.0-20180621001606-093424bec097/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v17.12.0-ce-rc1.0.20181225093023-5ddb1d410a8b+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v17.12.0-ce-rc1.0.20190115220918-5ec31380a5d3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v20.10.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= @@ -760,6 +761,8 @@ github.com/machine-drivers/docker-machine-driver-qemu v0.1.0 h1:r5bMzOqca0dZuCkn github.com/machine-drivers/docker-machine-driver-qemu v0.1.0/go.mod h1:1gevpWYcs2Gmvt9G9T6AbeRw9tGnQdQiKt56B7n2X7g= github.com/machine-drivers/docker-machine-driver-qemu v0.1.1-0.20180730160045-1d6b42f755d0 h1:1Yo+YdvX8hKRqAwFPLcR58DJgOhbZ1Sh2LPP1xtIMVA= github.com/machine-drivers/docker-machine-driver-qemu v0.1.1-0.20180730160045-1d6b42f755d0/go.mod h1:1gevpWYcs2Gmvt9G9T6AbeRw9tGnQdQiKt56B7n2X7g= +github.com/machine-drivers/docker-machine-driver-qemu v0.1.1-0.20220331133007-0324171328f7 h1:f9xnae3LZMVUXFJtqy1xuwQfwX+NQUS5LelCLM3RBxg= +github.com/machine-drivers/docker-machine-driver-qemu v0.1.1-0.20220331133007-0324171328f7/go.mod h1:yhDK3dYTcmZljNMDPXfmVRwSsHx1EoaEL32v7BANaYs= github.com/machine-drivers/docker-machine-driver-vmware v0.1.5 h1:51GqJ84u9EBATnn8rWsHNavcuRPlCLnDmvjzZVuliwY= github.com/machine-drivers/docker-machine-driver-vmware v0.1.5/go.mod h1:dTnTzUH3uzhMo0ddV1zRjGYWcVhQWwqiHPxz5l+HPd0= github.com/machine-drivers/machine v0.7.1-0.20211105063445-78a84df85426 h1:gVDPCmqwvHQ4ox/9svvnkomYJAAiV59smbPdTK4DIm4= @@ -1012,6 +1015,7 @@ github.com/shirou/gopsutil/v3 v3.22.2 h1:wCrArWFkHYIdDxx/FSfF5RB4dpJYW6t7rcp3+zL github.com/shirou/gopsutil/v3 v3.22.2/go.mod h1:WapW1AOOPlHyXr+yOyw3uYx36enocrtSoSBy0L5vUHY= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= +github.com/sirupsen/logrus v1.0.4/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -1184,6 +1188,7 @@ go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= golang.org/x/build v0.0.0-20190927031335-2835ba2e683f h1:hXVePvSFG7tPGX4Pwk1d10ePFfoTCc0QmISfpKOHsS8= golang.org/x/build v0.0.0-20190927031335-2835ba2e683f/go.mod h1:fYw7AShPAhGMdXqA9gRadk/CcMsvLlClpE5oBwnS3dM= +golang.org/x/crypto v0.0.0-20170704135851-51714a8c4ac1/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -1353,6 +1358,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180202135801-37707fdb30a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= From 77ed383578be615b2a1f3e8d064c875a66bbb33e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sun, 3 Apr 2022 18:14:43 +0200 Subject: [PATCH 05/18] Fork the machine qemu driver into minikube qemu2 This is an internal driver, with the new driver config rather than the machine flags of the external driver. --- go.mod | 2 +- pkg/drivers/qemu/qemu.go | 739 ++++++++++++++++++++++ pkg/minikube/driver/driver.go | 9 +- pkg/minikube/driver/driver_darwin.go | 4 +- pkg/minikube/driver/driver_linux.go | 1 + pkg/minikube/driver/driver_windows.go | 2 +- pkg/minikube/node/start.go | 2 +- pkg/minikube/registry/drvs/init.go | 1 + pkg/minikube/registry/drvs/qemu/qemu.go | 14 +- pkg/minikube/registry/drvs/qemu2/doc.go | 17 + pkg/minikube/registry/drvs/qemu2/qemu2.go | 103 +++ 11 files changed, 875 insertions(+), 19 deletions(-) create mode 100644 pkg/drivers/qemu/qemu.go create mode 100644 pkg/minikube/registry/drvs/qemu2/doc.go create mode 100644 pkg/minikube/registry/drvs/qemu2/qemu2.go diff --git a/go.mod b/go.mod index ed2a36fc236d..a0093835ec2e 100644 --- a/go.mod +++ b/go.mod @@ -47,6 +47,7 @@ require ( github.com/juju/version v0.0.0-20180108022336-b64dbd566305 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 github.com/klauspost/cpuid v1.2.0 + github.com/machine-drivers/docker-machine-driver-qemu v0.1.1-0.20220331133007-0324171328f7 github.com/machine-drivers/docker-machine-driver-vmware v0.1.5 github.com/mattbaird/jsonpatch v0.0.0-20200820163806-098863c1fc24 github.com/mattn/go-isatty v0.0.14 @@ -155,7 +156,6 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.13.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect - github.com/machine-drivers/docker-machine-driver-qemu v0.1.1-0.20220331133007-0324171328f7 github.com/magiconair/properties v1.8.5 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect diff --git a/pkg/drivers/qemu/qemu.go b/pkg/drivers/qemu/qemu.go new file mode 100644 index 000000000000..50fd4992bcec --- /dev/null +++ b/pkg/drivers/qemu/qemu.go @@ -0,0 +1,739 @@ +/* +Copyright 2018 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package qemu + +import ( + "archive/tar" + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "math/rand" + "net" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "syscall" + "time" + + "github.com/docker/machine/libmachine/drivers" + "github.com/docker/machine/libmachine/log" + "github.com/docker/machine/libmachine/mcnutils" + "github.com/docker/machine/libmachine/ssh" + "github.com/docker/machine/libmachine/state" + + pkgdrivers "k8s.io/minikube/pkg/drivers" +) + +const ( + isoFilename = "boot2docker.iso" + privateNetworkName = "docker-machines" + + defaultSSHUser = "docker" +) + +type Driver struct { + *drivers.BaseDriver + *pkgdrivers.CommonDriver + EnginePort int + FirstQuery bool + + Memory int + DiskSize int + CPU int + Program string + Display bool + DisplayType string + Nographic bool + VirtioDrives bool + Network string + PrivateNetwork string + Boot2DockerURL string + NetworkInterface string + NetworkAddress string + NetworkBridge string + CaCertPath string + PrivateKeyPath string + DiskPath string + CacheMode string + IOMode string + connectionString string + // conn *libvirt.Connect + // VM *libvirt.Domain + vmLoaded bool + UserDataFile string + CloudConfigRoot string + LocalPorts string +} + +func (d *Driver) GetMachineName() string { + return d.MachineName +} + +func (d *Driver) GetSSHHostname() (string, error) { + return "localhost", nil + //return d.GetIP() +} + +func (d *Driver) GetSSHKeyPath() string { + return d.ResolveStorePath("id_rsa") +} + +func (d *Driver) GetSSHPort() (int, error) { + if d.SSHPort == 0 { + d.SSHPort = 22 + } + + return d.SSHPort, nil +} + +func (d *Driver) GetSSHUsername() string { + if d.SSHUser == "" { + d.SSHUser = "docker" + } + + return d.SSHUser +} + +func (d *Driver) DriverName() string { + return "qemu2" +} + +func (d *Driver) GetURL() (string, error) { + log.Debugf("GetURL called") + if _, err := os.Stat(d.pidfilePath()); err != nil { + return "", nil + } + ip, err := d.GetIP() + if err != nil { + log.Warnf("Failed to get IP: %s", err) + return "", err + } + if ip == "" { + return "", nil + } + port := d.GetPort() + return fmt.Sprintf("tcp://%s:%d", ip, port), nil +} + +func NewDriver(hostName, storePath string) drivers.Driver { + return &Driver{ + PrivateNetwork: privateNetworkName, + BaseDriver: &drivers.BaseDriver{ + SSHUser: defaultSSHUser, + MachineName: hostName, + StorePath: storePath, + }, + } +} + +func (d *Driver) GetIP() (string, error) { + if d.Network == "user" { + return "127.0.0.1", nil + } + return d.NetworkAddress, nil +} + +func (d *Driver) GetPort() int { + var port = d.EnginePort + if d.FirstQuery { + d.FirstQuery = false + port = 2376 + } + return port +} + +func checkPid(pid int) error { + process, err := os.FindProcess(pid) + if err != nil { + return err + } + return process.Signal(syscall.Signal(0)) +} + +func (d *Driver) GetState() (state.State, error) { + + if _, err := os.Stat(d.pidfilePath()); err != nil { + return state.Stopped, nil + } + p, err := ioutil.ReadFile(d.pidfilePath()) + if err != nil { + return state.Error, err + } + pid, err := strconv.Atoi(strings.TrimSpace(string(p))) + if err != nil { + return state.Error, err + } + if err := checkPid(pid); err != nil { + // No pid, remove pidfile + os.Remove(d.pidfilePath()) + return state.Stopped, nil + } + ret, err := d.RunQMPCommand("query-status") + if err != nil { + return state.Error, err + } + // RunState is one of: + // 'debug', 'inmigrate', 'internal-error', 'io-error', 'paused', + // 'postmigrate', 'prelaunch', 'finish-migrate', 'restore-vm', + // 'running', 'save-vm', 'shutdown', 'suspended', 'watchdog', + // 'guest-panicked' + switch ret["status"] { + case "running": + return state.Running, nil + case "paused": + return state.Paused, nil + case "shutdown": + return state.Stopped, nil + } + return state.None, nil +} + +func (d *Driver) PreCreateCheck() error { + return nil +} + +func (d *Driver) Create() error { + var err error + if d.Network == "user" { + minPort, maxPort, err := parsePortRange(d.LocalPorts) + log.Debugf("port range: %d -> %d", minPort, maxPort) + if err != nil { + return err + } + d.SSHPort, err = getAvailableTCPPortFromRange(minPort, maxPort) + if err != nil { + return err + } + + for { + d.EnginePort, err = getAvailableTCPPortFromRange(minPort, maxPort) + if err != nil { + return err + } + if d.EnginePort == d.SSHPort { + // can't have both on same port + continue + } + break + } + } + b2dutils := mcnutils.NewB2dUtils(d.StorePath) + if err := b2dutils.CopyIsoToMachineDir(d.Boot2DockerURL, d.MachineName); err != nil { + return err + } + + log.Infof("Creating SSH key...") + if err := ssh.GenerateSSHKey(d.sshKeyPath()); err != nil { + return err + } + + log.Infof("Creating Disk image...") + if err := d.generateDiskImage(d.DiskSize); err != nil { + return err + } + + if d.UserDataFile != "" { + log.Infof("Creating Userdata Disk...") + if d.CloudConfigRoot, err = d.generateUserdataDisk(d.UserDataFile); err != nil { + return err + } + } + + log.Infof("Starting QEMU VM...") + return d.Start() +} + +func parsePortRange(rawPortRange string) (int, int, error) { + if rawPortRange == "" { + return 0, 65535, nil + } + + portRange := strings.Split(rawPortRange, "-") + + minPort, err := strconv.Atoi(portRange[0]) + if err != nil { + return 0, 0, fmt.Errorf("Invalid port range") + } + maxPort, err := strconv.Atoi(portRange[1]) + if err != nil { + return 0, 0, fmt.Errorf("Invalid port range") + } + + if maxPort < minPort { + return 0, 0, fmt.Errorf("Invalid port range") + } + + if maxPort-minPort < 2 { + return 0, 0, fmt.Errorf("Port range must be minimum 2 ports") + } + + return minPort, maxPort, nil +} + +func getRandomPortNumberInRange(min int, max int) int { + return rand.Intn(max-min) + min +} + +func getAvailableTCPPortFromRange(minPort int, maxPort int) (int, error) { + port := 0 + for i := 0; i <= 10; i++ { + var ln net.Listener + var err error + if minPort == 0 && maxPort == 65535 { + ln, err = net.Listen("tcp4", "127.0.0.1:0") + if err != nil { + return 0, err + } + } else { + port = getRandomPortNumberInRange(minPort, maxPort) + log.Debugf("testing port: %d", port) + ln, err = net.Listen("tcp4", fmt.Sprintf("127.0.0.1:%d", port)) + if err != nil { + log.Debugf("port already in use: %d", port) + continue + } + } + defer ln.Close() + addr := ln.Addr().String() + addrParts := strings.SplitN(addr, ":", 2) + p, err := strconv.Atoi(addrParts[1]) + if err != nil { + return 0, err + } + if p != 0 { + port = p + return port, nil + } + time.Sleep(1) + } + return 0, fmt.Errorf("unable to allocate tcp port") +} + +func (d *Driver) Start() error { + // fmt.Printf("Init qemu %s\n", i.VM) + machineDir := filepath.Join(d.StorePath, "machines", d.GetMachineName()) + + var startCmd []string + + if d.Display { + if d.DisplayType != "" { + startCmd = append(startCmd, + "-display", d.DisplayType, + ) + } else { + // Use the default graphic output + } + } else { + if d.Nographic { + startCmd = append(startCmd, + "-nographic", + ) + } else { + startCmd = append(startCmd, + "-display", "none", + ) + } + } + + startCmd = append(startCmd, + "-m", fmt.Sprintf("%d", d.Memory), + "-smp", fmt.Sprintf("%d", d.CPU), + "-boot", "d") + var isoPath = filepath.Join(machineDir, isoFilename) + if d.VirtioDrives { + startCmd = append(startCmd, + "-drive", fmt.Sprintf("file=%s,index=2,media=cdrom,if=virtio", isoPath)) + } else { + startCmd = append(startCmd, + "-cdrom", isoPath) + } + startCmd = append(startCmd, + "-qmp", fmt.Sprintf("unix:%s,server,nowait", d.monitorPath()), + "-pidfile", d.pidfilePath(), + ) + + if d.Network == "user" { + startCmd = append(startCmd, + "-nic", fmt.Sprintf("user,model=virtio,hostfwd=tcp::%d-:22,hostfwd=tcp::%d-:2376,hostname=%s", d.SSHPort, d.EnginePort, d.GetMachineName()), + ) + } else if d.Network == "tap" { + startCmd = append(startCmd, + "-nic", fmt.Sprintf("tap,model=virtio,ifname=%s,script=no,downscript=no", d.NetworkInterface), + ) + } else if d.Network == "bridge" { + startCmd = append(startCmd, + "-nic", fmt.Sprintf("bridge,model=virtio,br=%s", d.NetworkBridge), + ) + } else { + log.Errorf("Unknown network: %s", d.Network) + } + + startCmd = append(startCmd, "-daemonize") + + // other options + // "-enable-kvm" if its available + if _, err := os.Stat("/dev/kvm"); err == nil { + startCmd = append(startCmd, "-enable-kvm") + } + + if d.CloudConfigRoot != "" { + startCmd = append(startCmd, + "-fsdev", + fmt.Sprintf("local,security_model=passthrough,readonly,id=fsdev0,path=%s", d.CloudConfigRoot)) + startCmd = append(startCmd, "-device", "virtio-9p-pci,id=fs0,fsdev=fsdev0,mount_tag=config-2") + } + + if d.VirtioDrives { + startCmd = append(startCmd, + "-drive", fmt.Sprintf("file=%s,index=0,media=disk,if=virtio", d.diskPath())) + } else { + // last argument is always the name of the disk image + startCmd = append(startCmd, d.diskPath()) + } + + if stdout, stderr, err := cmdOutErr(d.Program, startCmd...); err != nil { + fmt.Printf("OUTPUT: %s\n", stdout) + fmt.Printf("ERROR: %s\n", stderr) + return err + //if err := cmdStart(d.Program, startCmd...); err != nil { + // return err + } + log.Infof("Waiting for VM to start (ssh -p %d docker@localhost)...", d.SSHPort) + + //return ssh.WaitForTCP(fmt.Sprintf("localhost:%d", d.SSHPort)) + return WaitForTCPWithDelay(fmt.Sprintf("localhost:%d", d.SSHPort), time.Second) +} + +func cmdOutErr(cmdStr string, args ...string) (string, string, error) { + cmd := exec.Command(cmdStr, args...) + log.Debugf("executing: %v %v", cmdStr, strings.Join(args, " ")) + var stdout bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + err := cmd.Run() + stderrStr := stderr.String() + log.Debugf("STDOUT: %v", stdout.String()) + log.Debugf("STDERR: %v", stderrStr) + if err != nil { + if ee, ok := err.(*exec.Error); ok && ee == exec.ErrNotFound { + err = fmt.Errorf("mystery error: %s", ee) + } + } else { + // also catch error messages in stderr, even if the return code + // looks OK + if strings.Contains(stderrStr, "error:") { + err = fmt.Errorf("%v %v failed: %v", cmdStr, strings.Join(args, " "), stderrStr) + } + } + return stdout.String(), stderrStr, err +} + +func cmdStart(cmdStr string, args ...string) error { + cmd := exec.Command(cmdStr, args...) + log.Debugf("executing: %v %v", cmdStr, strings.Join(args, " ")) + return cmd.Start() +} + +func (d *Driver) Stop() error { + // _, err := d.RunQMPCommand("stop") + _, err := d.RunQMPCommand("system_powerdown") + if err != nil { + return err + } + return nil +} + +func (d *Driver) Remove() error { + s, err := d.GetState() + if err != nil { + return err + } + if s == state.Running { + if err := d.Kill(); err != nil { + return err + } + } + if s != state.Stopped { + _, err = d.RunQMPCommand("quit") + if err != nil { + return err + } + } + return nil +} + +func (d *Driver) Restart() error { + s, err := d.GetState() + if err != nil { + return err + } + + if s == state.Running { + if err := d.Stop(); err != nil { + return err + } + } + return d.Start() +} + +func (d *Driver) Kill() error { + // _, err := d.RunQMPCommand("quit") + _, err := d.RunQMPCommand("system_powerdown") + if err != nil { + return err + } + return nil +} + +func (d *Driver) StartDocker() error { + return fmt.Errorf("hosts without a driver cannot start docker") +} + +func (d *Driver) StopDocker() error { + return fmt.Errorf("hosts without a driver cannot stop docker") +} + +func (d *Driver) GetDockerConfigDir() string { + return "" +} + +func (d *Driver) Upgrade() error { + return fmt.Errorf("hosts without a driver cannot be upgraded") +} + +//func (d *Driver) GetSSHCommand(args ...string) (*exec.Cmd, error) { +// return ssh.GetSSHCommand("localhost", d.SSHPort, "docker", d.sshKeyPath(), args...), nil +//} + +func (d *Driver) sshKeyPath() string { + machineDir := filepath.Join(d.StorePath, "machines", d.GetMachineName()) + return filepath.Join(machineDir, "id_rsa") +} + +func (d *Driver) publicSSHKeyPath() string { + return d.sshKeyPath() + ".pub" +} + +func (d *Driver) diskPath() string { + machineDir := filepath.Join(d.StorePath, "machines", d.GetMachineName()) + return filepath.Join(machineDir, "disk.qcow2") +} + +func (d *Driver) monitorPath() string { + machineDir := filepath.Join(d.StorePath, "machines", d.GetMachineName()) + return filepath.Join(machineDir, "monitor") +} + +func (d *Driver) pidfilePath() string { + machineDir := filepath.Join(d.StorePath, "machines", d.GetMachineName()) + return filepath.Join(machineDir, "qemu.pid") +} + +// Make a boot2docker VM disk image. +func (d *Driver) generateDiskImage(size int) error { + log.Debugf("Creating %d MB hard disk image...", size) + + magicString := "boot2docker, please format-me" + + buf := new(bytes.Buffer) + tw := tar.NewWriter(buf) + + // magicString first so the automount script knows to format the disk + file := &tar.Header{Name: magicString, Size: int64(len(magicString))} + if err := tw.WriteHeader(file); err != nil { + return err + } + if _, err := tw.Write([]byte(magicString)); err != nil { + return err + } + // .ssh/key.pub => authorized_keys + file = &tar.Header{Name: ".ssh", Typeflag: tar.TypeDir, Mode: 0700} + if err := tw.WriteHeader(file); err != nil { + return err + } + pubKey, err := ioutil.ReadFile(d.publicSSHKeyPath()) + if err != nil { + return err + } + file = &tar.Header{Name: ".ssh/authorized_keys", Size: int64(len(pubKey)), Mode: 0644} + if err := tw.WriteHeader(file); err != nil { + return err + } + if _, err := tw.Write([]byte(pubKey)); err != nil { + return err + } + file = &tar.Header{Name: ".ssh/authorized_keys2", Size: int64(len(pubKey)), Mode: 0644} + if err := tw.WriteHeader(file); err != nil { + return err + } + if _, err := tw.Write([]byte(pubKey)); err != nil { + return err + } + if err := tw.Close(); err != nil { + return err + } + rawFile := fmt.Sprintf("%s.raw", d.diskPath()) + if err := ioutil.WriteFile(rawFile, buf.Bytes(), 0644); err != nil { + return nil + } + if stdout, stderr, err := cmdOutErr("qemu-img", "convert", "-f", "raw", "-O", "qcow2", rawFile, d.diskPath()); err != nil { + fmt.Printf("OUTPUT: %s\n", stdout) + fmt.Printf("ERROR: %s\n", stderr) + return err + } + if stdout, stderr, err := cmdOutErr("qemu-img", "resize", d.diskPath(), fmt.Sprintf("+%dM", size)); err != nil { + fmt.Printf("OUTPUT: %s\n", stdout) + fmt.Printf("ERROR: %s\n", stderr) + return err + } + log.Debugf("DONE writing to %s and %s", rawFile, d.diskPath()) + + return nil +} + +func (d *Driver) generateUserdataDisk(userdataFile string) (string, error) { + // Start with virtio, add ISO & FAT format later + // Start with local file, add wget/fetct URL? (or if URL, use datasource..) + userdata, err := ioutil.ReadFile(userdataFile) + if err != nil { + return "", err + } + + machineDir := filepath.Join(d.StorePath, "machines", d.GetMachineName()) + ccRoot := filepath.Join(machineDir, "cloud-config") + os.MkdirAll(ccRoot, 0755) + + userDataDir := filepath.Join(ccRoot, "openstack/latest") + os.MkdirAll(userDataDir, 0755) + + writeFile := filepath.Join(userDataDir, "user_data") + if err := ioutil.WriteFile(writeFile, userdata, 0644); err != nil { + return "", err + } + + return ccRoot, nil + +} + +func (d *Driver) RunQMPCommand(command string) (map[string]interface{}, error) { + + // connect to monitor + conn, err := net.Dial("unix", d.monitorPath()) + if err != nil { + return nil, err + } + defer conn.Close() + + // initial QMP response + var buf [1024]byte + nr, err := conn.Read(buf[:]) + if err != nil { + return nil, err + } + type qmpInitialResponse struct { + QMP struct { + Version struct { + QEMU struct { + Micro int `json:"micro"` + Minor int `json:"minor"` + Major int `json:"major"` + } `json:"qemu"` + Package string `json:"package"` + } `json:"version"` + Capabilities []string `json:"capabilities"` + } `jason:"QMP"` + } + + var initialResponse qmpInitialResponse + json.Unmarshal(buf[:nr], &initialResponse) + + // run 'qmp_capabilities' to switch to command mode + // { "execute": "qmp_capabilities" } + type qmpCommand struct { + Command string `json:"execute"` + } + jsonCommand, err := json.Marshal(qmpCommand{Command: "qmp_capabilities"}) + if err != nil { + return nil, err + } + _, err = conn.Write(jsonCommand) + if err != nil { + return nil, err + } + nr, err = conn.Read(buf[:]) + if err != nil { + return nil, err + } + type qmpResponse struct { + Return map[string]interface{} `json:"return"` + } + var response qmpResponse + err = json.Unmarshal(buf[:nr], &response) + if err != nil { + return nil, err + } + // expecting empty response + if len(response.Return) != 0 { + return nil, fmt.Errorf("qmp_capabilities failed: %v", response.Return) + } + + // { "execute": command } + jsonCommand, err = json.Marshal(qmpCommand{Command: command}) + if err != nil { + return nil, err + } + _, err = conn.Write(jsonCommand) + if err != nil { + return nil, err + } + nr, err = conn.Read(buf[:]) + if err != nil { + return nil, err + } + err = json.Unmarshal(buf[:nr], &response) + if err != nil { + return nil, err + } + if strings.HasPrefix(command, "query-") { + return response.Return, nil + } + // non-query commands should return an empty response + if len(response.Return) != 0 { + return nil, fmt.Errorf("%s failed: %v", command, response.Return) + } + return response.Return, nil +} + +func WaitForTCPWithDelay(addr string, duration time.Duration) error { + for { + conn, err := net.Dial("tcp", addr) + if err != nil { + continue + } + defer conn.Close() + if _, err = conn.Read(make([]byte, 1)); err != nil { + time.Sleep(duration) + continue + } + break + } + return nil +} diff --git a/pkg/minikube/driver/driver.go b/pkg/minikube/driver/driver.go index 40bf67f554a4..cae2cdd08ef9 100644 --- a/pkg/minikube/driver/driver.go +++ b/pkg/minikube/driver/driver.go @@ -46,6 +46,8 @@ const ( SSH = "ssh" // KVM2 driver KVM2 = "kvm2" + // QEMU2 driver + QEMU2 = "qemu2" // QEMU driver QEMU = "qemu" // VirtualBox driver @@ -158,6 +160,11 @@ func IsKVM(name string) bool { return name == KVM2 || name == AliasKVM } +// IsQEMU checks if the driver is a QEMU[2] +func IsQEMU(name string) bool { + return name == QEMU2 || name == QEMU +} + // IsVM checks if the driver is a VM func IsVM(name string) bool { if IsKIC(name) || BareMetal(name) { @@ -183,7 +190,7 @@ func AllowsPreload(driverName string) bool { // NeedsPortForward returns true if driver is unable provide direct IP connectivity func NeedsPortForward(name string) bool { - if name == QEMU { + if IsQEMU(name) { return true } if !IsKIC(name) { diff --git a/pkg/minikube/driver/driver_darwin.go b/pkg/minikube/driver/driver_darwin.go index 505b97ff0bbd..cfeba75bbf88 100644 --- a/pkg/minikube/driver/driver_darwin.go +++ b/pkg/minikube/driver/driver_darwin.go @@ -27,7 +27,7 @@ var supportedDrivers = func() []string { if runtime.GOARCH == "arm64" { // on darwin/arm64 only docker and ssh are supported yet return []string{ - QEMU, + QEMU2, Docker, Podman, SSH, @@ -51,7 +51,7 @@ var supportedDrivers = func() []string { VMwareFusion, HyperKit, VMware, - QEMU, + QEMU2, Docker, Podman, SSH, diff --git a/pkg/minikube/driver/driver_linux.go b/pkg/minikube/driver/driver_linux.go index 9a5b4695ab72..e12c23d7021e 100644 --- a/pkg/minikube/driver/driver_linux.go +++ b/pkg/minikube/driver/driver_linux.go @@ -25,6 +25,7 @@ var supportedDrivers = []string{ VirtualBox, VMwareFusion, KVM2, + QEMU2, QEMU, VMware, None, diff --git a/pkg/minikube/driver/driver_windows.go b/pkg/minikube/driver/driver_windows.go index 473ee45ae9ca..51defd6228ed 100644 --- a/pkg/minikube/driver/driver_windows.go +++ b/pkg/minikube/driver/driver_windows.go @@ -32,7 +32,7 @@ var supportedDrivers = []string{ VMwareFusion, HyperV, VMware, - QEMU, + QEMU2, Docker, Podman, SSH, diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index 278253aa4c71..9a4058107409 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -632,7 +632,7 @@ func validateNetwork(h *host.Host, r command.Runner, imageRepository string) (st } } - if !driver.BareMetal(h.Driver.DriverName()) && !driver.IsKIC(h.Driver.DriverName()) { + if !driver.BareMetal(h.Driver.DriverName()) && !driver.IsKIC(h.Driver.DriverName()) && !driver.IsQEMU(h.Driver.DriverName()) { if err := trySSH(h, ip); err != nil { return ip, err } diff --git a/pkg/minikube/registry/drvs/init.go b/pkg/minikube/registry/drvs/init.go index 40c9ff18a5f4..6176d892ef48 100644 --- a/pkg/minikube/registry/drvs/init.go +++ b/pkg/minikube/registry/drvs/init.go @@ -26,6 +26,7 @@ import ( _ "k8s.io/minikube/pkg/minikube/registry/drvs/parallels" _ "k8s.io/minikube/pkg/minikube/registry/drvs/podman" _ "k8s.io/minikube/pkg/minikube/registry/drvs/qemu" + _ "k8s.io/minikube/pkg/minikube/registry/drvs/qemu2" _ "k8s.io/minikube/pkg/minikube/registry/drvs/ssh" _ "k8s.io/minikube/pkg/minikube/registry/drvs/virtualbox" _ "k8s.io/minikube/pkg/minikube/registry/drvs/vmware" diff --git a/pkg/minikube/registry/drvs/qemu/qemu.go b/pkg/minikube/registry/drvs/qemu/qemu.go index e9f13016603b..24f15693eb8d 100644 --- a/pkg/minikube/registry/drvs/qemu/qemu.go +++ b/pkg/minikube/registry/drvs/qemu/qemu.go @@ -20,7 +20,6 @@ import ( "fmt" "os/exec" "path/filepath" - "runtime" "github.com/docker/machine/libmachine/drivers" drvqemu "github.com/machine-drivers/docker-machine-driver-qemu" @@ -67,18 +66,7 @@ func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) { } func status() registry.State { - var qemuSystem string - arch := runtime.GOARCH - switch arch { - case "amd64": - qemuSystem = "qemu-system-x86_64" - case "arm64": - qemuSystem = "qemu-system-aarch64" - default: - return registry.State{Error: fmt.Errorf("unknown arch: %s", arch), Doc: docURL} - } - - _, err := exec.LookPath(qemuSystem) + _, err := exec.LookPath("qemu-system-x86_64") if err != nil { return registry.State{Error: err, Fix: "Install qemu-system", Doc: docURL} } diff --git a/pkg/minikube/registry/drvs/qemu2/doc.go b/pkg/minikube/registry/drvs/qemu2/doc.go new file mode 100644 index 000000000000..c05933150304 --- /dev/null +++ b/pkg/minikube/registry/drvs/qemu2/doc.go @@ -0,0 +1,17 @@ +/* +Copyright 2018 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package qemu2 diff --git a/pkg/minikube/registry/drvs/qemu2/qemu2.go b/pkg/minikube/registry/drvs/qemu2/qemu2.go new file mode 100644 index 000000000000..c66581f77163 --- /dev/null +++ b/pkg/minikube/registry/drvs/qemu2/qemu2.go @@ -0,0 +1,103 @@ +/* +Copyright 2018 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package qemu2 + +import ( + "fmt" + "os/exec" + "path/filepath" + "runtime" + + "github.com/docker/machine/libmachine/drivers" + "k8s.io/minikube/pkg/drivers/qemu" + + "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/download" + "k8s.io/minikube/pkg/minikube/driver" + "k8s.io/minikube/pkg/minikube/localpath" + "k8s.io/minikube/pkg/minikube/registry" +) + +const ( + docURL = "https://minikube.sigs.k8s.io/docs/reference/drivers/qemu2/" +) + +func init() { + if err := registry.Register(registry.DriverDef{ + Name: driver.QEMU2, + Init: func() drivers.Driver { return qemu.NewDriver("", "") }, + Config: configure, + Status: status, + Default: true, + Priority: registry.Experimental, + }); err != nil { + panic(fmt.Sprintf("register failed: %v", err)) + } +} + +func qemuSystemProgram() (string, error) { + arch := runtime.GOARCH + switch arch { + case "amd64": + return "qemu-system-x86_64", nil + case "arm64": + return "qemu-system-aarch64", nil + default: + return "", fmt.Errorf("unknown arch: %s", arch) + } +} + +func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) { + name := config.MachineName(cc, n) + qemuSystem, err := qemuSystemProgram() + if err != nil { + return nil, err + } + return qemu.Driver{ + BaseDriver: &drivers.BaseDriver{ + MachineName: name, + StorePath: localpath.MiniPath(), + SSHUser: "docker", + }, + Boot2DockerURL: download.LocalISOResource(cc.MinikubeISO), + DiskSize: cc.DiskSize, + Memory: cc.Memory, + CPU: cc.CPUs, + EnginePort: 2376, + FirstQuery: true, + DiskPath: filepath.Join(localpath.MiniPath(), "machines", name, fmt.Sprintf("%s.img", name)), + Program: qemuSystem, + VirtioDrives: false, + Network: "user", + CacheMode: "default", + IOMode: "threads", + }, nil +} + +func status() registry.State { + qemuSystem, err := qemuSystemProgram() + if err != nil { + return registry.State{Error: err, Doc: docURL} + } + + _, err = exec.LookPath(qemuSystem) + if err != nil { + return registry.State{Error: err, Fix: "Install qemu-system", Doc: docURL} + } + + return registry.State{Installed: true, Healthy: true, Running: true} +} From 69653073dab26cee30c07f62a91c58ca39e5dfe5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Mon, 4 Apr 2022 20:12:12 +0200 Subject: [PATCH 06/18] Set up the IP address for the qemu driver Assume "user" networking, for now at least. --- pkg/minikube/cluster/ip.go | 7 +++++++ pkg/minikube/machine/machine.go | 3 +++ 2 files changed, 10 insertions(+) diff --git a/pkg/minikube/cluster/ip.go b/pkg/minikube/cluster/ip.go index bb73ca895696..313fb72656ab 100644 --- a/pkg/minikube/cluster/ip.go +++ b/pkg/minikube/cluster/ip.go @@ -58,6 +58,10 @@ func HostIP(host *host.Host, clusterName string) (net.IP, error) { return []byte{}, errors.Wrap(err, "Error converting VM/Host IP address to IPv4 address") } return net.IPv4(vmIP[0], vmIP[1], vmIP[2], byte(1)), nil + case driver.QEMU2: + return net.ParseIP("10.0.2.2"), nil + case driver.QEMU: + return net.ParseIP("10.0.2.2"), nil case driver.HyperV: v := reflect.ValueOf(host.Driver).Elem() var hypervVirtualSwitch string @@ -147,6 +151,9 @@ func DriverIP(api libmachine.API, machineName string) (net.IP, error) { if driver.IsKIC(host.DriverName) { ipStr = oci.DefaultBindIPV4 } + if driver.IsQEMU(host.DriverName) { + ipStr = "127.0.0.1" + } ip := net.ParseIP(ipStr) if ip == nil { return nil, fmt.Errorf("parsing IP: %s", ipStr) diff --git a/pkg/minikube/machine/machine.go b/pkg/minikube/machine/machine.go index a465694e0bdb..9f96600fa35a 100644 --- a/pkg/minikube/machine/machine.go +++ b/pkg/minikube/machine/machine.go @@ -122,6 +122,9 @@ func saveHost(api libmachine.API, h *host.Host, cfg *config.ClusterConfig, n *co if err != nil { return err } + if ip == "127.0.0.1" && driver.IsQEMU(h.Driver.DriverName()) { + ip = "10.0.2.15" + } n.IP = ip return config.SaveNode(cfg, n) } From 191ff5bf0b496e001b631cd000607859ae0ca3e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Mon, 4 Apr 2022 20:13:28 +0200 Subject: [PATCH 07/18] Add tunneling of apiserver port for qemu driver --- pkg/minikube/config/types.go | 11 ++++++----- pkg/minikube/driver/endpoint.go | 4 +++- pkg/minikube/node/start.go | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 6 deletions(-) diff --git a/pkg/minikube/config/types.go b/pkg/minikube/config/types.go index c1af9ace7ae5..4566aeb27bef 100644 --- a/pkg/minikube/config/types.go +++ b/pkg/minikube/config/types.go @@ -52,11 +52,12 @@ type ClusterConfig struct { HypervVirtualSwitch string HypervUseExternalSwitch bool HypervExternalAdapter string - KVMNetwork string // Only used by the KVM2 driver - KVMQemuURI string // Only used by the KVM2 driver - KVMGPU bool // Only used by the KVM2 driver - KVMHidden bool // Only used by the KVM2 driver - KVMNUMACount int // Only used by the KVM2 driver + KVMNetwork string // Only used by the KVM2 driver + KVMQemuURI string // Only used by the KVM2 driver + KVMGPU bool // Only used by the KVM2 driver + KVMHidden bool // Only used by the KVM2 driver + KVMNUMACount int // Only used by the KVM2 driver + APIServerPort int DockerOpt []string // Each entry is formatted as KEY=VALUE. DisableDriverMounts bool // Only used by virtualbox NFSShare []string diff --git a/pkg/minikube/driver/endpoint.go b/pkg/minikube/driver/endpoint.go index 15203db100f8..aa02afc9af68 100644 --- a/pkg/minikube/driver/endpoint.go +++ b/pkg/minikube/driver/endpoint.go @@ -28,7 +28,7 @@ import ( // ControlPlaneEndpoint returns the location where callers can reach this cluster func ControlPlaneEndpoint(cc *config.ClusterConfig, cp *config.Node, driverName string) (string, net.IP, int, error) { - if NeedsPortForward(driverName) { + if NeedsPortForward(driverName) && IsKIC(driverName) { port, err := oci.ForwardedPort(cc.Driver, cc.Name, cp.Port) if err != nil { klog.Warningf("failed to get forwarded control plane port %v", err) @@ -45,6 +45,8 @@ func ControlPlaneEndpoint(cc *config.ClusterConfig, cp *config.Node, driverName hostname = cc.KubernetesConfig.APIServerName } return hostname, ips[0], port, err + } else if NeedsPortForward(driverName) && IsQEMU(driverName) { + return "localhost", net.IPv4(127, 0, 0, 1), cc.APIServerPort, nil } // https://github.com/kubernetes/minikube/issues/3878 diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index 9a4058107409..74bbcfe98eac 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -253,6 +253,15 @@ func handleAPIServer(starter Starter, cr cruntime.Manager, hostIP net.IP) (*kube return nil, bs, err } + // Tunnel apiserver to guest, if needed + if starter.Cfg.APIServerPort != 0 { + args := []string{"-f", "-NTL", fmt.Sprintf("%d:localhost:8443", starter.Cfg.APIServerPort)} + err := machine.CreateSSHShell(starter.MachineAPI, *starter.Cfg, *starter.Node, args, false) + if err != nil { + klog.Warningf("apiserver tunnel failed: %v", err) + } + } + // Write the kubeconfig to the file system after everything required (like certs) are created by the bootstrapper. if err := kubeconfig.Update(kcs); err != nil { return nil, bs, errors.Wrap(err, "Failed kubeconfig update") @@ -552,6 +561,14 @@ func startMachine(cfg *config.ClusterConfig, node *config.Node, delOnFail bool) return runner, preExists, m, host, errors.Wrap(err, "Failed to validate network") } + if driver.IsQEMU(host.Driver.DriverName()) { + apiServerPort, err := getPort() + if err != nil { + return runner, preExists, m, host, errors.Wrap(err, "Failed to find apiserver port") + } + cfg.APIServerPort = apiServerPort + } + // Bypass proxy for minikube's vm host ip err = proxy.ExcludeIP(ip) if err != nil { @@ -561,6 +578,21 @@ func startMachine(cfg *config.ClusterConfig, node *config.Node, delOnFail bool) return runner, preExists, m, host, err } +// getPort asks the kernel for a free open port that is ready to use +func getPort() (int, error) { + addr, err := net.ResolveTCPAddr("tcp", "localhost:0") + if err != nil { + panic(err) + } + + l, err := net.ListenTCP("tcp", addr) + if err != nil { + return -1, errors.Errorf("Error accessing port %d", addr.Port) + } + defer l.Close() + return l.Addr().(*net.TCPAddr).Port, nil +} + // startHostInternal starts a new minikube host using a VM or None func startHostInternal(api libmachine.API, cc *config.ClusterConfig, n *config.Node, delOnFail bool) (*host.Host, bool, error) { host, exists, err := machine.StartHost(api, cc, n) From 6a7f14ea01974a59eec8b3d815536c93a90f55b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Mon, 4 Apr 2022 20:14:23 +0200 Subject: [PATCH 08/18] Use the right port for docker in qemu driver --- cmd/minikube/cmd/docker-env.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/minikube/cmd/docker-env.go b/cmd/minikube/cmd/docker-env.go index 2a4efb31c0df..9f4127518858 100644 --- a/cmd/minikube/cmd/docker-env.go +++ b/cmd/minikube/cmd/docker-env.go @@ -38,6 +38,7 @@ import ( "k8s.io/klog/v2" "k8s.io/minikube/pkg/drivers/kic/oci" + "k8s.io/minikube/pkg/drivers/qemu" "k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/kverify" "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/constants" @@ -290,11 +291,13 @@ var dockerEnvCmd = &cobra.Command{ d := co.CP.Host.Driver port := constants.DockerDaemonPort - if driver.NeedsPortForward(driverName) { + if driver.NeedsPortForward(driverName) && driver.IsKIC(driverName) { port, err = oci.ForwardedPort(driverName, cname, port) if err != nil { exit.Message(reason.DrvPortForward, "Error getting port binding for '{{.driver_name}} driver: {{.error}}", out.V{"driver_name": driverName, "error": err}) } + } else if driver.NeedsPortForward(driverName) && driverName == driver.QEMU2 { + port = d.(*qemu.Driver).EnginePort } hostname, err := d.GetSSHHostname() From af07b2b5634e1f60f9a16471f3d1e6e24f650576 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Mon, 4 Apr 2022 20:23:14 +0200 Subject: [PATCH 09/18] Remove fluff from the qemu driver for lint --- pkg/drivers/qemu/qemu.go | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/pkg/drivers/qemu/qemu.go b/pkg/drivers/qemu/qemu.go index 50fd4992bcec..101cb5bbee60 100644 --- a/pkg/drivers/qemu/qemu.go +++ b/pkg/drivers/qemu/qemu.go @@ -88,7 +88,6 @@ func (d *Driver) GetMachineName() string { func (d *Driver) GetSSHHostname() (string, error) { return "localhost", nil - //return d.GetIP() } func (d *Driver) GetSSHKeyPath() string { @@ -412,12 +411,9 @@ func (d *Driver) Start() error { fmt.Printf("OUTPUT: %s\n", stdout) fmt.Printf("ERROR: %s\n", stderr) return err - //if err := cmdStart(d.Program, startCmd...); err != nil { - // return err } log.Infof("Waiting for VM to start (ssh -p %d docker@localhost)...", d.SSHPort) - //return ssh.WaitForTCP(fmt.Sprintf("localhost:%d", d.SSHPort)) return WaitForTCPWithDelay(fmt.Sprintf("localhost:%d", d.SSHPort), time.Second) } @@ -446,12 +442,6 @@ func cmdOutErr(cmdStr string, args ...string) (string, string, error) { return stdout.String(), stderrStr, err } -func cmdStart(cmdStr string, args ...string) error { - cmd := exec.Command(cmdStr, args...) - log.Debugf("executing: %v %v", cmdStr, strings.Join(args, " ")) - return cmd.Start() -} - func (d *Driver) Stop() error { // _, err := d.RunQMPCommand("stop") _, err := d.RunQMPCommand("system_powerdown") @@ -519,10 +509,6 @@ func (d *Driver) Upgrade() error { return fmt.Errorf("hosts without a driver cannot be upgraded") } -//func (d *Driver) GetSSHCommand(args ...string) (*exec.Cmd, error) { -// return ssh.GetSSHCommand("localhost", d.SSHPort, "docker", d.sshKeyPath(), args...), nil -//} - func (d *Driver) sshKeyPath() string { machineDir := filepath.Join(d.StorePath, "machines", d.GetMachineName()) return filepath.Join(machineDir, "id_rsa") @@ -577,14 +563,14 @@ func (d *Driver) generateDiskImage(size int) error { if err := tw.WriteHeader(file); err != nil { return err } - if _, err := tw.Write([]byte(pubKey)); err != nil { + if _, err := tw.Write(pubKey); err != nil { return err } file = &tar.Header{Name: ".ssh/authorized_keys2", Size: int64(len(pubKey)), Mode: 0644} if err := tw.WriteHeader(file); err != nil { return err } - if _, err := tw.Write([]byte(pubKey)); err != nil { + if _, err := tw.Write(pubKey); err != nil { return err } if err := tw.Close(); err != nil { @@ -619,10 +605,16 @@ func (d *Driver) generateUserdataDisk(userdataFile string) (string, error) { machineDir := filepath.Join(d.StorePath, "machines", d.GetMachineName()) ccRoot := filepath.Join(machineDir, "cloud-config") - os.MkdirAll(ccRoot, 0755) + err = os.MkdirAll(ccRoot, 0755) + if err != nil { + return "", err + } userDataDir := filepath.Join(ccRoot, "openstack/latest") - os.MkdirAll(userDataDir, 0755) + err = os.MkdirAll(userDataDir, 0755) + if err != nil { + return "", err + } writeFile := filepath.Join(userDataDir, "user_data") if err := ioutil.WriteFile(writeFile, userdata, 0644); err != nil { @@ -663,7 +655,10 @@ func (d *Driver) RunQMPCommand(command string) (map[string]interface{}, error) { } var initialResponse qmpInitialResponse - json.Unmarshal(buf[:nr], &initialResponse) + err = json.Unmarshal(buf[:nr], &initialResponse) + if err != nil { + return nil, err + } // run 'qmp_capabilities' to switch to command mode // { "execute": "qmp_capabilities" } From d38789a37b1f86b93841829d1829a53661d26151 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Mon, 4 Apr 2022 20:50:08 +0200 Subject: [PATCH 10/18] Add the parameters required for qemu arm64 --- pkg/drivers/qemu/qemu.go | 26 ++++++++++++++ pkg/minikube/registry/drvs/qemu2/qemu2.go | 42 +++++++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/pkg/drivers/qemu/qemu.go b/pkg/drivers/qemu/qemu.go index 101cb5bbee60..f68ac9a62df6 100644 --- a/pkg/drivers/qemu/qemu.go +++ b/pkg/drivers/qemu/qemu.go @@ -27,6 +27,7 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "strconv" "strings" "syscall" @@ -58,6 +59,10 @@ type Driver struct { DiskSize int CPU int Program string + BIOS bool + CPUType string + MachineType string + Firmware string Display bool DisplayType string Nographic bool @@ -133,6 +138,7 @@ func (d *Driver) GetURL() (string, error) { func NewDriver(hostName, storePath string) drivers.Driver { return &Driver{ + BIOS: runtime.GOARCH != "arm64", PrivateNetwork: privateNetworkName, BaseDriver: &drivers.BaseDriver{ SSHUser: defaultSSHUser, @@ -331,6 +337,26 @@ func (d *Driver) Start() error { var startCmd []string + if d.MachineType != "" { + startCmd = append(startCmd, + "-M", d.MachineType, + ) + } + if d.CPUType != "" { + startCmd = append(startCmd, + "-cpu", d.CPUType, + ) + } + + if !d.BIOS { + if d.Firmware != "" { + startCmd = append(startCmd, + "-drive", fmt.Sprintf("file=%s,readonly,format=raw,if=pflash", d.Firmware)) + } else { + return fmt.Errorf("unknown firmware") + } + } + if d.Display { if d.DisplayType != "" { startCmd = append(startCmd, diff --git a/pkg/minikube/registry/drvs/qemu2/qemu2.go b/pkg/minikube/registry/drvs/qemu2/qemu2.go index c66581f77163..9f5ae4a144df 100644 --- a/pkg/minikube/registry/drvs/qemu2/qemu2.go +++ b/pkg/minikube/registry/drvs/qemu2/qemu2.go @@ -18,6 +18,7 @@ package qemu2 import ( "fmt" + "os" "os/exec" "path/filepath" "runtime" @@ -61,12 +62,40 @@ func qemuSystemProgram() (string, error) { } } +func qemuFirmwarePath() (string, error) { + arch := runtime.GOARCH + switch arch { + case "amd64": + return "/usr/share/OVMF/OVMF_CODE.fd", nil + case "arm64": + return "/usr/share/AAVMF/AAVMF_CODE.fd", nil + default: + return "", fmt.Errorf("unknown arch: %s", arch) + } +} + func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) { name := config.MachineName(cc, n) qemuSystem, err := qemuSystemProgram() if err != nil { return nil, err } + var qemuMachine string + var qemuCPU string + switch runtime.GOARCH { + case "amd64": + qemuMachine = "" // default + qemuCPU = "" // default + case "arm64": + qemuMachine = "virt" + qemuCPU = "cortex-a72" + default: + return nil, fmt.Errorf("unknown arch: %s", runtime.GOARCH) + } + qemuFirmware, err := qemuFirmwarePath() + if err != nil { + return nil, err + } return qemu.Driver{ BaseDriver: &drivers.BaseDriver{ MachineName: name, @@ -81,6 +110,10 @@ func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) { FirstQuery: true, DiskPath: filepath.Join(localpath.MiniPath(), "machines", name, fmt.Sprintf("%s.img", name)), Program: qemuSystem, + BIOS: runtime.GOARCH != "arm64", + MachineType: qemuMachine, + CPUType: qemuCPU, + Firmware: qemuFirmware, VirtioDrives: false, Network: "user", CacheMode: "default", @@ -99,5 +132,14 @@ func status() registry.State { return registry.State{Error: err, Fix: "Install qemu-system", Doc: docURL} } + qemuFirmware, err := qemuFirmwarePath() + if err != nil { + return registry.State{Error: err, Doc: docURL} + } + + if _, err := os.Stat(qemuFirmware); err != nil && runtime.GOARCH == "arm64" { + return registry.State{Error: err, Fix: "Install uefi firmware", Doc: docURL} + } + return registry.State{Installed: true, Healthy: true, Running: true} } From e894c88783df7b8a2474db3a1bf98a4e0c9f7dc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Mon, 4 Apr 2022 21:01:12 +0200 Subject: [PATCH 11/18] Add missing machine type for QEMU2 for test --- pkg/minikube/driver/driver_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/minikube/driver/driver_test.go b/pkg/minikube/driver/driver_test.go index 52b0be0f711e..0b7806610e69 100644 --- a/pkg/minikube/driver/driver_test.go +++ b/pkg/minikube/driver/driver_test.go @@ -66,6 +66,7 @@ func TestMachineType(t *testing.T) { None: "bare metal machine", SSH: "bare metal machine", KVM2: "VM", + QEMU2: "VM", QEMU: "VM", VirtualBox: "VM", HyperKit: "VM", From e7495ab779a93e72e1cd953b9fc0d59831a305d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Tue, 5 Apr 2022 19:54:35 +0200 Subject: [PATCH 12/18] Make sure to set up apiserver tunnel on restart --- pkg/minikube/machine/fix.go | 9 +++++++++ pkg/minikube/machine/start.go | 9 +++++++++ pkg/minikube/node/start.go | 11 +---------- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/pkg/minikube/machine/fix.go b/pkg/minikube/machine/fix.go index 02c451128a09..ad7a32a9665a 100644 --- a/pkg/minikube/machine/fix.go +++ b/pkg/minikube/machine/fix.go @@ -87,6 +87,15 @@ func fixHost(api libmachine.API, cc *config.ClusterConfig, n *config.Node) (*hos return h, nil } + // Tunnel apiserver to guest, if needed + if cc.APIServerPort != 0 { + args := []string{"-f", "-NTL", fmt.Sprintf("%d:localhost:8443", cc.APIServerPort)} + err := CreateSSHShell(api, *cc, *n, args, false) + if err != nil { + klog.Warningf("apiserver tunnel failed: %v", err) + } + } + if err := postStartSetup(h, *cc); err != nil { return h, errors.Wrap(err, "post-start") } diff --git a/pkg/minikube/machine/start.go b/pkg/minikube/machine/start.go index 895d2993d858..8bd5738ce157 100644 --- a/pkg/minikube/machine/start.go +++ b/pkg/minikube/machine/start.go @@ -175,6 +175,15 @@ func createHost(api libmachine.API, cfg *config.ClusterConfig, n *config.Node) ( showHostInfo(h, *cfg) } + // Tunnel apiserver to guest, if needed + if cfg.APIServerPort != 0 { + args := []string{"-f", "-NTL", fmt.Sprintf("%d:localhost:8443", cfg.APIServerPort)} + err := CreateSSHShell(api, *cfg, *n, args, false) + if err != nil { + klog.Warningf("apiserver tunnel failed: %v", err) + } + } + if err := postStartSetup(h, *cfg); err != nil { return h, errors.Wrap(err, "post-start") } diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index 74bbcfe98eac..62f8b246b2d0 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -253,15 +253,6 @@ func handleAPIServer(starter Starter, cr cruntime.Manager, hostIP net.IP) (*kube return nil, bs, err } - // Tunnel apiserver to guest, if needed - if starter.Cfg.APIServerPort != 0 { - args := []string{"-f", "-NTL", fmt.Sprintf("%d:localhost:8443", starter.Cfg.APIServerPort)} - err := machine.CreateSSHShell(starter.MachineAPI, *starter.Cfg, *starter.Node, args, false) - if err != nil { - klog.Warningf("apiserver tunnel failed: %v", err) - } - } - // Write the kubeconfig to the file system after everything required (like certs) are created by the bootstrapper. if err := kubeconfig.Update(kcs); err != nil { return nil, bs, errors.Wrap(err, "Failed kubeconfig update") @@ -561,7 +552,7 @@ func startMachine(cfg *config.ClusterConfig, node *config.Node, delOnFail bool) return runner, preExists, m, host, errors.Wrap(err, "Failed to validate network") } - if driver.IsQEMU(host.Driver.DriverName()) { + if driver.IsQEMU(host.Driver.DriverName()) && cfg.APIServerPort == 0 { apiServerPort, err := getPort() if err != nil { return runner, preExists, m, host, errors.Wrap(err, "Failed to find apiserver port") From d21342c69fab422f955325e112c5e9d101fd8c1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Tue, 5 Apr 2022 20:01:50 +0200 Subject: [PATCH 13/18] Make sure to look for KIC before using oci port --- cmd/minikube/cmd/service.go | 7 ++++++- cmd/minikube/cmd/tunnel.go | 3 +-- pkg/addons/addons.go | 2 +- pkg/addons/addons_autopause.go | 2 +- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/cmd/minikube/cmd/service.go b/cmd/minikube/cmd/service.go index 2f15ca38ad5a..59e4bfee16fb 100644 --- a/cmd/minikube/cmd/service.go +++ b/cmd/minikube/cmd/service.go @@ -140,8 +140,10 @@ You may select another namespace by using 'minikube service {{.service}} -n Date: Sun, 10 Apr 2022 17:54:12 +0200 Subject: [PATCH 14/18] Revert "Make sure to set up apiserver tunnel on restart" This reverts commit e7495ab779a93e72e1cd953b9fc0d59831a305d5. --- pkg/minikube/machine/fix.go | 9 --------- pkg/minikube/machine/start.go | 9 --------- pkg/minikube/node/start.go | 11 ++++++++++- 3 files changed, 10 insertions(+), 19 deletions(-) diff --git a/pkg/minikube/machine/fix.go b/pkg/minikube/machine/fix.go index ad7a32a9665a..02c451128a09 100644 --- a/pkg/minikube/machine/fix.go +++ b/pkg/minikube/machine/fix.go @@ -87,15 +87,6 @@ func fixHost(api libmachine.API, cc *config.ClusterConfig, n *config.Node) (*hos return h, nil } - // Tunnel apiserver to guest, if needed - if cc.APIServerPort != 0 { - args := []string{"-f", "-NTL", fmt.Sprintf("%d:localhost:8443", cc.APIServerPort)} - err := CreateSSHShell(api, *cc, *n, args, false) - if err != nil { - klog.Warningf("apiserver tunnel failed: %v", err) - } - } - if err := postStartSetup(h, *cc); err != nil { return h, errors.Wrap(err, "post-start") } diff --git a/pkg/minikube/machine/start.go b/pkg/minikube/machine/start.go index 8bd5738ce157..895d2993d858 100644 --- a/pkg/minikube/machine/start.go +++ b/pkg/minikube/machine/start.go @@ -175,15 +175,6 @@ func createHost(api libmachine.API, cfg *config.ClusterConfig, n *config.Node) ( showHostInfo(h, *cfg) } - // Tunnel apiserver to guest, if needed - if cfg.APIServerPort != 0 { - args := []string{"-f", "-NTL", fmt.Sprintf("%d:localhost:8443", cfg.APIServerPort)} - err := CreateSSHShell(api, *cfg, *n, args, false) - if err != nil { - klog.Warningf("apiserver tunnel failed: %v", err) - } - } - if err := postStartSetup(h, *cfg); err != nil { return h, errors.Wrap(err, "post-start") } diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index 62f8b246b2d0..74bbcfe98eac 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -253,6 +253,15 @@ func handleAPIServer(starter Starter, cr cruntime.Manager, hostIP net.IP) (*kube return nil, bs, err } + // Tunnel apiserver to guest, if needed + if starter.Cfg.APIServerPort != 0 { + args := []string{"-f", "-NTL", fmt.Sprintf("%d:localhost:8443", starter.Cfg.APIServerPort)} + err := machine.CreateSSHShell(starter.MachineAPI, *starter.Cfg, *starter.Node, args, false) + if err != nil { + klog.Warningf("apiserver tunnel failed: %v", err) + } + } + // Write the kubeconfig to the file system after everything required (like certs) are created by the bootstrapper. if err := kubeconfig.Update(kcs); err != nil { return nil, bs, errors.Wrap(err, "Failed kubeconfig update") @@ -552,7 +561,7 @@ func startMachine(cfg *config.ClusterConfig, node *config.Node, delOnFail bool) return runner, preExists, m, host, errors.Wrap(err, "Failed to validate network") } - if driver.IsQEMU(host.Driver.DriverName()) && cfg.APIServerPort == 0 { + if driver.IsQEMU(host.Driver.DriverName()) { apiServerPort, err := getPort() if err != nil { return runner, preExists, m, host, errors.Wrap(err, "Failed to find apiserver port") From ae40c5762333781b6b1c2caa86369721cc451ad3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Mon, 11 Apr 2022 19:40:13 +0200 Subject: [PATCH 15/18] Add parameters needed for vde networking --- pkg/drivers/qemu/qemu.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/drivers/qemu/qemu.go b/pkg/drivers/qemu/qemu.go index f68ac9a62df6..2b784cc1b53e 100644 --- a/pkg/drivers/qemu/qemu.go +++ b/pkg/drivers/qemu/qemu.go @@ -72,6 +72,7 @@ type Driver struct { Boot2DockerURL string NetworkInterface string NetworkAddress string + NetworkSocket string NetworkBridge string CaCertPath string PrivateKeyPath string @@ -402,6 +403,10 @@ func (d *Driver) Start() error { startCmd = append(startCmd, "-nic", fmt.Sprintf("tap,model=virtio,ifname=%s,script=no,downscript=no", d.NetworkInterface), ) + } else if d.Network == "vde" { + startCmd = append(startCmd, + "-nic", fmt.Sprintf("vde,model=virtio,sock=%s", d.NetworkSocket), + ) } else if d.Network == "bridge" { startCmd = append(startCmd, "-nic", fmt.Sprintf("bridge,model=virtio,br=%s", d.NetworkBridge), From 3555b194a626360e960380a65c8477f59a3c7d6a Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Tue, 3 May 2022 14:01:00 -0700 Subject: [PATCH 16/18] fix firmware locations for darwin --- pkg/minikube/registry/drvs/qemu2/qemu2.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkg/minikube/registry/drvs/qemu2/qemu2.go b/pkg/minikube/registry/drvs/qemu2/qemu2.go index 9f5ae4a144df..229f3ba3ccad 100644 --- a/pkg/minikube/registry/drvs/qemu2/qemu2.go +++ b/pkg/minikube/registry/drvs/qemu2/qemu2.go @@ -66,8 +66,15 @@ func qemuFirmwarePath() (string, error) { arch := runtime.GOARCH switch arch { case "amd64": + // on macOS, we assume qemu is installed via homebrew for simplicity + if runtime.GOOS == "darwin" { + return "/usr/local/Cellar/qemu/6.2.0_1/share/qemu/edk2-x86_64-code.fd", nil + } return "/usr/share/OVMF/OVMF_CODE.fd", nil case "arm64": + if runtime.GOOS == "darwin" { + return "/opt/homebrew/Cellar/qemu/6.2.0_1/share/qemu/edk2-aarch64-code.fd", nil + } return "/usr/share/AAVMF/AAVMF_CODE.fd", nil default: return "", fmt.Errorf("unknown arch: %s", arch) From 9a0f1a012b7ec93a8174fac78a2497b870fbdc99 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Wed, 11 May 2022 14:54:08 -0700 Subject: [PATCH 17/18] support darwin hardware acceleration --- pkg/drivers/qemu/qemu.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/drivers/qemu/qemu.go b/pkg/drivers/qemu/qemu.go index 2b784cc1b53e..9d2cdd2f31fc 100644 --- a/pkg/drivers/qemu/qemu.go +++ b/pkg/drivers/qemu/qemu.go @@ -339,6 +339,11 @@ func (d *Driver) Start() error { var startCmd []string if d.MachineType != "" { + machineType := d.MachineType + if runtime.GOOS == "darwin" { + // highmem=off needed, see https://patchwork.kernel.org/project/qemu-devel/patch/20201126215017.41156-9-agraf@csgraf.de/#23800615 for details + machineType += ",accel=hvf,highmem=off" + } startCmd = append(startCmd, "-M", d.MachineType, ) @@ -352,7 +357,7 @@ func (d *Driver) Start() error { if !d.BIOS { if d.Firmware != "" { startCmd = append(startCmd, - "-drive", fmt.Sprintf("file=%s,readonly,format=raw,if=pflash", d.Firmware)) + "-drive", fmt.Sprintf("file=%s,readonly=on,format=raw,if=pflash", d.Firmware)) } else { return fmt.Errorf("unknown firmware") } From 60328d4d40a11ac7c18c6243f597bcfbb3050148 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Wed, 11 May 2022 15:06:39 -0700 Subject: [PATCH 18/18] fix lint --- pkg/drivers/qemu/qemu.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/drivers/qemu/qemu.go b/pkg/drivers/qemu/qemu.go index 9d2cdd2f31fc..08ceb5441d7b 100644 --- a/pkg/drivers/qemu/qemu.go +++ b/pkg/drivers/qemu/qemu.go @@ -345,7 +345,7 @@ func (d *Driver) Start() error { machineType += ",accel=hvf,highmem=off" } startCmd = append(startCmd, - "-M", d.MachineType, + "-M", machineType, ) } if d.CPUType != "" {