From 2a33bb66beed435773513f0c6f8019180903ef45 Mon Sep 17 00:00:00 2001 From: Jani Poikela Date: Fri, 17 Apr 2020 16:45:35 +0300 Subject: [PATCH 01/20] fix link to kubernetes 'pull-image-private-registry' --- site/content/en/docs/handbook/registry.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/site/content/en/docs/handbook/registry.md b/site/content/en/docs/handbook/registry.md index 5d6d8d83859f..0b04a6f3644f 100644 --- a/site/content/en/docs/handbook/registry.md +++ b/site/content/en/docs/handbook/registry.md @@ -29,7 +29,7 @@ registry-creds was successfully configured $ minikube addons enable registry-creds ``` -For additional information on private container registries, see [this page](https://kubernetes.io/docs/Handbook/configure-pod-container/pull-image-private-registry/). +For additional information on private container registries, see [this page](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/). We recommend you use _ImagePullSecrets_, but if you would like to configure access on the minikube VM you can place the `.dockercfg` in the `/home/docker` directory or the `config.json` in the `/var/lib/kubelet` directory. Make sure to restart your kubelet (for kubeadm) process with `sudo systemctl restart kubelet`. @@ -107,4 +107,4 @@ docker push localhost:5000/myimage After the image is pushed, refer to it by `localhost:5000/{name}` in kubectl specs. -## \ No newline at end of file +## From b32cb5e6ba6875fad00556d4a92f3428cc84eacc Mon Sep 17 00:00:00 2001 From: colvin Date: Thu, 11 Jun 2020 09:51:13 -0400 Subject: [PATCH 02/20] Create a podsecuritypolicies addon Create a new addon, `podsecuritypolicies` that applies the PodSecurityPolicy and related RBAC configuration from the https://minikube.sigs.k8s.io/docs/tutorials/using_psp/ tutorial. Apparently, recent work on the addons system has invalidated the procedure shown in that tutorial, as the configuration is no longer automatically applied. The last known working version is `1.6.2`. This allows clusters started with `--extra-configs=apiserver.enable-admission-plugins=PodSecurityPolicy` to succeed, so long as they also include `--addons=podsecuritypolicies`. --- .../podsecuritypolicies.yaml.tmpl | 132 ++++++++++++++++++ pkg/addons/config.go | 5 + pkg/minikube/assets/addons.go | 8 ++ 3 files changed, 145 insertions(+) create mode 100644 deploy/addons/podsecuritypolicies/podsecuritypolicies.yaml.tmpl diff --git a/deploy/addons/podsecuritypolicies/podsecuritypolicies.yaml.tmpl b/deploy/addons/podsecuritypolicies/podsecuritypolicies.yaml.tmpl new file mode 100644 index 000000000000..fa4171fa91ef --- /dev/null +++ b/deploy/addons/podsecuritypolicies/podsecuritypolicies.yaml.tmpl @@ -0,0 +1,132 @@ +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: privileged + annotations: + seccomp.security.alpha.kubernetes.io/allowedProfileNames: "*" + labels: + addonmanager.kubernetes.io/mode: EnsureExists +spec: + privileged: true + allowPrivilegeEscalation: true + allowedCapabilities: + - "*" + volumes: + - "*" + hostNetwork: true + hostPorts: + - min: 0 + max: 65535 + hostIPC: true + hostPID: true + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'RunAsAny' + fsGroup: + rule: 'RunAsAny' +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: restricted + labels: + addonmanager.kubernetes.io/mode: EnsureExists +spec: + privileged: false + allowPrivilegeEscalation: false + requiredDropCapabilities: + - ALL + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + readOnlyRootFilesystem: false +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: psp:privileged + labels: + addonmanager.kubernetes.io/mode: EnsureExists +rules: +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - privileged +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: psp:restricted + labels: + addonmanager.kubernetes.io/mode: EnsureExists +rules: +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - restricted +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: default:restricted + labels: + addonmanager.kubernetes.io/mode: EnsureExists +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:restricted +subjects: +- kind: Group + name: system:authenticated + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: default:privileged + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: EnsureExists +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:privileged +subjects: +- kind: Group + name: system:masters + apiGroup: rbac.authorization.k8s.io +- kind: Group + name: system:nodes + apiGroup: rbac.authorization.k8s.io +- kind: Group + name: system:serviceaccounts:kube-system + apiGroup: rbac.authorization.k8s.io diff --git a/pkg/addons/config.go b/pkg/addons/config.go index 80fb4ad6f5f3..81d24e85f6a3 100644 --- a/pkg/addons/config.go +++ b/pkg/addons/config.go @@ -149,4 +149,9 @@ var Addons = []*Addon{ set: SetBool, callbacks: []setFn{enableOrDisableAddon}, }, + { + name: "podsecuritypolicies", + set: SetBool, + callbacks: []setFn{enableOrDisableAddon}, + }, } diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go index 67403a4c14d6..d5bd8adfdc46 100644 --- a/pkg/minikube/assets/addons.go +++ b/pkg/minikube/assets/addons.go @@ -81,6 +81,14 @@ var Addons = map[string]*Addon{ "0640", false), }, true, "default-storageclass"), + "podsecuritypolicies": NewAddon([]*BinAsset{ + MustBinAsset( + "deploy/addons/podsecuritypolicies/podsecuritypolicies.yaml.tmpl", + vmpath.GuestAddonsDir, + "podsecuritypolicies.yaml", + "0640", + false), + }, false, "podsecuritypolicies"), "storage-provisioner": NewAddon([]*BinAsset{ MustBinAsset( "deploy/addons/storage-provisioner/storage-provisioner.yaml.tmpl", From 24c9ce079d9497e38c63904e749823f78802f3ed Mon Sep 17 00:00:00 2001 From: Li Zhijian Date: Tue, 16 Jun 2020 09:29:22 +0800 Subject: [PATCH 03/20] cleanup TempDir properly a TempDir is like /tmp/minipath255070191/.minikube Signed-off-by: Li Zhijian --- cmd/minikube/cmd/root_test.go | 2 +- pkg/drivers/common_test.go | 2 +- pkg/drivers/hyperkit/network_test.go | 2 +- pkg/minikube/bootstrapper/certs_test.go | 2 +- pkg/minikube/notify/notify_test.go | 4 ++-- pkg/minikube/tests/dir_utils.go | 7 +++++++ 6 files changed, 13 insertions(+), 6 deletions(-) diff --git a/cmd/minikube/cmd/root_test.go b/cmd/minikube/cmd/root_test.go index b0c29ca4b14b..8658fc9b10be 100644 --- a/cmd/minikube/cmd/root_test.go +++ b/cmd/minikube/cmd/root_test.go @@ -115,7 +115,7 @@ func hideEnv(t *testing.T) func(t *testing.T) { func TestPreRunDirectories(t *testing.T) { // Make sure we create the required directories. tempDir := tests.MakeTempDir() - defer os.RemoveAll(tempDir) + defer tests.RemoveTempDir(tempDir) runCommand(RootCmd.PersistentPreRun) diff --git a/pkg/drivers/common_test.go b/pkg/drivers/common_test.go index 98982b9b36ea..cc250e03de6f 100644 --- a/pkg/drivers/common_test.go +++ b/pkg/drivers/common_test.go @@ -27,7 +27,7 @@ import ( func Test_createDiskImage(t *testing.T) { tmpdir := tests.MakeTempDir() - defer os.RemoveAll(tmpdir) + defer tests.RemoveTempDir(tmpdir) sshPath := filepath.Join(tmpdir, "ssh") if err := ioutil.WriteFile(sshPath, []byte("mysshkey"), 0644); err != nil { diff --git a/pkg/drivers/hyperkit/network_test.go b/pkg/drivers/hyperkit/network_test.go index 3bd538a66bc9..ae97c5e660df 100644 --- a/pkg/drivers/hyperkit/network_test.go +++ b/pkg/drivers/hyperkit/network_test.go @@ -51,7 +51,7 @@ var validLeases = []byte(`{ func Test_getIpAddressFromFile(t *testing.T) { tmpdir := tests.MakeTempDir() - defer os.RemoveAll(tmpdir) + defer tests.RemoveTempDir(tmpdir) dhcpFile := filepath.Join(tmpdir, "dhcp") if err := ioutil.WriteFile(dhcpFile, validLeases, 0644); err != nil { diff --git a/pkg/minikube/bootstrapper/certs_test.go b/pkg/minikube/bootstrapper/certs_test.go index 4f93aad180e9..d4226283d93a 100644 --- a/pkg/minikube/bootstrapper/certs_test.go +++ b/pkg/minikube/bootstrapper/certs_test.go @@ -30,7 +30,7 @@ import ( func TestSetupCerts(t *testing.T) { tempDir := tests.MakeTempDir() - defer os.RemoveAll(tempDir) + defer tests.RemoveTempDir(tempDir) k8s := config.KubernetesConfig{ APIServerName: constants.APIServerName, diff --git a/pkg/minikube/notify/notify_test.go b/pkg/minikube/notify/notify_test.go index 8f926daea63a..46e44d558976 100644 --- a/pkg/minikube/notify/notify_test.go +++ b/pkg/minikube/notify/notify_test.go @@ -43,7 +43,7 @@ func TestMaybePrintUpdateTextFromGithub(t *testing.T) { func TestShouldCheckURL(t *testing.T) { tempDir := tests.MakeTempDir() - defer os.RemoveAll(tempDir) + defer tests.RemoveTempDir(tempDir) lastUpdateCheckFilePath := filepath.Join(tempDir, "last_update_check") @@ -152,7 +152,7 @@ func TestGetLatestVersionFromURLMalformed(t *testing.T) { func TestMaybePrintUpdateText(t *testing.T) { tempDir := tests.MakeTempDir() - defer os.RemoveAll(tempDir) + defer tests.RemoveTempDir(tempDir) outputBuffer := tests.NewFakeFile() out.SetErrFile(outputBuffer) diff --git a/pkg/minikube/tests/dir_utils.go b/pkg/minikube/tests/dir_utils.go index 361d4752acd9..795742b772fc 100644 --- a/pkg/minikube/tests/dir_utils.go +++ b/pkg/minikube/tests/dir_utils.go @@ -45,6 +45,13 @@ func MakeTempDir() string { return localpath.MiniPath() } +func RemoveTempDir(tempdir string) { + if filepath.Base(tempdir) == ".minikube" { + tempdir = filepath.Dir(tempdir) + } + os.RemoveAll(tempdir) +} + // FakeFile satisfies fdWriter type FakeFile struct { b bytes.Buffer From e7121b72901c0e6e30068ba62ebd282fd289520c Mon Sep 17 00:00:00 2001 From: Li Zhijian Date: Tue, 16 Jun 2020 09:53:58 +0800 Subject: [PATCH 04/20] unify TempDir for testing Signed-off-by: Li Zhijian --- pkg/minikube/machine/client_test.go | 28 +++++---------------------- pkg/minikube/machine/filesync_test.go | 25 +++--------------------- 2 files changed, 8 insertions(+), 45 deletions(-) diff --git a/pkg/minikube/machine/client_test.go b/pkg/minikube/machine/client_test.go index b2cafc9428b2..cb9091fb5173 100644 --- a/pkg/minikube/machine/client_test.go +++ b/pkg/minikube/machine/client_test.go @@ -19,18 +19,15 @@ package machine import ( "bufio" "fmt" - "io/ioutil" - "log" "net" "os" - "path/filepath" "testing" "github.com/docker/machine/libmachine/drivers/plugin/localbinary" "k8s.io/minikube/pkg/minikube/driver" - "k8s.io/minikube/pkg/minikube/localpath" _ "k8s.io/minikube/pkg/minikube/registry/drvs/virtualbox" + testutil "k8s.io/minikube/pkg/minikube/tests" ) const vboxConfig = ` @@ -113,24 +110,9 @@ func TestLocalClientNewHost(t *testing.T) { } } -func makeTempDir() string { - tempDir, err := ioutil.TempDir("", "minipath") - if err != nil { - log.Fatal(err) - } - tempDir = filepath.Join(tempDir, ".minikube") - os.Setenv(localpath.MinikubeHome, tempDir) - return localpath.MiniPath() -} - func TestRunNotDriver(t *testing.T) { - tempDir := makeTempDir() - defer func() { //clean up tempdir - err := os.RemoveAll(tempDir) - if err != nil { - t.Errorf("failed to clean up temp folder %q", tempDir) - } - }() + tempDir := testutil.MakeTempDir() + defer testutil.RemoveTempDir(tempDir) StartDriver() if !localbinary.CurrentBinaryIsDockerMachine { t.Fatal("CurrentBinaryIsDockerMachine not set. This will break driver initialization.") @@ -140,8 +122,8 @@ func TestRunNotDriver(t *testing.T) { func TestRunDriver(t *testing.T) { // This test is a bit complicated. It verifies that when the root command is // called with the proper environment variables, we setup the libmachine driver. - tempDir := makeTempDir() - defer os.RemoveAll(tempDir) + tempDir := testutil.MakeTempDir() + defer testutil.RemoveTempDir(tempDir) os.Setenv(localbinary.PluginEnvKey, localbinary.PluginEnvVal) os.Setenv(localbinary.PluginEnvDriverName, driver.VirtualBox) diff --git a/pkg/minikube/machine/filesync_test.go b/pkg/minikube/machine/filesync_test.go index a69be410f025..a766ce0f1e28 100644 --- a/pkg/minikube/machine/filesync_test.go +++ b/pkg/minikube/machine/filesync_test.go @@ -17,26 +17,16 @@ limitations under the License. package machine import ( - "io/ioutil" "os" "path/filepath" "testing" "github.com/google/go-cmp/cmp" "k8s.io/minikube/pkg/minikube/localpath" + testutil "k8s.io/minikube/pkg/minikube/tests" "k8s.io/minikube/pkg/minikube/vmpath" ) -func setupTestDir() (string, error) { - path, err := ioutil.TempDir("", "minipath") - if err != nil { - return "", err - } - - os.Setenv(localpath.MinikubeHome, path) - return path, err -} - func TestAssetsFromDir(t *testing.T) { tests := []struct { description string @@ -107,17 +97,8 @@ func TestAssetsFromDir(t *testing.T) { for _, test := range tests { t.Run(test.description, func(t *testing.T) { - testDir, err := setupTestDir() - defer func() { //clean up tempdir - err := os.RemoveAll(testDir) - if err != nil { - t.Errorf("failed to clean up temp folder %q", testDir) - } - }() - if err != nil { - t.Errorf("got unexpected error creating test dir: %v", err) - return - } + testDir := testutil.MakeTempDir() + defer testutil.RemoveTempDir(testDir) testDirs = append(testDirs, testDir) testFileBaseDir := filepath.Join(testDir, test.baseDir) From 0b68d1d970b32866f56edc6b710a753edc229f2f Mon Sep 17 00:00:00 2001 From: Li Zhijian Date: Tue, 16 Jun 2020 09:29:59 +0800 Subject: [PATCH 05/20] remove executable permission Signed-off-by: Li Zhijian --- Makefile | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 Makefile diff --git a/Makefile b/Makefile old mode 100755 new mode 100644 From e6b905608d2584bf7c0f604353c529f71a5e99ba Mon Sep 17 00:00:00 2001 From: Li Zhijian Date: Wed, 17 Jun 2020 18:31:17 +0800 Subject: [PATCH 06/20] kubeconfig_test: cleanup temp kubeconfig file Signed-off-by: Li Zhijian --- pkg/minikube/kubeconfig/context_test.go | 1 + pkg/minikube/kubeconfig/kubeconfig_test.go | 3 +++ 2 files changed, 4 insertions(+) diff --git a/pkg/minikube/kubeconfig/context_test.go b/pkg/minikube/kubeconfig/context_test.go index 7725294441d7..a7bcc4af30c0 100644 --- a/pkg/minikube/kubeconfig/context_test.go +++ b/pkg/minikube/kubeconfig/context_test.go @@ -26,6 +26,7 @@ import ( func TestDeleteContext(t *testing.T) { // See kubeconfig_test fn := tempFile(t, kubeConfigWithoutHTTPS) + defer os.Remove(fn) if err := DeleteContext("la-croix", fn); err != nil { t.Fatal(err) } diff --git a/pkg/minikube/kubeconfig/kubeconfig_test.go b/pkg/minikube/kubeconfig/kubeconfig_test.go index 8a0df1712011..54e585b5d5e8 100644 --- a/pkg/minikube/kubeconfig/kubeconfig_test.go +++ b/pkg/minikube/kubeconfig/kubeconfig_test.go @@ -263,6 +263,7 @@ func TestVerifyEndpoint(t *testing.T) { t.Run(test.description, func(t *testing.T) { t.Parallel() configFilename := tempFile(t, test.existing) + defer os.Remove(configFilename) err := VerifyEndpoint("minikube", test.hostname, test.port, configFilename) if err != nil && !test.err { t.Errorf("Got unexpected error: %v", err) @@ -330,6 +331,7 @@ func TestUpdateIP(t *testing.T) { t.Run(test.description, func(t *testing.T) { t.Parallel() configFilename := tempFile(t, test.existing) + defer os.Remove(configFilename) statusActual, err := UpdateEndpoint("minikube", test.hostname, test.port, configFilename) if err != nil && !test.err { t.Errorf("Got unexpected error: %v", err) @@ -419,6 +421,7 @@ func Test_Endpoint(t *testing.T) { for _, test := range tests { t.Run(test.description, func(t *testing.T) { configFilename := tempFile(t, test.cfg) + defer os.Remove(configFilename) hostname, port, err := Endpoint("minikube", configFilename) if err != nil && !test.err { t.Errorf("Got unexpected error: %v", err) From 0ad929f17e1e15d9395b124348b6f0fc38504c65 Mon Sep 17 00:00:00 2001 From: Li Zhijian Date: Wed, 17 Jun 2020 18:35:38 +0800 Subject: [PATCH 07/20] test.sh: cleanup cov_tmp Signed-off-by: Li Zhijian --- test.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/test.sh b/test.sh index 79a350164853..82b3cb2bf29a 100755 --- a/test.sh +++ b/test.sh @@ -67,6 +67,7 @@ then ${pkgs} \ && echo ok || ((exitcode += 32)) tail -n +2 "${cov_tmp}" >>"${COVERAGE_PATH}" + rm ${cov_tmp} fi exit "${exitcode}" From 3d2ae5423cd1c0723a00ce778bc512d60e76e61a Mon Sep 17 00:00:00 2001 From: colvin Date: Thu, 18 Jun 2020 17:00:08 -0400 Subject: [PATCH 08/20] Rename podsecuritypolicies addon to pod-security-policy --- .../pod-security-policy.yaml.tmpl} | 0 pkg/addons/config.go | 2 +- pkg/minikube/assets/addons.go | 8 ++++---- 3 files changed, 5 insertions(+), 5 deletions(-) rename deploy/addons/{podsecuritypolicies/podsecuritypolicies.yaml.tmpl => pod-security-policy/pod-security-policy.yaml.tmpl} (100%) diff --git a/deploy/addons/podsecuritypolicies/podsecuritypolicies.yaml.tmpl b/deploy/addons/pod-security-policy/pod-security-policy.yaml.tmpl similarity index 100% rename from deploy/addons/podsecuritypolicies/podsecuritypolicies.yaml.tmpl rename to deploy/addons/pod-security-policy/pod-security-policy.yaml.tmpl diff --git a/pkg/addons/config.go b/pkg/addons/config.go index 81d24e85f6a3..6ffe0f111311 100644 --- a/pkg/addons/config.go +++ b/pkg/addons/config.go @@ -150,7 +150,7 @@ var Addons = []*Addon{ callbacks: []setFn{enableOrDisableAddon}, }, { - name: "podsecuritypolicies", + name: "pod-security-policy", set: SetBool, callbacks: []setFn{enableOrDisableAddon}, }, diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go index d5bd8adfdc46..6d57174252c5 100644 --- a/pkg/minikube/assets/addons.go +++ b/pkg/minikube/assets/addons.go @@ -81,14 +81,14 @@ var Addons = map[string]*Addon{ "0640", false), }, true, "default-storageclass"), - "podsecuritypolicies": NewAddon([]*BinAsset{ + "pod-security-policy": NewAddon([]*BinAsset{ MustBinAsset( - "deploy/addons/podsecuritypolicies/podsecuritypolicies.yaml.tmpl", + "deploy/addons/pod-security-policy/pod-security-policy.yaml.tmpl", vmpath.GuestAddonsDir, - "podsecuritypolicies.yaml", + "pod-security-policy.yaml", "0640", false), - }, false, "podsecuritypolicies"), + }, false, "pod-security-policy"), "storage-provisioner": NewAddon([]*BinAsset{ MustBinAsset( "deploy/addons/storage-provisioner/storage-provisioner.yaml.tmpl", From 08ee21fd439b988ce46c30153aad38291732ce0a Mon Sep 17 00:00:00 2001 From: colvin Date: Thu, 18 Jun 2020 17:33:34 -0400 Subject: [PATCH 09/20] update the Pod Security Policies tutorial --- site/content/en/docs/tutorials/using_psp.md | 44 +++++++++++++++++++-- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/site/content/en/docs/tutorials/using_psp.md b/site/content/en/docs/tutorials/using_psp.md index b77fc4c46f9c..38123c73c0af 100644 --- a/site/content/en/docs/tutorials/using_psp.md +++ b/site/content/en/docs/tutorials/using_psp.md @@ -13,18 +13,33 @@ This tutorial explains how to start minikube with Pod Security Policies (PSP) en ## Prerequisites -- Minikube 1.5.2 with Kubernetes 1.16.x or higher +- Minikube 1.11.1 with Kubernetes 1.16.x or higher ## Tutorial -Before starting minikube, you need to give it the PSP YAMLs in order to allow minikube to bootstrap. +Start minikube with the `PodSecurityPolicy` admission controller and the +`pod-security-policy` addon enabled. -Create the directory: +`minikube start --extra-config=apiserver.enable-admission-plugins=PodSecurityPolicy --addons=pod-security-policy` + +The `pod-security-policy` addon must be enabled along with the admission +controller to prevent issues during bootstrap. + +## Older versions of minikube + +Older versions of minikube do not ship with the `pod-security-policy` addon, so +the policies that addon enables must be separately applied to the cluster. + +## Minikube 1.5.2 through 1.6.2 + +Before starting minikube, you need to give it the PSP YAMLs in order to allow minikube to bootstrap. + +Create the directory: `mkdir -p ~/.minikube/files/etc/kubernetes/addons` Copy the YAML below into this file: `~/.minikube/files/etc/kubernetes/addons/psp.yaml` -Now start minikube: +Now start minikube: `minikube start --extra-config=apiserver.enable-admission-plugins=PodSecurityPolicy` ```yaml @@ -161,3 +176,24 @@ subjects: name: system:serviceaccounts:kube-system apiGroup: rbac.authorization.k8s.io ``` + +### Minikube between 1.6.2 and 1.11.1 + +With minikube versions greater than 1.6.2 and less than 1.11.1, the YAML files +shown above will not be automatically applied to the cluster. You may have +errors during bootstrap of the cluster if the admission controller is enabled. + +To use Pod Security Policies with these versions of minikube, first start a +cluster without the `PodSecurityPolicy` admission controller enabled. + +Next, apply the YAML shown above to the cluster. + +Finally, stop the cluster and then restart it with the admission controller +enabled. + +``` +minikube start +kubectl apply -f /path/to/psp.yaml +minikube stop +minikube start --extra-config=apiserver.enable-admission-plugins=PodSecurityPolicy +``` From 79eaa8778afaac18010ead36cee2647557e1ac66 Mon Sep 17 00:00:00 2001 From: Ilya Danilkin Date: Thu, 14 May 2020 18:13:14 +0300 Subject: [PATCH 10/20] bsutil: fix error msg for `--extra-config` parsing --- pkg/minikube/bootstrapper/bsutil/extraconfig.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/minikube/bootstrapper/bsutil/extraconfig.go b/pkg/minikube/bootstrapper/bsutil/extraconfig.go index 0f64b89f10db..47b61d92e4cf 100644 --- a/pkg/minikube/bootstrapper/bsutil/extraconfig.go +++ b/pkg/minikube/bootstrapper/bsutil/extraconfig.go @@ -136,7 +136,7 @@ func newComponentOptions(opts config.ExtraOptionSlice, version semver.Version, f var kubeadmExtraArgs []componentOptions for _, extraOpt := range opts { if _, ok := componentToKubeadmConfigKey[extraOpt.Component]; !ok { - return nil, fmt.Errorf("unknown component %q. valid components are: %v", componentToKubeadmConfigKey, componentToKubeadmConfigKey) + return nil, fmt.Errorf("unknown component %q. valid components are: %v", extraOpt.Component, componentToKubeadmConfigKey) } } From d1ab5312d5641d7d9fa6c0b090bfd0f6b3b65d08 Mon Sep 17 00:00:00 2001 From: Ilya Danilkin Date: Fri, 15 May 2020 17:40:01 +0300 Subject: [PATCH 11/20] bsutil: list valid keys as slice instead of a map --- pkg/minikube/bootstrapper/bsutil/extraconfig.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/minikube/bootstrapper/bsutil/extraconfig.go b/pkg/minikube/bootstrapper/bsutil/extraconfig.go index 47b61d92e4cf..775f04441492 100644 --- a/pkg/minikube/bootstrapper/bsutil/extraconfig.go +++ b/pkg/minikube/bootstrapper/bsutil/extraconfig.go @@ -134,18 +134,18 @@ func defaultOptionsForComponentAndVersion(component string, version semver.Versi // newComponentOptions creates a new componentOptions func newComponentOptions(opts config.ExtraOptionSlice, version semver.Version, featureGates string, cp config.Node) ([]componentOptions, error) { var kubeadmExtraArgs []componentOptions - for _, extraOpt := range opts { - if _, ok := componentToKubeadmConfigKey[extraOpt.Component]; !ok { - return nil, fmt.Errorf("unknown component %q. valid components are: %v", extraOpt.Component, componentToKubeadmConfigKey) - } - } - keys := []string{} for k := range componentToKubeadmConfigKey { keys = append(keys, k) } sort.Strings(keys) + for _, extraOpt := range opts { + if _, ok := componentToKubeadmConfigKey[extraOpt.Component]; !ok { + return nil, fmt.Errorf("unknown component %q. valid components are: %v", extraOpt.Component, keys) + } + } + for _, component := range keys { kubeadmComponentKey := componentToKubeadmConfigKey[component] if kubeadmComponentKey == "" { From ccffa40cfe221c095b7352685f5b835b802979e6 Mon Sep 17 00:00:00 2001 From: Ilya Danilkin Date: Sun, 17 May 2020 15:04:44 +0300 Subject: [PATCH 12/20] bsutil: extra-opts validation --- cmd/minikube/cmd/start.go | 14 +++++ .../bootstrapper/bsutil/extraconfig.go | 31 ++++++---- .../bootstrapper/bsutil/extraconfig_test.go | 59 +++++++++++++++++++ pkg/minikube/bootstrapper/bsutil/kubeadm.go | 16 ++++- 4 files changed, 105 insertions(+), 15 deletions(-) create mode 100644 pkg/minikube/bootstrapper/bsutil/extraconfig_test.go diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index 85e705ced311..f2cdf882c2ed 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -872,6 +872,20 @@ func validateFlags(cmd *cobra.Command, drvName string) { } } + // validate kubeadm extra args + if invalidOpts := bsutil.FindInvalidExtraConfigFlags(config.ExtraOptions); invalidOpts != nil { + out.ErrT( + out.Warning, + "These --extra-config parameters are invalid: {{.invalid_extra_opts}}", + out.V{"invalid_extra_opts": invalidOpts}, + ) + exit.WithCodeT( + exit.Config, + "Valid components are: {{.valid_extra_opts}}", + out.V{"valid_extra_opts": bsutil.KubeadmExtraConfigOpts}, + ) + } + // check that kubeadm extra args contain only allowed parameters for param := range config.ExtraOptions.AsMap().Get(bsutil.Kubeadm) { if !config.ContainsParam(bsutil.KubeadmExtraArgsAllowed[bsutil.KubeadmCmdParam], param) && diff --git a/pkg/minikube/bootstrapper/bsutil/extraconfig.go b/pkg/minikube/bootstrapper/bsutil/extraconfig.go index 775f04441492..825d22390a54 100644 --- a/pkg/minikube/bootstrapper/bsutil/extraconfig.go +++ b/pkg/minikube/bootstrapper/bsutil/extraconfig.go @@ -95,6 +95,21 @@ func CreateFlagsFromExtraArgs(extraOptions config.ExtraOptionSlice) string { return convertToFlags(kubeadmExtraOpts) } +// FindInvalidExtraConfigFlags returns all invalid 'extra-config' options +func FindInvalidExtraConfigFlags(opts config.ExtraOptionSlice) []string { + invalidOptsMap := make(map[string]struct{}) + var invalidOpts []string + for _, extraOpt := range opts { + if _, ok := componentToKubeadmConfigKey[extraOpt.Component]; !ok { + if _, ok := invalidOptsMap[extraOpt.Component]; !ok { + invalidOpts = append(invalidOpts, extraOpt.Component) + invalidOptsMap[extraOpt.Component] = struct{}{} + } + } + } + return invalidOpts +} + // extraConfigForComponent generates a map of flagname-value pairs for a k8s // component. func extraConfigForComponent(component string, opts config.ExtraOptionSlice, version semver.Version) (map[string]string, error) { @@ -133,20 +148,12 @@ func defaultOptionsForComponentAndVersion(component string, version semver.Versi // newComponentOptions creates a new componentOptions func newComponentOptions(opts config.ExtraOptionSlice, version semver.Version, featureGates string, cp config.Node) ([]componentOptions, error) { - var kubeadmExtraArgs []componentOptions - keys := []string{} - for k := range componentToKubeadmConfigKey { - keys = append(keys, k) - } - sort.Strings(keys) - - for _, extraOpt := range opts { - if _, ok := componentToKubeadmConfigKey[extraOpt.Component]; !ok { - return nil, fmt.Errorf("unknown component %q. valid components are: %v", extraOpt.Component, keys) - } + if invalidOpts := FindInvalidExtraConfigFlags(opts); invalidOpts != nil { + return nil, fmt.Errorf("unknown components %v. valid components are: %v", invalidOpts, KubeadmExtraConfigOpts) } - for _, component := range keys { + var kubeadmExtraArgs []componentOptions + for _, component := range KubeadmExtraConfigOpts { kubeadmComponentKey := componentToKubeadmConfigKey[component] if kubeadmComponentKey == "" { continue diff --git a/pkg/minikube/bootstrapper/bsutil/extraconfig_test.go b/pkg/minikube/bootstrapper/bsutil/extraconfig_test.go new file mode 100644 index 000000000000..cb43a77f1507 --- /dev/null +++ b/pkg/minikube/bootstrapper/bsutil/extraconfig_test.go @@ -0,0 +1,59 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package bsutil will eventually be renamed to kubeadm package after getting rid of older one +package bsutil + +import ( + "reflect" + "testing" + + "k8s.io/minikube/pkg/minikube/config" +) + +func TestFindInvalidExtraConfigFlags(t *testing.T) { + defaultOpts := getExtraOpts() + badOption1 := config.ExtraOption{Component: "bad_option_1"} + badOption2 := config.ExtraOption{Component: "bad_option_2"} + tests := []struct { + name string + opts config.ExtraOptionSlice + want []string + }{ + { + name: "with valid options only", + opts: defaultOpts, + want: nil, + }, + { + name: "with invalid options", + opts: append(defaultOpts, badOption1, badOption2), + want: []string{"bad_option_1", "bad_option_2"}, + }, + { + name: "with invalid options and duplicates", + opts: append(defaultOpts, badOption2, badOption1, badOption1), + want: []string{"bad_option_2", "bad_option_1"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := FindInvalidExtraConfigFlags(tt.opts); !reflect.DeepEqual(got, tt.want) { + t.Errorf("FindInvalidExtraConfigFlags() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/minikube/bootstrapper/bsutil/kubeadm.go b/pkg/minikube/bootstrapper/bsutil/kubeadm.go index 7fa1f157dfc9..0ebd55453196 100644 --- a/pkg/minikube/bootstrapper/bsutil/kubeadm.go +++ b/pkg/minikube/bootstrapper/bsutil/kubeadm.go @@ -147,15 +147,25 @@ func GenerateKubeadmYAML(cc config.ClusterConfig, n config.Node, r cruntime.Mana // These are the components that can be configured // through the "extra-config" const ( - Kubelet = "kubelet" - Kubeadm = "kubeadm" Apiserver = "apiserver" - Scheduler = "scheduler" ControllerManager = "controller-manager" + Kubeadm = "kubeadm" + Kubelet = "kubelet" Kubeproxy = "kube-proxy" + Scheduler = "scheduler" Etcd = "etcd" ) +// KubeadmExtraConfigOpts is a list of allowed "extra-config" components +var KubeadmExtraConfigOpts = []string{ + Apiserver, + ControllerManager, + Kubeadm, + Kubelet, + Kubeproxy, + Scheduler, +} + // InvokeKubeadm returns the invocation command for Kubeadm func InvokeKubeadm(version string) string { return fmt.Sprintf("sudo env PATH=%s:$PATH kubeadm", binRoot(version)) From b470860a5a44c4a9bcace83c6159ef72d3a008e4 Mon Sep 17 00:00:00 2001 From: Ilya Danilkin Date: Mon, 25 May 2020 00:17:07 +0300 Subject: [PATCH 13/20] fix condition check for extraOpts validation --- cmd/minikube/cmd/start.go | 2 +- pkg/minikube/bootstrapper/bsutil/extraconfig.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index f2cdf882c2ed..9ed093816f8c 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -873,7 +873,7 @@ func validateFlags(cmd *cobra.Command, drvName string) { } // validate kubeadm extra args - if invalidOpts := bsutil.FindInvalidExtraConfigFlags(config.ExtraOptions); invalidOpts != nil { + if invalidOpts := bsutil.FindInvalidExtraConfigFlags(config.ExtraOptions); len(invalidOpts) > 0 { out.ErrT( out.Warning, "These --extra-config parameters are invalid: {{.invalid_extra_opts}}", diff --git a/pkg/minikube/bootstrapper/bsutil/extraconfig.go b/pkg/minikube/bootstrapper/bsutil/extraconfig.go index 825d22390a54..946d087836d2 100644 --- a/pkg/minikube/bootstrapper/bsutil/extraconfig.go +++ b/pkg/minikube/bootstrapper/bsutil/extraconfig.go @@ -148,7 +148,7 @@ func defaultOptionsForComponentAndVersion(component string, version semver.Versi // newComponentOptions creates a new componentOptions func newComponentOptions(opts config.ExtraOptionSlice, version semver.Version, featureGates string, cp config.Node) ([]componentOptions, error) { - if invalidOpts := FindInvalidExtraConfigFlags(opts); invalidOpts != nil { + if invalidOpts := FindInvalidExtraConfigFlags(opts); len(invalidOpts) > 0 { return nil, fmt.Errorf("unknown components %v. valid components are: %v", invalidOpts, KubeadmExtraConfigOpts) } From 83bb1e32d2dbecf0400f2f3e959956efef2cabb7 Mon Sep 17 00:00:00 2001 From: Ilya Danilkin Date: Sun, 28 Jun 2020 22:51:29 +0300 Subject: [PATCH 14/20] reoder extra-config consts --- pkg/minikube/bootstrapper/bsutil/kubeadm.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/minikube/bootstrapper/bsutil/kubeadm.go b/pkg/minikube/bootstrapper/bsutil/kubeadm.go index 0ebd55453196..87c0e9eddee6 100644 --- a/pkg/minikube/bootstrapper/bsutil/kubeadm.go +++ b/pkg/minikube/bootstrapper/bsutil/kubeadm.go @@ -149,11 +149,11 @@ func GenerateKubeadmYAML(cc config.ClusterConfig, n config.Node, r cruntime.Mana const ( Apiserver = "apiserver" ControllerManager = "controller-manager" - Kubeadm = "kubeadm" - Kubelet = "kubelet" - Kubeproxy = "kube-proxy" Scheduler = "scheduler" Etcd = "etcd" + Kubeadm = "kubeadm" + Kubeproxy = "kube-proxy" + Kubelet = "kubelet" ) // KubeadmExtraConfigOpts is a list of allowed "extra-config" components From e1f443e121daaa2fdd082b90614746183a927523 Mon Sep 17 00:00:00 2001 From: Ilya Danilkin Date: Sun, 28 Jun 2020 23:29:25 +0300 Subject: [PATCH 15/20] bsutil: add 'etcd' to the KubeadmExtraConfigOpts --- pkg/minikube/bootstrapper/bsutil/kubeadm.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/minikube/bootstrapper/bsutil/kubeadm.go b/pkg/minikube/bootstrapper/bsutil/kubeadm.go index 87c0e9eddee6..db5e24485ff4 100644 --- a/pkg/minikube/bootstrapper/bsutil/kubeadm.go +++ b/pkg/minikube/bootstrapper/bsutil/kubeadm.go @@ -160,10 +160,11 @@ const ( var KubeadmExtraConfigOpts = []string{ Apiserver, ControllerManager, + Scheduler, + Etcd, Kubeadm, Kubelet, Kubeproxy, - Scheduler, } // InvokeKubeadm returns the invocation command for Kubeadm From 39eaf835d0b64a991ce43c5320220896185608b7 Mon Sep 17 00:00:00 2001 From: Matt Broberg Date: Tue, 30 Jun 2020 10:35:15 -0500 Subject: [PATCH 16/20] Steps to get podman running - Remove repetitive notes in podman documentation - Fix links in the usage document - Tidy the language and grammar --- .../en/docs/drivers/includes/podman_usage.inc | 20 ++++++------ site/content/en/docs/drivers/podman.md | 31 ++++++++++++++----- 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/site/content/en/docs/drivers/includes/podman_usage.inc b/site/content/en/docs/drivers/includes/podman_usage.inc index 22c71a96f8a1..63940a3c548c 100644 --- a/site/content/en/docs/drivers/includes/podman_usage.inc +++ b/site/content/en/docs/drivers/includes/podman_usage.inc @@ -1,20 +1,22 @@ -## experimental +## Experimental -This is an experimental driver. please use it only for experimental reasons. -for a better kubernetes in container experience, use docker [driver]({{< ref "/docs/drivers/docker/" >}}) +This is an experimental driver. Please use it only for experimental reasons until it has reached maturity. For a more reliable minikube experience, use a non-experimental driver, like [Docker]({{< ref "/docs/drivers/docker.md" >}}). -## Install Podman +## Usage -- [Podman](https://podman.io/getting-started/installation.html) +It's recommended to run minikube with the podman driver and [CRI-O container runtime](https://https://cri-o.io/): -## Usage +```shell +minikube start --driver=podman --container-runtime=cri-o +``` -Start a cluster using the podman driver: +Alternatively, start minikube with the podman driver only: ```shell -minikube start --driver=podman +minikube start --driver=podman ``` -To make docker the default driver: + +To make podman the default driver: ```shell minikube config set driver podman diff --git a/site/content/en/docs/drivers/podman.md b/site/content/en/docs/drivers/podman.md index 286429b0effc..e47ebc6b8034 100644 --- a/site/content/en/docs/drivers/podman.md +++ b/site/content/en/docs/drivers/podman.md @@ -11,21 +11,36 @@ aliases: This driver is experimental and in active development. Help wanted! {{% /pageinfo %}} -The podman driver is another kubernetes in container driver for minikube. similar to [docker](https://minikube.sigs.k8s.io/docs/drivers/docker/) driver. The podman driver is experimental, and only supported on Linux and macOS (with a remote podman server). +The podman driver is an alternative container runtime to the [Docker]({{< ref "/docs/drivers/docker.md >}}) driver. ## Requirements -- Install [Podman](https://podman.io/getting-started/installation) -- amd64 system +- Linux or macOS operating systems on amd64 architecture +- Install [podman](https://podman.io/getting-started/installation.html) -## Try it with CRI-O container runtime. - -```shell -minikube start --driver=podman --container-runtime=cri-o -``` {{% readfile file="/docs/drivers/includes/podman_usage.inc" %}} ## Known Issues - Podman driver is not supported on non-amd64 architectures such as arm yet. For non-amd64 archs please use [other drivers]({{< ref "/docs/drivers/_index.md" >}}) +- Podman requirements passwordless running of sudo. If you run into an error about sudo, do the following: + +```shell +$ sudo visudo +``` +Then append the following to the section *at the very bottom* of the file where `username` is your user account. + +```shell +username ALL=(ALL) NOPASSWD: /usr/bin/podman +``` + +Be sure this text is *after* `#includedir /etc/sudoers.d`. To confirm it worked, try: + +```shell +sudo -k -n podman version +``` + +## Troubleshooting + +- Run `minikube start --alsologtostderr -v=7` to debug errors and crashes From aed150d1804a0324abb6efca0446be7add6f9a16 Mon Sep 17 00:00:00 2001 From: Medya Ghazizadeh Date: Tue, 30 Jun 2020 18:11:34 -0700 Subject: [PATCH 17/20] Update kvm2.md --- site/content/en/docs/drivers/kvm2.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/site/content/en/docs/drivers/kvm2.md b/site/content/en/docs/drivers/kvm2.md index 3f3e75986ccb..016f7a22e74e 100644 --- a/site/content/en/docs/drivers/kvm2.md +++ b/site/content/en/docs/drivers/kvm2.md @@ -37,6 +37,11 @@ The `minikube start` command supports 3 additional kvm specific flags: Also see [co/kvm2 open issues](https://github.com/kubernetes/minikube/labels/co%2Fkvm2) +### Nested Virtulization + +if you are running kvm inside a nested virtualized VM follow [this tutorial](https://stafwag.github.io/blog/blog/2018/06/04/nested-virtualization-in-kvm/) to config the kernel modules + + ## Troubleshooting * Run `minikube start --alsologtostderr -v=7` to debug crashes From 0be0781cd4d09780d7d2191909472675d81ae758 Mon Sep 17 00:00:00 2001 From: Medya Ghazizadeh Date: Tue, 30 Jun 2020 18:36:49 -0700 Subject: [PATCH 18/20] Update kvm2.md --- site/content/en/docs/drivers/kvm2.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/site/content/en/docs/drivers/kvm2.md b/site/content/en/docs/drivers/kvm2.md index 016f7a22e74e..59b561501e25 100644 --- a/site/content/en/docs/drivers/kvm2.md +++ b/site/content/en/docs/drivers/kvm2.md @@ -39,8 +39,7 @@ Also see [co/kvm2 open issues](https://github.com/kubernetes/minikube/labels/co% ### Nested Virtulization -if you are running kvm inside a nested virtualized VM follow [this tutorial](https://stafwag.github.io/blog/blog/2018/06/04/nested-virtualization-in-kvm/) to config the kernel modules - +if you are running kvm inside a nested virtualized VM follow [this tutorial](https://stafwag.github.io/blog/blog/2018/06/04/nested-virtualization-in-kvm/) to config the kernel modules. also https://computingforgeeks.com/how-to-install-kvm-virtualization-on-debian/ ## Troubleshooting From 1de1195cfa006c0a9d8e91775820e28a84e87942 Mon Sep 17 00:00:00 2001 From: Medya Ghazizadeh Date: Tue, 7 Jul 2020 16:01:47 -0700 Subject: [PATCH 19/20] Update kvm2.md --- site/content/en/docs/drivers/kvm2.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/content/en/docs/drivers/kvm2.md b/site/content/en/docs/drivers/kvm2.md index 59b561501e25..69ecf67e7cad 100644 --- a/site/content/en/docs/drivers/kvm2.md +++ b/site/content/en/docs/drivers/kvm2.md @@ -39,7 +39,7 @@ Also see [co/kvm2 open issues](https://github.com/kubernetes/minikube/labels/co% ### Nested Virtulization -if you are running kvm inside a nested virtualized VM follow [this tutorial](https://stafwag.github.io/blog/blog/2018/06/04/nested-virtualization-in-kvm/) to config the kernel modules. also https://computingforgeeks.com/how-to-install-kvm-virtualization-on-debian/ +If you are running KVM in a nested virtualization environment ensure your config the kernel modules correctly follow either [this](https://stafwag.github.io/blog/blog/2018/06/04/nested-virtualization-in-kvm/) or [this](VM follow to config the kernel modules. also https://computingforgeeks.com/how-to-install-kvm-virtualization-on-debian/) tutorial. ## Troubleshooting From 1a769c3c91c173afa62be77b2f425cb11b390981 Mon Sep 17 00:00:00 2001 From: vinu2003 Date: Thu, 9 Jul 2020 02:44:37 +1000 Subject: [PATCH 20/20] fixed the unterminated qouted string --- site/content/en/docs/drivers/podman.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/content/en/docs/drivers/podman.md b/site/content/en/docs/drivers/podman.md index e47ebc6b8034..d1e827130682 100644 --- a/site/content/en/docs/drivers/podman.md +++ b/site/content/en/docs/drivers/podman.md @@ -11,7 +11,7 @@ aliases: This driver is experimental and in active development. Help wanted! {{% /pageinfo %}} -The podman driver is an alternative container runtime to the [Docker]({{< ref "/docs/drivers/docker.md >}}) driver. +The podman driver is an alternative container runtime to the [Docker]({{< ref "/docs/drivers/docker.md" >}}) driver. ## Requirements