Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fixes get ETCD version from kubernetes constants #11290 #12084

Merged
merged 15 commits into from
Dec 7, 2021
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -990,6 +990,12 @@ else
go run update_kubernetes_version.go)
endif

.PHONY: update-kubeadm-constants
update-kubeadm-constants:
(cd hack/update/kubeadm_constants && \
go run update_kubeadm_constants.go)
gofmt -w pkg/minikube/constants/constants_kubeadm_images.go

.PHONY: stress
stress: ## run the stress tests
go test -test.v -test.timeout=2h ./test/stress -loops=10 | tee "./out/testout_$(COMMIT_SHORT).txt"
Expand Down
205 changes: 205 additions & 0 deletions hack/update/kubeadm_constants/update_kubeadm_constants.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,205 @@
/*
Copyright 2021 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
"bytes"
"context"
"errors"
"flag"
"fmt"
"io"
"net/http"
"os"
"os/exec"
"strings"
"text/template"
"time"

"golang.org/x/mod/semver"
"k8s.io/klog/v2"
"k8s.io/minikube/hack/update"
"k8s.io/minikube/pkg/minikube/constants"
)

const (
// default context timeout
cxTimeout = 300 * time.Second
kubeadmReleaseURL = "https://storage.googleapis.com/kubernetes-release/release/%s/bin/linux/amd64/kubeadm"
kubeadmBinaryName = "kubeadm-linux-amd64-%s"
minikubeConstantsFilePath = "pkg/minikube/constants/constants_kubeadm_images.go"
kubeadmImagesTemplate = `
{{- range $version, $element := .}}
"{{$version}}": {
{{- range $image, $tag := $element}}
"{{$image}}": "{{$tag}}",
{{- end}}
},{{- end}}`
)

// Data contains kubeadm Images map
type Data struct {
ImageMap string `json:"ImageMap"`
}

func main() {

inputVersion := flag.Lookup("kubernetes-version").Value.String()

imageVersions := make([]string, 0)

// set a context with defined timeout
ctx, cancel := context.WithTimeout(context.Background(), cxTimeout)
defer cancel()

if inputVersion == "latest" {
stableImageVersion, latestImageVersion, _, _, err := getK8sVersions(ctx, "kubernetes", "kubernetes")
if err != nil {
klog.Fatal(err)
}
imageVersions = append(imageVersions, stableImageVersion, latestImageVersion)
} else if semver.IsValid(inputVersion) {
imageVersions = append(imageVersions, inputVersion)
} else {
klog.Fatal(errors.New("invalid version"))
}

for _, imageVersion := range imageVersions {
imageMapString, err := getKubeadmImagesMapString(imageVersion)
if err != nil {
klog.Fatalln(err)
}

var data Data
schema := map[string]update.Item{
minikubeConstantsFilePath: {
Replace: map[string]string{},
},
}

majorMinorVersion := semver.MajorMinor(imageVersion)

if _, ok := constants.KubeadmImages[majorMinorVersion]; !ok {
data = Data{ImageMap: imageMapString}
schema[minikubeConstantsFilePath].Replace[`KubeadmImages = .*`] =
`KubeadmImages = map[string]map[string]string{ {{.ImageMap}}`
} else {
data = Data{ImageMap: strings.TrimLeft(imageMapString, "\n")}
versionIdentifier := fmt.Sprintf(`"%s": {[^}]+},`, majorMinorVersion)
schema[minikubeConstantsFilePath].Replace[versionIdentifier] = "{{.ImageMap}}"
}

update.Apply(ctx, schema, data, "", "", -1)
}
}

func getKubeadmImagesMapString(version string) (string, error) {
url := fmt.Sprintf(kubeadmReleaseURL, version)
fileName := fmt.Sprintf(kubeadmBinaryName, version)
if err := downloadFile(url, fileName); err != nil {
klog.Errorf("failed to download kubeadm binary %s", err.Error())
return "", err
}

kubeadmCommand := fmt.Sprintf("./%s", fileName)
args := []string{"config", "images", "list"}
imageListString, err := executeCommand(kubeadmCommand, args...)
if err != nil {
klog.Errorf("failed to execute kubeadm command %s", kubeadmCommand)
return "", err
}

if err := os.Remove(fileName); err != nil {
klog.Errorf("failed to remove binary %s", fileName)
}

return formatKubeadmImageList(version, imageListString)
}

func formatKubeadmImageList(version, data string) (string, error) {
templateData := make(map[string]map[string]string)
majorMinorVersion := semver.MajorMinor(version)
templateData[majorMinorVersion] = make(map[string]string)
lines := strings.Split(data, "\n")
for _, line := range lines {
imageTag := strings.Split(line, ":")
if len(imageTag) == 2 {
templateData[majorMinorVersion][imageTag[0]] = imageTag[1]
}
}

imageTemplate := template.New("kubeadmImage")
t, err := imageTemplate.Parse(kubeadmImagesTemplate)
if err != nil {
klog.Errorf("failed to create kubeadm image map template %s", err.Error())
return "", err
}

var bytesBuffer bytes.Buffer
if err := t.Execute(&bytesBuffer, &templateData); err != nil {
return "", err
}

return bytesBuffer.String(), nil
}

func downloadFile(url, fileName string) error {
file, err := os.Create(fileName)
if err != nil {
return err
}
defer file.Close()

response, err := http.Get(url)
if err != nil {
return err
}
defer response.Body.Close()

if response.StatusCode != http.StatusOK {
return fmt.Errorf("non success status code, while downloading file: %s from: %s", fileName, url)
}

if _, err := io.Copy(file, response.Body); err != nil {
return err
}

return os.Chmod(fileName, os.ModePerm)
}

func executeCommand(command string, args ...string) (string, error) {
output, err := exec.Command(command, args...).Output()
if err != nil {
return "", err
}
return string(output), nil
}

// getK8sVersion returns Kubernetes versions.
func getK8sVersions(ctx context.Context, owner, repo string) (stable, latest, latestMM, latestP0 string, err error) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

latestMM & latestP0 aren't used anywhere, couldn't we just replace this whole function with a direct call to update.GHReleases?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

removed latestMM & latestP0.

// get Kubernetes versions from GitHub Releases
stable, latest, err = update.GHReleases(ctx, owner, repo)
if err != nil || !semver.IsValid(stable) || !semver.IsValid(latest) {
return "", "", "", "", err
}
latestMM = semver.MajorMinor(latest)
latestP0 = latestMM + ".0"
if semver.Compare(stable, latestP0) == -1 {
latestP0 = latest
}
return stable, latest, latestMM, latestP0, nil
}
3 changes: 3 additions & 0 deletions hack/update/update.go
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,9 @@ func init() {
if err := flag.Set("alsologtostderr", "true"); err != nil {
klog.Warningf("Unable to set flag value for alsologtostderr: %v", err)
}

// used in update_kubeadm_constants.go
flag.String("kubernetes-version", "latest", "kubernetes-version")
flag.Parse()
defer klog.Flush()

Expand Down
71 changes: 25 additions & 46 deletions pkg/minikube/bootstrapper/images/images.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ import (
"fmt"
"path"

"k8s.io/minikube/pkg/minikube/constants"

"github.com/blang/semver/v4"

"k8s.io/minikube/pkg/version"
Expand All @@ -31,18 +33,17 @@ func Pause(v semver.Version, mirror string) string {
// Note: changing this logic requires bumping the preload version
// Should match `PauseVersion` in:
// https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/constants/constants.go
pv := "3.5"
// https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/constants/constants_unix.go
if semver.MustParseRange("<1.22.0-alpha.3")(v) {
pv = "3.4.1"
}
if semver.MustParseRange("<1.21.0-alpha.3")(v) {
pv = "3.2"
}
if semver.MustParseRange("<1.18.0-alpha.0")(v) {
pv = "3.1"
pv := "3.5"

majorMinorVersion := fmt.Sprintf("v%d.%d", v.Major, v.Minor)
imageName := path.Join(kubernetesRepo(mirror), "pause")

if pVersion, ok := constants.KubeadmImages[majorMinorVersion][imageName]; ok {
pv = pVersion
}
return path.Join(kubernetesRepo(mirror), "pause:"+pv)

return fmt.Sprintf("%s:%s", imageName, pv)
}

// essentials returns images needed too bootstrap a Kubernetes
Expand Down Expand Up @@ -70,30 +71,20 @@ func coreDNS(v semver.Version, mirror string) string {
// Note: changing this logic requires bumping the preload version
// Should match `CoreDNSImageName` and `CoreDNSVersion` in
// https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/constants/constants.go

cv := "1.8.4"
in := "coredns/coredns"
if semver.MustParseRange("<1.21.0-alpha.1")(v) {
in = "coredns"
}
cv := "v1.8.4"
switch v.Minor {
case 21:
cv = "v1.8.0"
case 20, 19:
cv = "1.7.0"
case 18:
cv = "1.6.7"
case 17:
cv = "1.6.5"
case 16:
cv = "1.6.2"
case 15, 14:
cv = "1.3.1"
case 13:
cv = "1.2.6"
case 12:
cv = "1.2.2"

majorMinorVersion := fmt.Sprintf("v%d.%d", v.Major, v.Minor)
imageName := path.Join(kubernetesRepo(mirror), in)
if cVersion, ok := constants.KubeadmImages[majorMinorVersion][imageName]; ok {
cv = cVersion
}
return path.Join(kubernetesRepo(mirror), in+":"+cv)

return fmt.Sprintf("%s:%s", imageName, cv)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this seems like missing "kubernetesRepo(mirror)" don't we need that for --image-repositories and mirrors and china users?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fixed it.

}

// etcd returns the image used for etcd
Expand All @@ -102,26 +93,14 @@ func etcd(v semver.Version, mirror string) string {
// Should match `DefaultEtcdVersion` in:
// https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/constants/constants.go
ev := "3.5.0-0"
majorMinorVersion := fmt.Sprintf("v%d.%d", v.Major, v.Minor)
imageName := path.Join(kubernetesRepo(mirror), "etcd")

switch v.Minor {
case 19, 20, 21:
ev = "3.4.13-0"
case 17, 18:
ev = "3.4.3-0"
case 16:
ev = "3.3.15-0"
case 14, 15:
ev = "3.3.10"
case 12, 13:
ev = "3.2.24"
}

// An awkward special case for v1.19.0 - do not imitate unless necessary
if v.Equals(semver.MustParse("1.19.0")) {
ev = "3.4.9-1"
if eVersion, ok := constants.KubeadmImages[majorMinorVersion][imageName]; ok {
ev = eVersion
}

return path.Join(kubernetesRepo(mirror), "etcd:"+ev)
return fmt.Sprintf("%s:%s", imageName, ev)
}

// auxiliary returns images that are helpful for running minikube
Expand Down
22 changes: 11 additions & 11 deletions pkg/minikube/bootstrapper/images/kubeadm_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,17 +46,17 @@ func TestKubeadmImages(t *testing.T) {
"docker.io/kubernetesui/dashboard:v2.1.0",
"docker.io/kubernetesui/metrics-scraper:v1.0.4",
}},
{"v1.16.1", "mirror.k8s.io", false, []string{
"mirror.k8s.io/kube-proxy:v1.16.1",
"mirror.k8s.io/kube-scheduler:v1.16.1",
"mirror.k8s.io/kube-controller-manager:v1.16.1",
"mirror.k8s.io/kube-apiserver:v1.16.1",
"mirror.k8s.io/coredns:1.6.2",
"mirror.k8s.io/etcd:3.3.15-0",
"mirror.k8s.io/pause:3.1",
"mirror.k8s.io/k8s-minikube/storage-provisioner:" + version.GetStorageProvisionerVersion(),
"mirror.k8s.io/kubernetesui/dashboard:v2.1.0",
"mirror.k8s.io/kubernetesui/metrics-scraper:v1.0.4",
{"v1.16.1", "k8s.gcr.io", false, []string{
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why these are changed from "mirror.k8s.io" to "k8s.gcr.io"

this doen''t sound like doing what the PR title suggests it is doing

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This was due to constants having the k8s.gcr.io as fixed repo. I have fixed this. Thanks.

"k8s.gcr.io/kube-proxy:v1.16.1",
"k8s.gcr.io/kube-scheduler:v1.16.1",
"k8s.gcr.io/kube-controller-manager:v1.16.1",
"k8s.gcr.io/kube-apiserver:v1.16.1",
"k8s.gcr.io/coredns:1.6.2",
"k8s.gcr.io/etcd:3.3.15-0",
"k8s.gcr.io/pause:3.1",
"k8s.gcr.io/k8s-minikube/storage-provisioner:" + version.GetStorageProvisionerVersion(),
"k8s.gcr.io/kubernetesui/dashboard:v2.1.0",
"k8s.gcr.io/kubernetesui/metrics-scraper:v1.0.4",
}},
{"v1.15.0", "", false, []string{
"k8s.gcr.io/kube-proxy:v1.15.0",
Expand Down
Loading