Add hack/update package to automate PR generation

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/blang/semver v3.5.0+incompatible
 	github.com/c4milo/gotoolkit v0.0.0-20170318115440-bcc06269efa9 // indirect
 	github.com/cenkalti/backoff v2.2.1+incompatible
+	github.com/cenkalti/backoff/v4 v4.1.0
 	github.com/cheggaaa/pb/v3 v3.0.1
 	github.com/cloudevents/sdk-go/v2 v2.1.0
 	github.com/cloudfoundry-attic/jibber_jabber v0.0.0-20151120183258-bcc4c8345a21
@@ -74,6 +75,7 @@ require (
 	golang.org/x/build v0.0.0-20190927031335-2835ba2e683f
 	golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37
 	golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6
+	golang.org/x/mod v0.3.0
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
 	golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a
 	golang.org/x/sys v0.0.0-20200523222454-059865788121
@@ -85,6 +87,7 @@ require (
 	k8s.io/api v0.17.4
 	k8s.io/apimachinery v0.17.4
 	k8s.io/client-go v0.17.4
+	k8s.io/klog/v2 v2.3.0
 	k8s.io/kubectl v0.0.0
 	k8s.io/kubernetes v1.18.5
 	sigs.k8s.io/sig-storage-lib-external-provisioner v4.0.0+incompatible // indirect

go.sum

@@ -196,6 +196,8 @@ github.com/cavaliercoder/go-cpio v0.0.0-20180626203310-925f9528c45e/go.mod h1:oD
 github.com/cenkalti/backoff v2.1.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
 github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
 github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
+github.com/cenkalti/backoff/v4 v4.1.0 h1:c8LkOFQTzuO0WBM/ae5HdGQuZPfPxp7lqBRwQRm4fSc=
+github.com/cenkalti/backoff/v4 v4.1.0/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
 github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/prettybench v0.0.0-20150116022406-03b8cfe5406c/go.mod h1:Xe6ZsFhtM8HrDku0pxJ3/Lr51rwykrzgFwpmTzleatY=
@@ -352,6 +354,8 @@ github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTD
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
+github.com/go-logr/logr v0.2.0 h1:QvGt2nLcHH0WK9orKa+ppBPAxREcH364nPUedEpK0TY=
+github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
 github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
 github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI=
 github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
@@ -1638,6 +1642,8 @@ k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUc
 k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
 k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
 k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
+k8s.io/klog/v2 v2.3.0 h1:WmkrnW7fdrm0/DMClc+HIxtftvxVIPAhlVwMQo5yLco=
+k8s.io/klog/v2 v2.3.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
 k8s.io/kube-aggregator v0.17.3/go.mod h1:1dMwMFQbmH76RKF0614L7dNenMl3dwnUJuOOyZ3GMXA=
 k8s.io/kube-controller-manager v0.17.3/go.mod h1:22B/TsgVviuCVuNwUrqgyTi5D4AYjMFaK9c8h1oonkY=
 k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a h1:UcxjrRMyNx/i/y8G7kPvLyy7rfbeuf1PYyBf973pgyU=

hack/update/filesystem.go

@@ -0,0 +1,53 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package update
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+)
+
+// fsUpdate updates local filesystem repo files according to the given schema and data,
+// returns if the update actually changed anything, and any error occurred
+func fsUpdate(fsRoot string, schema map[string]Item, data interface{}) (changed bool, err error) {
+	for path, item := range schema {
+		path = filepath.Join(fsRoot, path)
+		blob, err := ioutil.ReadFile(path)
+		if err != nil {
+			return false, err
+		}
+		info, err := os.Stat(path)
+		if err != nil {
+			return false, err
+		}
+		mode := info.Mode()
+
+		item.Content = blob
+		chg, err := item.apply(data)
+		if err != nil {
+			return false, err
+		}
+		if chg {
+			changed = true
+		}
+		if err := ioutil.WriteFile(path, item.Content, mode); err != nil {
+			return false, err
+		}
+	}
+	return changed, nil
+}

hack/update/github.go

@@ -0,0 +1,261 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package update
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"golang.org/x/mod/semver"
+	"golang.org/x/oauth2"
+
+	"github.com/google/go-github/v32/github"
+	"k8s.io/klog/v2"
+)
+
+const (
+	// ghListPerPage uses max value (100) for PerPage to avoid hitting the rate limits
+	// (ref: https://godoc.org/github.com/google/go-github/github#hdr-Rate_Limiting)
+	ghListPerPage = 100
+
+	// ghSearchLimit limits the number of searched items to be <= N * ListPerPage
+	ghSearchLimit = 100
+)
+
+var (
+	// GitHub repo data
+	ghToken = os.Getenv("GITHUB_TOKEN")
+	ghOwner = "kubernetes"
+	ghRepo  = "minikube"
+	ghBase  = "master" // could be "main" in the future?
+)
+
+// ghCreatePR returns PR created in the GitHub owner/repo, applying the changes to the base head
+// commit fork, as defined by the schema and data, and also returns any error occurred
+// PR branch will be named by the branch, sufixed by '_' and first 7 characters of fork commit SHA
+// PR itself will be named by the title and will reference the issue
+func ghCreatePR(ctx context.Context, owner, repo, base, branch, title string, issue int, token string, schema map[string]Item, data interface{}) (*github.PullRequest, error) {
+	ghc := ghClient(ctx, token)
+
+	// get base branch
+	baseBranch, _, err := ghc.Repositories.GetBranch(ctx, owner, repo, base)
+	if err != nil {
+		return nil, fmt.Errorf("error getting base branch: %w", err)
+	}
+
+	// get base commit
+	baseCommit, _, err := ghc.Repositories.GetCommit(ctx, owner, repo, *baseBranch.Commit.SHA)
+	if err != nil {
+		return nil, fmt.Errorf("error getting base commit: %w", err)
+	}
+
+	// get base tree
+	baseTree, _, err := ghc.Git.GetTree(ctx, owner, repo, baseCommit.GetSHA(), true)
+	if err != nil {
+		return nil, fmt.Errorf("error getting base tree: %w", err)
+	}
+
+	// update files
+	changes, err := ghUpdate(ctx, owner, repo, baseTree, token, schema, data)
+	if err != nil {
+		return nil, fmt.Errorf("error updating files: %w", err)
+	}
+	if changes == nil {
+		return nil, nil
+	}
+
+	// create fork
+	fork, resp, err := ghc.Repositories.CreateFork(ctx, owner, repo, nil)
+	// https://pkg.go.dev/github.com/google/go-github/v32@v32.1.0/github#RepositoriesService.CreateFork
+	// This method might return an *AcceptedError and a status code of 202. This is because this is
+	// the status that GitHub returns to signify that it is now computing creating the fork in a
+	// background task. In this event, the Repository value will be returned, which includes the
+	// details about the pending fork. A follow up request, after a delay of a second or so, should
+	// result in a successful request.
+	if resp.StatusCode == 202 { // *AcceptedError
+		time.Sleep(time.Second * 5)
+	} else if err != nil {
+		return nil, fmt.Errorf("error creating fork: %w", err)
+	}
+
+	// create fork tree from base and changed files
+	forkTree, _, err := ghc.Git.CreateTree(ctx, *fork.Owner.Login, *fork.Name, *baseTree.SHA, changes)
+	if err != nil {
+		return nil, fmt.Errorf("error creating fork tree: %w", err)
+	}
+
+	// create fork commit
+	forkCommit, _, err := ghc.Git.CreateCommit(ctx, *fork.Owner.Login, *fork.Name, &github.Commit{
+		Message: github.String(title),
+		Tree:    &github.Tree{SHA: forkTree.SHA},
+		Parents: []*github.Commit{{SHA: baseCommit.SHA}},
+	})
+	if err != nil {
+		return nil, fmt.Errorf("error creating fork commit: %w", err)
+	}
+	klog.Infof("PR commit '%s' created: %s", forkCommit.GetSHA(), forkCommit.GetHTMLURL())
+
+	// create PR branch
+	prBranch := branch + forkCommit.GetSHA()[:7]
+	prRef, _, err := ghc.Git.CreateRef(ctx, *fork.Owner.Login, *fork.Name, &github.Reference{
+		Ref: github.String("refs/heads/" + prBranch),
+		Object: &github.GitObject{
+			Type: github.String("commit"),
+			SHA:  forkCommit.SHA,
+		},
+	})
+	if err != nil {
+		return nil, fmt.Errorf("error creating PR branch: %w", err)
+	}
+	klog.Infof("PR branch '%s' created: %s", prBranch, prRef.GetURL())
+
+	// create PR
+	plan, err := GetPlan(schema, data)
+	if err != nil {
+		klog.Fatalf("Error parsing schema: %v\n%s", err, plan)
+	}
+	modifiable := true
+	pr, _, err := ghc.PullRequests.Create(ctx, owner, repo, &github.NewPullRequest{
+		Title:               github.String(title),
+		Head:                github.String(*fork.Owner.Login + ":" + prBranch),
+		Base:                github.String(base),
+		Body:                github.String(fmt.Sprintf("fixes #%d\n\nAutomatically created PR to update repo according to the Plan:\n\n```\n%s\n```", issue, plan)),
+		MaintainerCanModify: &modifiable,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("error creating pull request: %w", err)
+	}
+	return pr, nil
+}
+
+// ghUpdate updates remote GitHub owner/repo tree according to the given token, schema and data,
+// returns resulting changes, and any error occurred
+func ghUpdate(ctx context.Context, owner, repo string, tree *github.Tree, token string, schema map[string]Item, data interface{}) (changes []*github.TreeEntry, err error) {
+	ghc := ghClient(ctx, token)
+
+	// load each schema item content and update it creating new GitHub TreeEntries
+	cnt := len(schema) // expected number of files to change
+	for _, org := range tree.Entries {
+		if *org.Type == "blob" {
+			if item, match := schema[*org.Path]; match {
+				blob, _, err := ghc.Git.GetBlobRaw(ctx, owner, repo, *org.SHA)
+				if err != nil {
+					return nil, fmt.Errorf("error getting file: %w", err)
+				}
+				item.Content = blob
+				changed, err := item.apply(data)
+				if err != nil {
+					return nil, fmt.Errorf("error updating file: %w", err)
+				}
+				if changed {
+					// add github.TreeEntry that will replace original path content with updated one
+					changes = append(changes, &github.TreeEntry{
+						Path:    org.Path,
+						Mode:    org.Mode,
+						Type:    org.Type,
+						Content: github.String(string(item.Content)),
+					})
+				}
+				if cnt--; cnt == 0 {
+					break
+				}
+			}
+		}
+	}
+	if cnt != 0 {
+		return nil, fmt.Errorf("error finding all the files (%d missing) - check the Plan: %w", cnt, err)
+	}
+	return changes, nil
+}
+
+// ghFindPR returns URL of the PR if found in the given GitHub ower/repo base and any error occurred
+func ghFindPR(ctx context.Context, title, owner, repo, base, token string) (url string, err error) {
+	ghc := ghClient(ctx, token)
+
+	// walk through the paginated list of all pull requests, from latest to older releases
+	opts := &github.PullRequestListOptions{State: "all", Base: base, ListOptions: github.ListOptions{PerPage: ghListPerPage}}
+	for (opts.Page+1)*ghListPerPage <= ghSearchLimit {
+		prs, resp, err := ghc.PullRequests.List(ctx, owner, repo, opts)
+		if err != nil {
+			return "", err
+		}
+		for _, pr := range prs {
+			if pr.GetTitle() == title {
+				return pr.GetHTMLURL(), nil
+			}
+		}
+		if resp.NextPage == 0 {
+			break
+		}
+		opts.Page = resp.NextPage
+	}
+	return "", nil
+}
+
+// ghClient returns GitHub Client with a given context and optional token for authenticated requests
+func ghClient(ctx context.Context, token string) *github.Client {
+	if token == "" {
+		return github.NewClient(nil)
+	}
+	ts := oauth2.StaticTokenSource(
+		&oauth2.Token{AccessToken: token},
+	)
+	tc := oauth2.NewClient(ctx, ts)
+	return github.NewClient(tc)
+}
+
+// GHVersions returns greatest current stable release and greatest latest rc or beta pre-release
+// from GitHub owner/repo repository, and any error;
+// if latest pre-release version is lower than current stable release, then it
+// will return current stable release for both
+func GHVersions(ctx context.Context, owner, repo string) (stable, latest string, err error) {
+	ghc := ghClient(ctx, ghToken)
+
+	// walk through the paginated list of all owner/repo releases, from newest to oldest
+	opts := &github.ListOptions{PerPage: ghListPerPage}
+	for {
+		rls, resp, err := ghc.Repositories.ListReleases(ctx, owner, repo, opts)
+		if err != nil {
+			return "", "", err
+		}
+		for _, rl := range rls {
+			ver := rl.GetName()
+			if !semver.IsValid(ver) {
+				continue
+			}
+			// check if ver version is release (ie, 'v1.19.2') or pre-release (ie, 'v1.19.3-rc.0' or 'v1.19.0-beta.2')
+			prerls := semver.Prerelease(ver)
+			if prerls == "" {
+				stable = semver.Max(ver, stable)
+			} else if strings.HasPrefix(prerls, "-rc") || strings.HasPrefix(prerls, "-beta") {
+				latest = semver.Max(ver, latest)
+			}
+			// make sure that latest >= stable
+			if semver.Compare(latest, stable) == -1 {
+				latest = stable
+			}
+		}
+		if resp.NextPage == 0 {
+			break
+		}
+		opts.Page = resp.NextPage
+	}
+	return stable, latest, nil
+}