Merge pull request GoogleCloudPlatform#35 from GoogleCloudPlatform/release

Release local builder

Author: skelterjohn

build/build.go

@@ -75,7 +75,7 @@ type Build struct {
 	Mu               sync.Mutex
 	Request          cb.Build
 	HasMultipleSteps bool
-	Tokensource      oauth2.TokenSource
+	TokenSource      oauth2.TokenSource
 	Log              *buildlog.BuildLog
 	Status           BuildStatus
 	imageDigests     map[string]string // docker image tag to digest (for built images)
@@ -103,6 +103,7 @@ type Build struct {
 	hostWorkspaceDir string
 	local            bool
 	push             bool
+	dryrun           bool
 
 	// For UpdateDockerAccessToken, previous GCR auth value to replace.
 	prevGCRAuth string
@@ -114,11 +115,11 @@ type kms interface {
 
 // New constructs a new Build.
 func New(r runner.Runner, rq cb.Build, ts oauth2.TokenSource,
-	bl *buildlog.BuildLog, hostWorkspaceDir string, local, push bool) *Build {
+	bl *buildlog.BuildLog, hostWorkspaceDir string, local, push, dryrun bool) *Build {
 	return &Build{
 		Runner:           r,
 		Request:          rq,
-		Tokensource:      ts,
+		TokenSource:      ts,
 		imageDigests:     map[string]string{},
 		stepDigests:      make([]string, len(rq.Steps)),
 		Log:              bl,
@@ -129,6 +130,7 @@ func New(r runner.Runner, rq cb.Build, ts oauth2.TokenSource,
 		hostWorkspaceDir: hostWorkspaceDir,
 		local:            local,
 		push:             push,
+		dryrun:           dryrun,
 	}
 }
 
@@ -641,12 +643,17 @@ func (b *Build) getKMSClient() (kms, error) {
 		return b.kms, nil
 	}
 
+	if b.dryrun {
+		b.kms = dryRunKMS{}
+		return b.kms, nil
+	}
+
 
 	// automatically gets (and refreshes) credentials from the metadata server
 	// when spoofing metadata works by IP. Until then, we'll just fetch the token
 	// and pass it to all HTTP requests.
 	svc, err := cloudkms.New(&http.Client{
-		Transport: &tokenTransport{b.Tokensource},
+		Transport: &tokenTransport{b.TokenSource},
 	})
 	if err != nil {
 		return nil, err
@@ -655,6 +662,14 @@ func (b *Build) getKMSClient() (kms, error) {
 	return b.kms, nil
 }
 
+// dryRunKMS always returns a base64-encoded placeholder string instead of real
+// decrypted value, for use in dryrun mode.
+type dryRunKMS struct{}
+
+func (dryRunKMS) Decrypt(string, string) (string, error) {
+	return base64.StdEncoding.EncodeToString([]byte("<REDACTED>")), nil
+}
+
 type realKMS struct {
 	svc *cloudkms.Service
 }

build/build_test.go

@@ -399,7 +399,7 @@ func TestFetchBuilder(t *testing.T) {
 	}}
 	for _, tc := range testCases {
 		r := newMockRunner(t, tc.name)
-		b := New(r, tc.buildRequest, mockTokenSource(), &buildlog.BuildLog{}, "", true, false)
+		b := New(r, tc.buildRequest, mockTokenSource(), &buildlog.BuildLog{}, "", true, false, false)
 		var gotErr error
 		var gotDigest string
 		wantDigest := ""
@@ -638,7 +638,7 @@ func TestRunBuildSteps(t *testing.T) {
 			}
 			return nil
 		}
-		b := New(r, tc.buildRequest, mockTokenSource(), &buildlog.BuildLog{}, "", true, false)
+		b := New(r, tc.buildRequest, mockTokenSource(), &buildlog.BuildLog{}, "", true, false, false)
 		gotErr := b.runBuildSteps()
 		if !reflect.DeepEqual(gotErr, tc.wantErr) {
 			t.Errorf("%s: Wanted error %q, but got %q", tc.name, tc.wantErr, gotErr)
@@ -872,7 +872,7 @@ func TestBuildStepOrder(t *testing.T) {
 			}
 			return nil
 		}
-		b := New(r, tc.buildRequest, mockTokenSource(), &buildlog.BuildLog{}, "", true, false)
+		b := New(r, tc.buildRequest, mockTokenSource(), &buildlog.BuildLog{}, "", true, false, false)
 		errorFromFunction := make(chan error)
 		go func() {
 			errorFromFunction <- b.runBuildSteps()
@@ -922,7 +922,7 @@ func TestPushImages(t *testing.T) {
 	}}
 	for _, tc := range testCases {
 		r := newMockRunner(t, tc.name)
-		b := New(r, tc.buildRequest, mockTokenSource(), &buildlog.BuildLog{}, "", true, true)
+		b := New(r, tc.buildRequest, mockTokenSource(), &buildlog.BuildLog{}, "", true, true, false)
 		r.remotePushesFail = tc.remotePushesFail
 		gotErr := b.pushImages()
 		if !reflect.DeepEqual(gotErr, tc.wantErr) {
@@ -1013,7 +1013,7 @@ func TestBuildStepConcurrency(t *testing.T) {
 	}
 
 	// Run the build.
-	b := New(r, req, mockTokenSource(), &buildlog.BuildLog{}, "", true, false)
+	b := New(r, req, mockTokenSource(), &buildlog.BuildLog{}, "", true, false, false)
 	ret := make(chan error)
 	go func() {
 		ret <- b.runBuildSteps()
@@ -1052,7 +1052,7 @@ type fakeRunner struct {
 func (f *fakeRunner) Run(args []string, _ io.Reader, _, _ io.Writer, _ string) error {
 	// The "+1" is for the name of the container which is appended to the
 	// dockerRunArgs base command.
-	b := New(nil, cb.Build{}, nil, &buildlog.BuildLog{}, "", true, false)
+	b := New(nil, cb.Build{}, nil, &buildlog.BuildLog{}, "", true, false, false)
 	argCount := len(b.dockerRunArgs("", 0)) + 1
 	switch {
 	case !startsWith(args, "docker", "run"):
@@ -1103,7 +1103,7 @@ func dockerRunString(idx int) string {
 }
 
 func dockerRunInStepDir(idx int, stepDir string) string {
-	b := New(nil, cb.Build{}, nil, &buildlog.BuildLog{}, "", true, false)
+	b := New(nil, cb.Build{}, nil, &buildlog.BuildLog{}, "", true, false, false)
 	dockerRunArg := b.dockerRunArgs(stepDir, idx)
 	return strings.Join(dockerRunArg, " ")
 }
@@ -1146,7 +1146,7 @@ func TestErrorCollection(t *testing.T) {
 		"got an http status: 300: it's a mystery to me",
 		"got another http status: 300: it's a double mystery",
 	}
-	b := New(nil, cb.Build{}, nil, &buildlog.BuildLog{}, "", true, false)
+	b := New(nil, cb.Build{}, nil, &buildlog.BuildLog{}, "", true, false, false)
 	for _, o := range outputs {
 		b.detectPushFailure(o)
 	}
@@ -1214,7 +1214,7 @@ func TestEntrypoint(t *testing.T) {
 			stepArgs <- strings.Join(args, " ")
 			return nil
 		}
-		b := New(r, tc.buildRequest, mockTokenSource(), &buildlog.BuildLog{}, "", true, false)
+		b := New(r, tc.buildRequest, mockTokenSource(), &buildlog.BuildLog{}, "", true, false, false)
 		errorFromFunction := make(chan error)
 		go func() {
 			errorFromFunction <- b.runBuildSteps()
@@ -1305,7 +1305,7 @@ func TestSecrets(t *testing.T) {
 			gotCommand = strings.Join(args, " ")
 			return nil
 		}
-		b := New(r, buildRequest, mockTokenSource(), &buildlog.BuildLog{}, "", true, false)
+		b := New(r, buildRequest, mockTokenSource(), &buildlog.BuildLog{}, "", true, false, false)
 		b.kms = fakeKMS{
 			plaintext: c.plaintext,
 			err:       c.kmsErr,
@@ -1544,7 +1544,7 @@ func TestStart(t *testing.T) {
 			r.localImages["gcr.io/build"] = true
 			return nil
 		}
-		b := New(r, tc.buildRequest, mockTokenSource(), &buildlog.BuildLog{}, "", true, tc.push)
+		b := New(r, tc.buildRequest, mockTokenSource(), &buildlog.BuildLog{}, "", true, tc.push, false)
 		b.Start()
 		<-b.Done
 
@@ -1566,7 +1566,7 @@ func TestUpdateDockerAccessToken(t *testing.T) {
 	t.Parallel()
 	r := newMockRunner(t, "TestUpdateDockerAccessToken")
 	r.dockerRunHandler = func(args []string, _, _ io.Writer) error { return nil }
-	b := New(r, cb.Build{}, nil, nil, "", false, false)
+	b := New(r, cb.Build{}, nil, nil, "", false, false, false)
 
 	// If UpdateDockerAccessToken is called before SetDockerAccessToken, we
 	// should get an error.

localbuilder_main.go

@@ -26,6 +26,7 @@ import (
 	"time"
 
 	computeMetadata "cloud.google.com/go/compute/metadata"
+	"golang.org/x/oauth2"
 	"github.com/google/uuid"
 
 	"github.com/GoogleCloudPlatform/container-builder-local/build"
@@ -49,7 +50,7 @@ const (
 var (
 	configFile    = flag.String("config", "cloudbuild.yaml", "cloud build config file path")
 	substitutions = flag.String("substitutions", "", `substitutions key=value pairs separated by comma; for example _FOO=bar,_BAZ=baz`)
-	dryRun        = flag.Bool("dryrun", true, "If true, nothing will be run")
+	dryRun        = flag.Bool("dryrun", true, "If true, the config file is linted and the commands printed, but they are not run")
 	push          = flag.Bool("push", false, "If true, the images will be pushed")
 	noSource      = flag.Bool("no-source", false, "Specify that no source should be used for this build.")
 	help          = flag.Bool("help", false, "If true, print the help message")
@@ -167,6 +168,11 @@ func run(source string) error {
 		return fmt.Errorf("Error applying substitutions: %v", err)
 	}
 
+	// Validate the build after substitutions.
+	if err := validate.CheckBuildAfterSubstitutions(buildConfig); err != nil {
+		return fmt.Errorf("Error validating build after substitutions: %v", err)
+	}
+
 	// Create a volume, a helper container to copy the source, and defer cleaning.
 	volumeName := fmt.Sprintf("%s%s", volumeNamePrefix, uuid.New())
 	if !*dryRun {
@@ -188,10 +194,22 @@ func run(source string) error {
 		defer vol.Close()
 	}
 
-	b := build.New(r, *buildConfig, nil, &buildlog.BuildLog{}, volumeName, true, *push)
+	b := build.New(r, *buildConfig, nil /* TokenSource */, &buildlog.BuildLog{}, volumeName, true, *push, *dryRun)
 
 	// Do not run the spoofed metadata server on a dryrun.
 	if !*dryRun {
+		// Set initial Docker credentials.
+		tok, err := gcloud.AccessToken(r)
+		if err != nil {
+			return fmt.Errorf("Error getting access token to set docker credentials: %v", err)
+		}
+		if err := b.SetDockerAccessToken(tok.AccessToken); err != nil {
+			return fmt.Errorf("Error setting docker credentials: %v", err)
+		}
+		b.TokenSource = oauth2.StaticTokenSource(&oauth2.Token{
+			AccessToken: tok.AccessToken,
+		})
+
 		// On GCE, do not create a spoofed metadata server, use the existing one.
 		// The cloudbuild network is still needed, with a private subnet.
 		if computeMetadata.OnGCE() {
@@ -213,15 +231,6 @@ func run(source string) error {
 			go supplyTokenToMetadata(metadataUpdater, r, stopchan)
 		}
 
-		// Set initial Docker credentials.
-		tok, err := gcloud.AccessToken(r)
-		if err != nil {
-			return fmt.Errorf("Error getting access token to set docker credentials: %v", err)
-		}
-		if err := b.SetDockerAccessToken(tok.AccessToken); err != nil {
-			return fmt.Errorf("Error setting docker credentials: %v", err)
-		}
-
 		// Write docker credentials for GCR. This writes the initial
 		// ~/.docker/config.json, which is made available to build steps, and keeps
 		// a fresh token available. Note that the user could `gcloud auth` to
@@ -246,6 +255,10 @@ func run(source string) error {
 					if err := b.UpdateDockerAccessToken(tok.AccessToken); err != nil {
 						log.Printf("Error updating docker credentials: %v", err)
 					}
+
+					b.TokenSource = oauth2.StaticTokenSource(&oauth2.Token{
+						AccessToken: tok.AccessToken,
+					})
 				case <-stopchan:
 					return
 				}

subst/subst.go

@@ -23,9 +23,10 @@ import (
 )
 
 var (
-	unbracedKeyRE   = `([A-Z_][A-Z0-9_]*)`
-	bracedKeyRE     = fmt.Sprintf(`{%s}`, unbracedKeyRE)
-	validSubstKeyRE = regexp.MustCompile(`^\$(?:` + bracedKeyRE + `|` + unbracedKeyRE + `)`)
+	unbracedKeyRE     = `([A-Z_][A-Z0-9_]*)`
+	bracedKeyRE       = fmt.Sprintf(`{%s}`, unbracedKeyRE)
+	validSubstKeyRE   = regexp.MustCompile(`^\$(?:` + bracedKeyRE + `|` + unbracedKeyRE + `)`)
+	maxShortShaLength = 7
 )
 
 // SubstituteBuildFields does an in-place string substitution of build parameters.
@@ -38,6 +39,7 @@ func SubstituteBuildFields(b *cb.Build) error {
 	branchName := ""
 	tagName := ""
 	commitSHA := ""
+	shortSHA := ""
 
 	if s := b.GetSource(); s != nil {
 		if rs := s.GetRepoSource(); rs != nil {
@@ -49,6 +51,11 @@ func SubstituteBuildFields(b *cb.Build) error {
 	if sp := b.GetSourceProvenance(); sp != nil {
 		if rrs := sp.GetResolvedRepoSource(); rrs != nil {
 			commitSHA = rrs.GetCommitSha()
+			shortSHA = commitSHA
+			// Length of commit SHA can be less than maxShortShaLength.
+			if len(shortSHA) > maxShortShaLength {
+				shortSHA = shortSHA[0:maxShortShaLength]
+			}
 		}
 	}
 
@@ -61,6 +68,7 @@ func SubstituteBuildFields(b *cb.Build) error {
 		"TAG_NAME":    tagName,
 		"REVISION_ID": commitSHA,
 		"COMMIT_SHA":  commitSHA,
+		"SHORT_SHA":   shortSHA,
 	}
 
 	// Add user-defined substitutions, overriding built-in substitutions.