Skip to content

Add initial "multi-stage" support in bashbrew #5929

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 12 commits into from
Jun 13, 2019
Merged

Add initial "multi-stage" support in bashbrew #5929

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
67098be
Add initial "multi-stage" support in bashbrew
tianon May 17, 2019
f6d9672
Update "naughty-from.sh" to support multi-stage
tianon May 18, 2019
81e8f4f
Unexpose "DockerfileMetadata" and add new exposed "DockerFroms" (sinc…
tianon May 20, 2019
a898c73
Remove "DockerFrom" function entirely to cause scripts to fail and fo…
tianon May 20, 2019
e1a94ef
Update "diff-pr.sh" to use the new DockerFroms function
tianon May 20, 2019
0ef77ba
Add new "ArchLastStageFrom" function for getting the effective "base"…
tianon May 20, 2019
828b8ba
Drop "ADD --from=" (which isn't supported by Docker)
tianon May 20, 2019
1f2cab5
Improve Dockerfile fetching error reporting and "bashbrew children" /…
tianon Jun 6, 2019
8969734
Apply gofmt
tianon Jun 7, 2019
9cd890f
Remove "apply-constraints" from Travis to make the test more useful (…
tianon Jun 7, 2019
22c68d5
Switch most "git" operations over to https://github.com/src-d/go-git …
tianon Jun 11, 2019
bcc52e4
Add new script to also verify "Constraints:" values are correct
tianon Jun 12, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
2 changes: 1 addition & 1 deletion bashbrew/go/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ require (
github.com/codegangsta/cli v1.20.0
github.com/docker-library/go-dockerlibrary v0.0.0-20190605224921-7d4d3222cdd1
golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c // indirect
golang.org/x/sys v0.0.0-20190322080309-f49334f85ddc // indirect
gopkg.in/src-d/go-git.v4 v4.11.0
pault.ag/go/debian v0.0.0-20190109175134-a131cb0ae041
pault.ag/go/topsort v0.0.0-20160530003732-f98d2ad46e1a
)
60 changes: 57 additions & 3 deletions bashbrew/go/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,70 @@
github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs=
github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs=
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
github.com/codegangsta/cli v1.20.0 h1:iX1FXEgwzd5+XN6wk5cVHOGQj6Q3Dcp20lUeS4lHNTw=
github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA=
github.com/docker-library/go-dockerlibrary v0.0.0-20190129000321-7e50189a05d4 h1:Jl830zF5XyeMipP9Ag2J64TAeheWDvOsqLuaDHn4pb8=
github.com/docker-library/go-dockerlibrary v0.0.0-20190129000321-7e50189a05d4/go.mod h1:ijRhN3WM71dD8TfohKoUdX46BT2uz/Ek5O+5PINI880=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/docker-library/go-dockerlibrary v0.0.0-20190605224921-7d4d3222cdd1 h1:eTBMV614FnFNrm4PKUB+8UtOyrcF1uINuBak//m6YI0=
github.com/docker-library/go-dockerlibrary v0.0.0-20190605224921-7d4d3222cdd1/go.mod h1:ijRhN3WM71dD8TfohKoUdX46BT2uz/Ek5O+5PINI880=
github.com/emirpasic/gods v1.9.0 h1:rUF4PuzEjMChMiNsVjdI+SyLu7rEqpQ5reNFnhC7oFo=
github.com/emirpasic/gods v1.9.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ=
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
github.com/gliderlabs/ssh v0.1.1 h1:j3L6gSLQalDETeEg/Jg0mGY0/y/N6zI2xX1978P0Uqw=
github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e h1:RgQk53JHp/Cjunrr1WlsXSZpqXn+uREuHvUVcK82CV8=
github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
github.com/kjk/lzma v0.0.0-20161016003348-3fd93898850d/go.mod h1:phT/jsRPBAEqjAibu1BurrabCBNTYiVI+zbmyCZJY6Q=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/mitchellh/go-homedir v1.0.0 h1:vKb8ShqSby24Yrqr/yDYkuFz8d0WUjys40rvnGC8aR0=
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/pelletier/go-buffruneio v0.2.0 h1:U4t4R6YkofJ5xHm3dJzuRpPZ0mr5MMCoAWooScCR7aA=
github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo=
github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4=
github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI=
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/xanzy/ssh-agent v0.2.0 h1:Adglfbi5p9Z0BmK2oKU9nTG+zKfniSfnaMYB+ULd+Ro=
github.com/xanzy/ssh-agent v0.2.0/go.mod h1:0NyE30eGUDliuLEHJgYte/zncp2zdTStcOnWhgSqHD8=
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c h1:Vj5n4GlwjmQteupaxJ9+0FNOmBrHfq7vN4btdGoDZgI=
golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190322080309-f49334f85ddc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/src-d/go-billy.v4 v4.2.1 h1:omN5CrMrMcQ+4I8bJ0wEhOBPanIRWzFC953IiXKdYzo=
gopkg.in/src-d/go-billy.v4 v4.2.1/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk=
gopkg.in/src-d/go-git-fixtures.v3 v3.1.1 h1:XWW/s5W18RaJpmo1l0IYGqXKuJITWRFuA45iOf1dKJs=
gopkg.in/src-d/go-git-fixtures.v3 v3.1.1/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g=
gopkg.in/src-d/go-git.v4 v4.11.0 h1:cJwWgJ0DXifrNrXM6RGN1Y2yR60Rr1zQ9Q5DX5S9qgU=
gopkg.in/src-d/go-git.v4 v4.11.0/go.mod h1:Vtut8izDyrM8BUVQnzJ+YvmNcem2J89EmfZYCkLokZk=
gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
pault.ag/go/debian v0.0.0-20190109175134-a131cb0ae041 h1:LmTwXQVDWXMigTB88hFtd6mc3l5eLAlgdtfPI0F69bQ=
pault.ag/go/debian v0.0.0-20190109175134-a131cb0ae041/go.mod h1:e7Gva9AMoKtUKYJ1G9kIesbh+4VS2JnAOS8VWafyTCk=
pault.ag/go/topsort v0.0.0-20160530003732-f98d2ad46e1a h1:WwS7vlB5H2AtwKj1jsGwp2ZLud1x6WXRXh2fXsRqrcA=
Expand Down
38 changes: 21 additions & 17 deletions bashbrew/go/src/bashbrew/cmd-build.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,27 +44,29 @@ func cmdBuild(c *cli.Context) error {
continue
}

from, err := r.DockerFrom(entry)
froms, err := r.DockerFroms(entry)
if err != nil {
return cli.NewMultiError(fmt.Errorf(`failed fetching/scraping FROM for %q (tags %q)`, r.RepoName, entry.TagsString()), err)
}

if from != "scratch" && pull != "never" {
doPull := false
switch pull {
case "always":
doPull = true
case "missing":
_, err := dockerInspect("{{.Id}}", from)
doPull = (err != nil)
default:
return fmt.Errorf(`unexpected value for --pull: %s`, pull)
}
if doPull {
// TODO detect if "from" is something we've built (ie, "python:3-onbuild" is "FROM python:3" but we don't want to pull "python:3" if we "bashbrew build python")
fmt.Printf("Pulling %s (%s)\n", from, r.EntryIdentifier(entry))
if !dryRun {
dockerPull(from)
for _, from := range froms {
if from != "scratch" && pull != "never" {
doPull := false
switch pull {
case "always":
doPull = true
case "missing":
_, err := dockerInspect("{{.Id}}", from)
doPull = (err != nil)
default:
return fmt.Errorf(`unexpected value for --pull: %s`, pull)
}
if doPull {
// TODO detect if "from" is something we've built (ie, "python:3-onbuild" is "FROM python:3" but we don't want to pull "python:3" if we "bashbrew build python")
fmt.Printf("Pulling %s (%s)\n", from, r.EntryIdentifier(entry))
if !dryRun {
dockerPull(from)
}
}
}
}
Expand All @@ -90,6 +92,8 @@ func cmdBuild(c *cli.Context) error {
}
defer archive.Close()

// TODO use "meta.StageNames" to do "docker build --target" so we can tag intermediate stages too for cache (streaming "git archive" directly to "docker build" makes that a little hard to accomplish without re-streaming)
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

IMO this catch is worth pointing out explicitly here too -- we don't ever have a "local" copy of the context, and instead stream git archive directly to docker build, so even if docker build --target=N were possible to do for a numbered stage, we'd still have an optimization problem implementing it (and I'd love to at least tag named stages that way, but the git archive | docker build optimization makes that a little harder to swallow).

The practical implication of this is that build cache for these untagged stages will still be considered "ripe" by our tooling and thus we will end up spending extra time rebuilding things after each cleaning of "ripe" images on our build servers.

For this reason, I think that for the official images, we should plan to still discourage multi-stage image use unless the gains are clear and/or the final artifacts are reproducible (and thus can avoid unnecessary image updates), and even then encourage use of explicitly named stages so that we can hopefully tag them in the future to help avoid this issue.


err = dockerBuild(cacheTag, entry.ArchFile(arch), archive)
if err != nil {
return cli.NewMultiError(fmt.Errorf(`failed building %q (tags %q)`, r.RepoName, entry.TagsString()), err)
Expand Down
19 changes: 14 additions & 5 deletions bashbrew/go/src/bashbrew/cmd-deps.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,12 +67,21 @@ func cmdFamily(parents bool, c *cli.Context) error {
continue
}

from, err := r.DockerFrom(entry)
if err != nil {
return cli.NewMultiError(fmt.Errorf(`failed fetching/scraping FROM for %q (tags %q)`, r.RepoName, entry.TagsString()), err)
entryArches := []string{arch}
if !applyConstraints {
entryArches = entry.Architectures
}
for _, tag := range r.Tags("", false, entry) {
network.AddEdge(from, tag)

for _, entryArch := range entryArches {
froms, err := r.ArchDockerFroms(entryArch, entry)
if err != nil {
return cli.NewMultiError(fmt.Errorf(`failed fetching/scraping FROM for %q (tags %q, arch %q)`, r.RepoName, entry.TagsString(), entryArch), err)
}
for _, from := range froms {
for _, tag := range r.Tags("", false, entry) {
network.AddEdge(from, tag)
}
}
}
}
}
Expand Down
27 changes: 23 additions & 4 deletions bashbrew/go/src/bashbrew/cmd-from.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package main

import (
"fmt"
"strings"

"github.com/codegangsta/cli"
)
Expand All @@ -27,13 +28,31 @@ func cmdFrom(c *cli.Context) error {
continue
}

from, err := r.DockerFrom(entry)
if err != nil {
return cli.NewMultiError(fmt.Errorf(`failed fetching/scraping FROM for %q (tags %q)`, r.RepoName, entry.TagsString()), err)
entryArches := []string{arch}
if !applyConstraints {
entryArches = entry.Architectures
}

froms := []string{}
for _, entryArch := range entryArches {
archFroms, err := r.ArchDockerFroms(entryArch, entry)
if err != nil {
return cli.NewMultiError(fmt.Errorf(`failed fetching/scraping FROM for %q (tags %q, arch %q)`, r.RepoName, entry.TagsString(), entryArch), err)
}
ArchFroms:
for _, archFrom := range archFroms {
for _, from := range froms {
if from == archFrom {
continue ArchFroms
}
}
froms = append(froms, archFrom)
}
}

fromsString := strings.Join(froms, " ")
for _, tag := range r.Tags(namespace, uniq, entry) {
fmt.Printf("%s: %s\n", tag, from)
fmt.Printf("%s: %s\n", tag, fromsString)
}
}
}
Expand Down
Loading