Merge pull request #106 from HeavyWombat/add/bundle

Add source bundle to `upload` command

Author: openshift-merge-robot

.github/workflows/e2e.yaml

@@ -16,11 +16,12 @@ jobs:
           - v1.21.2
       max-parallel: 2
     runs-on: ubuntu-latest
+
     steps:
       - name: Setup Go
         uses: actions/setup-go@v2
         with:
-          go-version: ^1.17
+          go-version: 1.17.x
 
       - name: Check out code into the Go module directory
         uses: actions/checkout@v2
@@ -41,6 +42,7 @@ jobs:
           version: v0.11.1
           node_image: kindest/node:${{ matrix.kubernetes }}
           cluster_name: kind
+          config: test/kind/config.yaml
           wait: 120s
 
       - name: Verify KinD cluster
@@ -49,11 +51,15 @@ jobs:
       - name: Install Registry
         run: make install-registry
 
+      - name: Add cluster hosted registry name to /etc/hosts
+        run: echo "127.0.0.1 registry.registry.svc.cluster.local" | sudo tee -a /etc/hosts
+
       - name: Installing Shipwright Build Controller
         run: make install-shipwright
 
       - name: Build Application (shp)
         run: make build
 
       - name: End-to-End Tests
-        run: make test-e2e
+        run: |
+          make test-e2e

.github/workflows/unit.yaml

@@ -19,7 +19,7 @@ jobs:
     - name: Setup Go
       uses: actions/setup-go@v2
       with:
-        go-version: ^1.17
+        go-version: 1.17.x
 
     - name: Check out code into the Go module directory
       uses: actions/checkout@v2

docs/local_source_upload.md

@@ -1,44 +1,52 @@
-Local Source Upload
--------------------
+# Local Source Upload
 
 The purpose of project Shipwright is to transform source code into container images, using known strategies to accomplish this task.
 
 Now, with local source upload, we move Shipwright closer to the developer's inner loop. As a developer, you can use the local source upload feature to stream local content to a Build Controller running in a Kubernetes cluster and create a container image from it. This way, you can try out Shipwright before submitting a pull-request and use the cluster's computing power to build the image.
 
-# Usage
+## Usage
 
 To build an image by using the local source upload feature:
 
 1. Create a Build or use a pre-existing one. Register the standard settings, such as the Build Strategy, on the Build resource. For example:
 
-```bash
-shp build create sample-nodejs \
-    --source-url="https://github.com/shipwright-io/sample-nodejs.git" \
-    --output-image="docker.io/<namespace>/sample-nodejs:latest"
-```
+   ```bash
+   shp build create sample-nodejs \
+       --source-url="https://github.com/shipwright-io/sample-nodejs.git" \
+       --output-image="docker.io/<namespace>/sample-nodejs:latest"
+   ```
 
-2. Clone the repository or use a pre-existing one. For example:
+1. Clone the repository or use a pre-existing one. For example:
 
-```bash
-git clone https://github.com/shipwright-io/sample-nodejs.git && \
-    cd sample-nodejs
-```
+   ```bash
+   git clone https://github.com/shipwright-io/sample-nodejs.git && \
+       cd sample-nodejs
+   ```
 
-3. Keep working on the project. When you're ready to build or rebuild a container image with the local changes, run:
+1. Keep working on the project. When you're ready to build or rebuild a container image with the local changes, run:
 
-```bash
-shp build upload sample-nodejs --follow \
-    --output-image="docker.io/<namespace>/sample-nodejs:<tag>"
-```
+   ```bash
+   shp build upload sample-nodejs --follow \
+       --output-image="docker.io/<namespace>/sample-nodejs:<tag>"
+   ```
+
+Notes:
 
-Notes: 
 - In the preceding examples, replace placeholders like `<namespace>` and `<tag>` with proper values.
 - For `--output-image`, you can specify any Container Registry, use it in combination with `--output-credentials-secret` when needed.
 
-# Streaming
+## Streaming
 
 The subcommand `build upload` creates a new `BuildRun` for the informed `Build`. The newly created `BuildRun` contains settings to instruct the Build Controller to wait for the local user upload instead of cloning the external repository as usual.
 
 The command-line interface orchestrates the process of making the `BuildRun`'s Pod wait, and streaming the specified directory when the Pod is ready for it.
 
 The data streamed to the cluster skips the `.git` directory, if present, and any entries specified by the `.gitignore` file.
+
+## Bundling
+
+Alternatively, the `build upload` command can also make use of the `bundle` feature of the Shipwright Build Controller. Instead of a stream into the build pod, with bundle images the local source code is packed (bundled) together into a container image and then pushed into a container registry. The Pod created as a result of the `BuildRun` will pull this image and extract its content. Please note, if the container registry being used is a separate service, make sure to use private images and authentication to protect the source code.
+
+The bundling feature is used in case the `Build` configures a source bundle image name in the source section. This also needs to have a reference to a secret with the credentials for private images.
+
+Files that should be ignored by the bundle feature need to be configured in `.shpignore` file. It follows Git ignore [patterns](https://git-scm.com/docs/gitignore#_pattern_format).

docs/shp_build_create.md

@@ -30,8 +30,10 @@ shp build create <name> [flags]
       --retention-succeeded-limit uint           number of succeeded BuildRuns to be kept (default 65535)
       --retention-ttl-after-failed duration      duration to delete a failed BuildRun after completion
       --retention-ttl-after-succeeded duration   duration to delete a succeeded BuildRun after completion
+      --source-bundle-image string               source bundle image location, e.g. ghcr.io/shipwright-io/sample-go/source-bundle:latest
+      --source-bundle-prune pruneOption          source bundle prune option, either Never, or AfterPull (default Never)
       --source-context-dir string                use a inner directory as context directory
-      --source-credentials-secret string         name of the secret with git repository credentials
+      --source-credentials-secret string         name of the secret with credentials to access the source, e.g. git or registry credentials
       --source-revision string                   git repository source revision
       --source-url string                        git repository source URL
       --strategy-apiversion string               kubernetes api-version of the build-strategy resource (default "v1alpha1")

docs/shp_build_upload.md

@@ -5,12 +5,17 @@ Run a Build with local data
 ### Synopsis
 
 
-Creates a new BuildRun instance and instructs the Build Controller to wait for the data streamed,
-instead of executing "git clone". Therefore, you can employ Shipwright Builds from a local repository
-clone.
+Creates a new BuildRun instance and instructs the Build Controller to use data from a local directory
+to be used for the Build. Two options are supported: streaming and bundling. With these, you can
+employ Shipwright Builds from a local repository clone.
 
-The upload skips the ".git" directory completely, and it follows the ".gitignore" directives, when
-the file is found at the root of the directory uploaded.
+When streaming is used, the Build Controller waits for the data being streamed to the build pod,
+instead of executing "git clone". The upload skips the ".git" directory completely, and it follows
+the ".gitignore" directives, when the file is found at the root of the directory uploaded.
+
+In case a source bundle image is defined, the bundling feature is used, which will bundle the local
+source code into a bundle container and upload it to the specified container registry. Instead of
+executing using Git in the source step, it will use the container registry to obtain the source code.
 
 	$ shp buildrun upload <build-name>
 	$ shp buildrun upload <build-name> /path/to/repository

go.mod

@@ -3,8 +3,10 @@ module github.com/shipwright-io/cli
 go 1.17
 
 require (
+	github.com/google/go-containerregistry v0.8.1-0.20220216220642-00c59d91847c
 	github.com/onsi/gomega v1.19.0
 	github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06
+	github.com/schollz/progressbar/v3 v3.8.6
 	github.com/shipwright-io/build v0.9.0
 	github.com/spf13/cobra v1.4.0
 	github.com/spf13/pflag v1.0.5
@@ -32,13 +34,19 @@ require (
 	github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd // indirect
 	github.com/PuerkitoBio/purell v1.1.1 // indirect
 	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
+	github.com/acomagu/bufpipe v1.0.3 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blendle/zapdriver v1.3.1 // indirect
 	github.com/census-instrumentation/opencensus-proto v0.3.0 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 // indirect
+	github.com/containerd/stargz-snapshotter/estargz v0.11.3 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/docker/cli v20.10.14+incompatible // indirect
+	github.com/docker/distribution v2.8.1+incompatible // indirect
+	github.com/docker/docker v20.10.14+incompatible // indirect
+	github.com/docker/docker-credential-helpers v0.6.4 // indirect
 	github.com/emicklei/go-restful v2.15.0+incompatible // indirect
 	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
 	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
@@ -47,20 +55,21 @@ require (
 	github.com/fvbommel/sortorder v1.0.1 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/go-errors/errors v1.0.1 // indirect
+	github.com/go-git/gcfg v1.5.0 // indirect
+	github.com/go-git/go-billy/v5 v5.3.1 // indirect
+	github.com/go-git/go-git/v5 v5.4.2 // indirect
 	github.com/go-kit/log v0.2.0 // indirect
 	github.com/go-logfmt/logfmt v0.5.1 // indirect
 	github.com/go-logr/logr v1.2.3 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
 	github.com/go-openapi/jsonreference v0.19.6 // indirect
-	github.com/go-openapi/spec v0.20.4 // indirect
 	github.com/go-openapi/swag v0.21.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt/v4 v4.4.1 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/btree v1.0.1 // indirect
 	github.com/google/go-cmp v0.5.7 // indirect
-	github.com/google/go-containerregistry v0.8.1-0.20220216220642-00c59d91847c // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/google/uuid v1.3.0 // indirect
@@ -69,20 +78,26 @@ require (
 	github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
-	github.com/hashicorp/golang-lru v0.5.4 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/compress v1.15.1 // indirect
 	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/mattn/go-runewidth v0.0.13 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
+	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/go-wordwrap v1.0.0 // indirect
 	github.com/moby/spdystream v0.2.0 // indirect
 	github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.0.3-0.20220303224323-02efb9a75ee1 // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
@@ -91,10 +106,13 @@ require (
 	github.com/prometheus/common v0.33.0 // indirect
 	github.com/prometheus/procfs v0.7.3 // indirect
 	github.com/prometheus/statsd_exporter v0.22.4 // indirect
+	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/russross/blackfriday v1.6.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/sirupsen/logrus v1.8.1 // indirect
 	github.com/stretchr/testify v1.7.0 // indirect
 	github.com/tektoncd/pipeline v0.34.1 // indirect
+	github.com/vbatts/tar-split v0.11.2 // indirect
 	github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca // indirect
 	go.opencensus.io v0.23.0 // indirect
 	go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect
@@ -116,6 +134,7 @@ require (
 	google.golang.org/grpc v1.45.0 // indirect
 	google.golang.org/protobuf v1.28.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 	k8s.io/component-base v0.23.4 // indirect