test(e2e): add E2E test infra

.github/licenserc.yml

@@ -41,3 +41,8 @@ header:
 dependency:
   files:
     - go.mod
+  licenses:
+    # known issue: https://github.com/apache/skywalking-eyes/pull/107#issuecomment-1129761574
+    - name: github.com/chzyer/logex
+      version: v1.1.10
+      license: MIT

.github/workflows/build.yml

@@ -39,6 +39,22 @@ jobs:
         run: make build-linux-amd64
       - name: Run Unit Tests
         run: make test
+      - name: Run E2E Tests
+        run: |
+          go install -mod=mod github.com/onsi/ginkgo/v2/ginkgo
+          tar -xvf $GITHUB_WORKSPACE/test/e2e/testdata/distribution/mount.tar -C $GITHUB_WORKSPACE/test/e2e/testdata/distribution/ > /dev/null
+          docker kill oras-e2e || true
+          trap 'docker kill oras-e2e || true' ERR
+          docker run -d -p 5000:5000 --rm --name oras-e2e \
+            --mount type=bind,source=$GITHUB_WORKSPACE/test/e2e/testdata/distribution/config-example-with-extensions.yml,target=/etc/docker/registry/config.yml \
+            --mount type=bind,source=$GITHUB_WORKSPACE/test/e2e/testdata/distribution/passwd_bcrypt,target=/etc/docker/registry/passwd \
+            --mount type=bind,source=$GITHUB_WORKSPACE/test/e2e/testdata/distribution/docker,target=/opt/data/registry-root-dir/docker \
+            ghcr.io/oras-project/registry:latest
+          ginkgo -r -p $GITHUB_WORKSPACE/test/e2e/suite
+          docker kill oras-e2e || true
+        env:
+          ORAS_PATH: bin/linux/amd64/oras
+          ORAS_REGISTRY_HOST: localhost:5000
       - name: Check Version
         run: bin/linux/amd64/oras version
       - name: Upload coverage to codecov.io

.gitignore

@@ -38,3 +38,4 @@ dist/
 *.tar.gz
 vendor/
 _dist/
+test/e2e/testdata/distribution/docker/

Makefile

@@ -32,7 +32,7 @@ LDFLAGS += -X $(PROJECT_PKG)/internal/version.GitTreeState=${GIT_DIRTY}
 
 .PHONY: test
 test: vendor check-encoding
-	go test -race -v -coverprofile=coverage.txt -covermode=atomic ./...
+	go test -race -v -coverprofile=coverage.txt -covermode=atomic $(shell go list ./... | grep -v test/e2e/)
 
 .PHONY: covhtml
 covhtml:

go.mod

@@ -6,6 +6,8 @@ require (
 	github.com/docker/cli v20.10.18+incompatible
 	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6
 	github.com/need-being/go-tree v0.1.0
+	github.com/onsi/ginkgo/v2 v2.1.6
+	github.com/onsi/gomega v1.20.2
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799
 	github.com/oras-project/artifacts-spec v1.0.0-rc.2
@@ -17,11 +19,15 @@ require (
 
 require (
 	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
-	github.com/docker/docker v20.10.17+incompatible // indirect
+	github.com/docker/docker v20.10.18+incompatible // indirect
 	github.com/docker/docker-credential-helpers v0.6.4 // indirect
+	github.com/google/go-cmp v0.5.8 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/opencontainers/distribution-spec/specs-go v0.0.0-20220620172159-4ab4752c3b86 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
+	golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
-	golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect
+	golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect
+	golang.org/x/text v0.3.7 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -9,19 +9,25 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/docker/cli v20.10.18+incompatible h1:f/GQLsVpo10VvToRay2IraVA1wHz9KktZyjev3SIVDU=
 github.com/docker/cli v20.10.18+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
-github.com/docker/docker v20.10.17+incompatible h1:JYCuMrWaVNophQTOrMMoSwudOVEfcegoZZrleKc1xwE=
-github.com/docker/docker v20.10.17+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v20.10.18+incompatible h1:SN84VYXTBNGn92T/QwIRPlum9zfemfitN7pbsp26WSc=
+github.com/docker/docker v20.10.18+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/docker-credential-helpers v0.6.4 h1:axCks+yV+2MR3/kZhAmy07yC56WZ2Pwu/fKWtKuZB0o=
 github.com/docker/docker-credential-helpers v0.6.4/go.mod h1:ofX3UI0Gz1TteYBjtgs07O36Pyasyp66D2uKT7H8W1c=
+github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
+github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 h1:dcztxKSvZ4Id8iPpHERQBbIJfabdt4wUm5qy3wOL2Zc=
 github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw=
 github.com/need-being/go-tree v0.1.0 h1:blQrtD006cFm97UDeMUfixwPc9o06A6c+uLaUskdNNw=
 github.com/need-being/go-tree v0.1.0/go.mod h1:UOHUchuOm+lxM+EtvQ9h/IO88hK/ke7FHai4oGhhEoI=
+github.com/onsi/ginkgo/v2 v2.1.6 h1:Fx2POJZfKRQcM1pH49qSZiYeu319wji004qX+GDovrU=
+github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk=
+github.com/onsi/gomega v1.20.2 h1:8uQq0zMgLEfa0vRrrBgaJF2gyW9Da9BmfGV+OyUzfkY=
+github.com/onsi/gomega v1.20.2/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc=
 github.com/opencontainers/distribution-spec/specs-go v0.0.0-20220620172159-4ab4752c3b86 h1:Oumw+lPnO8qNLTY2mrqPJZMoGExLi/0h/DdikoLTXVU=
 github.com/opencontainers/distribution-spec/specs-go v0.0.0-20220620172159-4ab4752c3b86/go.mod h1:aA4vdXRS8E1TG7pLZOz85InHi3BiPdErh8IpJN6E0x4=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
@@ -49,22 +55,30 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b h1:PxfKdU9lEEDYjdIzOtC4qFWgkU2rGHdKlKowJSMN9h0=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools/v3 v3.0.2 h1:kG1BFyqVHuQoVQiR1bWGnfz/fmHvvuiSPIV7rvl360E=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
 oras.land/oras-go/v2 v2.0.0-rc.3.0.20220922092058-3f9653f7bf69 h1:HpwQf1GbCkpVTdmHSsBiJpfZMWxqPjPjcKbtX6VXtGI=

test/e2e/README.md

@@ -0,0 +1,93 @@
+# ORAS End-to-End Testing Dev Guide
+
+## Setting up
+Minimal setup: Run the script in **step 3**
+
+### 1. Common dev setup for ORAS CLI
+https://hackmd.io/_nRHGW8WRfOOvngWc6u-sQ
+
+### 2. [Optional] Install Ginkgo
+This will enable you use`ginkgo` directly in CLI.
+```
+go install github.com/onsi/ginkgo/v2/ginkgo@latest
+```
+If you skip step 2, you can only run tests via `go test`. 
+
+### 3. Run distribution
+The backend of E2E test is a [oras-distribution](https://github.com/oras-project/distribution).
+```bash
+REPO_ROOT=$(pwd) # Set REPO_ROOT as root folder of oras CLI
+PORT=5000
+docker run -dp $PORT:5000 --rm --name oras-e2e \
+    --mount type=bind,source=$REPO_ROOT/test/e2e/testdata/distribution/config-example-with-extensions.yml,target=/etc/docker/registry/config.yml \
+    --mount type=bind,source=$REPO_ROOT/test/e2e/testdata/distribution/passwd_bcrypt,target=/etc/docker/registry/passwd \
+    ghcr.io/oras-project/registry:latest
+```
+If the image cannot be pulled, try create a Github PAT and docker/oras login.
+
+### 4. [Optional] Setup backend
+```bash
+export ORAS_REGISTRY_HOST="localhost:$PORT" # replace with right os/arch
+# for PowerShell, use $env:ORAS_REGISTRY_HOST = "localhost:$PORT"
+```
+If you skip step 4, E2E test will look for distribution ran in `localhost:5000`
+
+### 5. [Optional] Setup ORAS binary for testing
+```bash
+# Set REPO_ROOT as root folder of oras CLI code
+cd $REPO_ROOT
+make build
+```
+### 6. [Optional] Setup pre-built binary
+You need to setup below environmental variables to debug a pre-built ORAS binary:
+```bash
+export ORAS_PATH="bin/linux/amd64/oras" # change target platform if needed
+export GITHUB_WORKSPACE=$REPO_ROOT
+```
+If you skip step 5 or 6, Gomega will build a temp binary, which will include all the CLI code changes in the working directory.
+
+### 7. [Optional] Mount test data
+If you want to run command suite, you need to unzip the test file tarball and mount to the distribution
+```bash
+tar -xvf $REPO_ROOT/test/e2e/testdata/distribution/mount.tar -C $REPO_ROOT/test/e2e/testdata/distribution/
+
+REPO_ROOT=$(pwd)
+PORT=5000
+docker run -dp $PORT:5000 --rm --name oras-e2e \
+    --mount type=bind,source=$REPO_ROOT/test/e2e/testdata/distribution/config-example-with-extensions.yml,target=/etc/docker/registry/config.yml \
+    --mount type=bind,source=$REPO_ROOT/test/e2e/testdata/distribution/passwd_bcrypt,target=/etc/docker/registry/passwd \
+    --mount type=bind,source=$REPO_ROOT/test/e2e/testdata/distribution/docker,target=/opt/data/registry-root-dir/docker \ # mount test data
+    ghcr.io/oras-project/registry:latest
+```
+Skipping step 7 you will not be able to run Command suite.
+
+
+## Debugging
+### 1. Constant Build & Watch
+This is a good choice if you want to debug certain re-runnable specs
+```bash
+cd $REPO_ROOT/test/e2e
+ginkgo watch -r
+```
+
+### 2. Debug certain action
+Use [focused spec](https://onsi.github.io/ginkgo/#focused-specs)
+
+
+### 3. Trouble-shooting CLI
+Executed command should be shown in the ginkgo logs after `[It]`,
+
+## Development
+### Adding New Tests Using Ginkgo DSL
+Two suites will be maintained for E2E testing:
+- command: contains test specs for single oras command execution
+- scenario: contains featured scenarios with several oras commands execution
+
+Inside a suite, please follow below model when building the hierarchical collections of specs:
+```
+Describe: <Role>
+  Context: Scenario or command specific description
+    When: <Action>
+      It: <Result> (per-command execution)
+       Expect: <Result> (detailed checks for execution results)
+```

test/e2e/internal/utils/init.go

@@ -0,0 +1,61 @@
+/*
+Copyright The ORAS Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	. "github.com/onsi/ginkgo/v2"
+	"github.com/onsi/gomega"
+	"github.com/onsi/gomega/gexec"
+	"oras.land/oras-go/v2/registry"
+)
+
+var OrasPath string
+var Host string
+
+func init() {
+	Host = os.Getenv("ORAS_REGISTRY_HOST")
+	if Host == "" {
+		Host = "localhost:5000"
+		os.Stderr.Write([]byte(fmt.Sprintln("cannot find host name in ORAS_REGISTRY_HOST, using " + Host + " instead")))
+	}
+	if err := (registry.Reference{Registry: Host}).ValidateRegistry(); Host == "" || err != nil {
+		panic(err)
+	}
+	var _ = BeforeSuite(func() {
+		OrasPath = os.Getenv("ORAS_PATH")
+		if filepath.IsAbs(OrasPath) {
+			// test against OrasPath directly
+			fmt.Printf("Testing based on pre-built binary locates in %q\n", OrasPath)
+		} else if workspacePath := os.Getenv("GITHUB_WORKSPACE"); OrasPath != "" && workspacePath != "" {
+			// add workspacePath as prefix, both path env should not be empty
+			OrasPath = filepath.Join(workspacePath, OrasPath)
+			var err error
+			OrasPath, err = filepath.Abs(OrasPath)
+			gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+			fmt.Printf("Testing based on pre-built binary locates in %q\n", OrasPath)
+		} else {
+			// fallback to native build to facilitate local debugging
+			var err error
+			OrasPath, err = gexec.Build("oras.land/oras/cmd/oras")
+			gomega.Expect(err).NotTo(gomega.HaveOccurred())
+			DeferCleanup(gexec.CleanupBuildArtifacts)
+			fmt.Printf("Testing based on temp binary locates in %q\n", OrasPath)
+		}
+	})
+}

test/e2e/suite/command/command_suite_test.go

@@ -0,0 +1,28 @@
+/*
+Copyright The ORAS Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package command
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestCommand(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Command Suite")
+}

test/e2e/suite/command/version.go

@@ -0,0 +1,38 @@
+/*
+Copyright The ORAS Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package command
+
+import (
+	"os/exec"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"github.com/onsi/gomega/gexec"
+	"oras.land/oras/test/e2e/internal/utils"
+)
+
+var _ = Describe("ORAS user", func() {
+	Context("checks oras version", func() {
+		When("running version command", func() {
+			It("should success", func() {
+				session, err := gexec.Start(exec.Command(utils.OrasPath, "version"), nil, nil)
+				Expect(err).ShouldNot(HaveOccurred())
+				Eventually(session, "10s").Should(gexec.Exit(0))
+			})
+		})
+
+	})
+})