diff --git a/download/oci_download.go b/download/oci_download.go index e506098eed..8544df246f 100644 --- a/download/oci_download.go +++ b/download/oci_download.go @@ -253,7 +253,7 @@ func (d *OCIDownloader) download(ctx context.Context, m metrics.Metrics) (*downl return nil, err } loader := bundle.NewTarballLoaderWithBaseURL(fileReader, d.localStorePath) - reader := bundle.NewCustomReader(loader).WithBaseDir(d.localStorePath). + reader := bundle.NewCustomReader(loader). WithMetrics(m). WithBundleVerificationConfig(d.bvc). WithBundleEtag(etag) diff --git a/download/oci_download_test.go b/download/oci_download_test.go index 1eb17f0265..2653cf021a 100644 --- a/download/oci_download_test.go +++ b/download/oci_download_test.go @@ -7,6 +7,7 @@ import ( "context" "encoding/base64" "fmt" + "github.com/open-policy-agent/opa/bundle" "net/http" "strings" "testing" @@ -16,6 +17,48 @@ import ( "github.com/open-policy-agent/opa/plugins/rest" ) +// when changed the layer hash & size should be updated in signed.manifest +//go:generate go run github.com/open-policy-agent/opa build -b --signing-alg HS256 --signing-key secret testdata/signed_bundle_data --output testdata/signed.tar.gz + +func TestOCIDownloaderWithBundleVerificationConfig(t *testing.T) { + vc := bundle.NewVerificationConfig(map[string]*bundle.KeyConfig{"default": {Key: "secret", Algorithm: "HS256"}}, "", "", nil) + ctx := context.Background() + fixture := newTestFixture(t) + fixture.server.expEtag = "sha256:c5834dbce332cabe6ae68a364de171a50bf5b08024c27d7c08cc72878b4df7ff" + + updates := make(chan *Update) + + config := Config{} + if err := config.ValidateAndInjectDefaults(); err != nil { + t.Fatal(err) + } + + d := NewOCI(config, fixture.client, "ghcr.io/org/repo:signed", "/tmp/opa/").WithCallback(func(_ context.Context, u Update) { + if u.Error != nil { + t.Fatalf("expected no error but got: %v", u.Error) + } + updates <- &u + }).WithBundleVerificationConfig(vc) + + d.Start(ctx) + + // Give time for some download events to occur + time.Sleep(1 * time.Second) + + u1 := <-updates + + if u1.Bundle == nil || len(u1.Bundle.Modules) == 0 { + t.Fatal("expected bundle with at least one module but got:", u1) + } + + if !strings.HasSuffix(u1.Bundle.Modules[0].URL, u1.Bundle.Modules[0].Path) { + t.Fatalf("expected URL to have path as suffix but got %v and %v", u1.Bundle.Modules[0].URL, u1.Bundle.Modules[0].Path) + } + + d.Stop(ctx) + +} + func TestOCIStartStop(t *testing.T) { ctx := context.Background() fixture := newTestFixture(t) diff --git a/download/testdata/manifest.layer b/download/testdata/latest.manifest similarity index 100% rename from download/testdata/manifest.layer rename to download/testdata/latest.manifest diff --git a/download/testdata/tar.layer b/download/testdata/latest.tar.gz similarity index 100% rename from download/testdata/tar.layer rename to download/testdata/latest.tar.gz diff --git a/download/testdata/signed.manifest b/download/testdata/signed.manifest new file mode 100644 index 0000000000..4aecee2583 --- /dev/null +++ b/download/testdata/signed.manifest @@ -0,0 +1,19 @@ +{ + "schemaVersion":2, + "config":{ + "mediaType":"application/vnd.oci.image.config.v1+json", + "digest":"sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + "size":2 + }, + "layers":[ + { + "mediaType":"application/vnd.oci.image.layer.v1.tar+gzip", + "digest":"sha256:e060c7b9558fad3ec85df5ffa19d0d019f839c36d7ec146977c871dcbc70885e", + "size":629, + "annotations":{ + "org.opencontainers.image.created":"2022-02-11T09:00:07Z", + "org.opencontainers.image.title":"dani/testpol" + } + } + ] +} \ No newline at end of file diff --git a/download/testdata/signed.tar.gz b/download/testdata/signed.tar.gz new file mode 100644 index 0000000000..f08d5185ca Binary files /dev/null and b/download/testdata/signed.tar.gz differ diff --git a/download/testdata/signed_bundle_data/a/b/c/data.json b/download/testdata/signed_bundle_data/a/b/c/data.json new file mode 100644 index 0000000000..3a26a2e5e9 --- /dev/null +++ b/download/testdata/signed_bundle_data/a/b/c/data.json @@ -0,0 +1 @@ +[1,2,3] \ No newline at end of file diff --git a/download/testdata/signed_bundle_data/http/policy/policy.rego b/download/testdata/signed_bundle_data/http/policy/policy.rego new file mode 100644 index 0000000000..714f149b1d --- /dev/null +++ b/download/testdata/signed_bundle_data/http/policy/policy.rego @@ -0,0 +1 @@ +package example \ No newline at end of file diff --git a/download/testharness.go b/download/testharness.go index c0d39324b2..b262bc1da5 100644 --- a/download/testharness.go +++ b/download/testharness.go @@ -6,9 +6,11 @@ package download import ( "bytes" "context" + "crypto/sha256" "encoding/json" "errors" "fmt" + "io" "net/http" "net/http/httptest" "os" @@ -246,6 +248,11 @@ func (t *testFixture) oneShot(ctx context.Context, u Update) { t.etags["test/bundle1"] = u.ETag } +type fileInfo struct { + name string + length int64 +} + type testServer struct { t *testing.T customAuth func(http.ResponseWriter, *http.Request) error @@ -256,6 +263,7 @@ type testServer struct { server *httptest.Server etagInResponse bool longPoll bool + testdataHashes map[string]fileInfo } func newTestServer(t *testing.T) *testServer { @@ -339,68 +347,58 @@ func (t *testServer) handle(w http.ResponseWriter, r *http.Request) { var buf bytes.Buffer - if r.URL.Path == "/v2/org/repo/manifests/latest" { - w.Header().Add("Content-Length", "596") - w.Header().Add("Content-Type", "application/vnd.oci.image.manifest.v1+json") - w.Header().Add("Docker-Content-Digest", "sha256:fe9c2930b6d8cc1bf3fa0c560996a95c75f0d0668bee71138355d9784c8c99b8") - w.WriteHeader(200) - return - } - if r.URL.Path == "/v2/org/repo/manifests/sha256:fe9c2930b6d8cc1bf3fa0c560996a95c75f0d0668bee71138355d9784c8c99b8" { - w.Header().Add("Content-Length", "596") - w.Header().Add("Content-Type", "application/vnd.oci.image.manifest.v1+json") - w.Header().Add("Docker-Content-Digest", "sha256:fe9c2930b6d8cc1bf3fa0c560996a95c75f0d0668bee71138355d9784c8c99b8") - w.WriteHeader(200) - bs, err := os.ReadFile("testdata/manifest.layer") - if err != nil { - w.WriteHeader(404) - return + if strings.HasPrefix(r.URL.Path, "/v2/org/repo/") { + // build test data to hash map to serve testdata files by hash + if t.testdataHashes == nil { + t.testdataHashes = make(map[string]fileInfo) + files, err := os.ReadDir("testdata") + if err != nil { + t.t.Fatalf("failed to read testdata directory: %s", err) + } + for _, file := range files { + if file.IsDir() { + continue + } + hash, length, err := getFileSHAandSize("testdata/" + file.Name()) + if err != nil { + t.t.Fatalf("failed to read testdata file: %s", err) + } + t.testdataHashes[fmt.Sprintf("%x", hash)] = fileInfo{name: file.Name(), length: length} + } } - buf.WriteString(string(bs)) - w.Write(buf.Bytes()) - return } - if r.URL.Path == "/v2/org/repo/blobs/sha256:c5834dbce332cabe6ae68a364de171a50bf5b08024c27d7c08cc72878b4df7ff" { - w.Header().Add("Content-Length", "464") - w.Header().Add("Content-Type", "application/vnd.oci.image.layer.v1.tar+gzip,application/vnd.oci.image.config.v1+json") - w.Header().Add("Docker-Content-Digest", "sha256:c5834dbce332cabe6ae68a364de171a50bf5b08024c27d7c08cc72878b4df7ff") - w.WriteHeader(200) - bs, err := os.ReadFile("testdata/manifest.layer") - if err != nil { - w.WriteHeader(404) - return - } - buf.WriteString(string(bs)) - buf.WriteTo(w) - return - } - if r.URL.Path == "/v2/org/repo/blobs/sha256:b206ac766b0f3f880f6a62c4bb5ba5192d29deaefd989a1961603346a7555bdd" { - w.Header().Add("Content-Length", "568") - w.Header().Add("Content-Type", "application/vnd.oci.image.layer.v1.tar+gzip") - w.Header().Add("Docker-Content-Digest", "sha256:b206ac766b0f3f880f6a62c4bb5ba5192d29deaefd989a1961603346a7555bdd") - w.WriteHeader(200) - bs, err := os.ReadFile("testdata/tar.layer") - if err != nil { - w.WriteHeader(404) + if strings.HasPrefix(r.URL.Path, "/v2/org/repo/blobs/sha256:") || strings.HasPrefix(r.URL.Path, "/v2/org/repo/manifests/sha256:") { + sha := strings.TrimPrefix(strings.TrimPrefix(r.URL.Path, "/v2/org/repo/blobs/sha256:"), "/v2/org/repo/manifests/sha256:") + if fileInfo, ok := t.testdataHashes[sha]; ok { + w.Header().Add("Content-Length", strconv.Itoa(int(fileInfo.length))) + w.Header().Add("Content-Type", "application/gzip") + w.Header().Add("Docker-Content-Digest", "sha256:"+sha) + w.WriteHeader(200) + bs, err := os.ReadFile("testdata/" + fileInfo.name) + if err != nil { + w.WriteHeader(404) + return + } + buf.WriteString(string(bs)) + w.Write(buf.Bytes()) return } - buf.WriteString(string(bs)) - w.Write(buf.Bytes()) + w.WriteHeader(404) return } - if r.URL.Path == "/v2/org/repo/blobs/sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" { - w.Header().Add("Content-Length", "2") - w.Header().Add("Content-Type", "application/vnd.oci.image.config.v1+json") - w.Header().Add("Docker-Content-Digest", "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a") - w.WriteHeader(200) - bs, err := os.ReadFile("testdata/config.layer") + + if strings.HasPrefix(r.URL.Path, "/v2/org/repo/manifests/") { + sha, size, err := getFileSHAandSize("testdata/" + strings.TrimPrefix(r.URL.Path, "/v2/org/repo/manifests/") + ".manifest") if err != nil { w.WriteHeader(404) return } - buf.WriteString(string(bs)) - w.Write(buf.Bytes()) + + w.Header().Add("Content-Length", strconv.Itoa(int(size))) + w.Header().Add("Content-Type", "application/vnd.oci.image.manifest.v1+json") + w.Header().Add("Docker-Content-Digest", "sha256:"+fmt.Sprintf("%x", sha)) + w.WriteHeader(200) return } name := strings.TrimPrefix(r.URL.Path, "/bundles/") @@ -487,3 +485,17 @@ func getPreferHeaderField(r *http.Request, field string) string { } return "" } + +func getFileSHAandSize(filePath string) ([]byte, int64, error) { + f, err := os.Open(filePath) + if err != nil { + return nil, 0, err + } + defer f.Close() + hash := sha256.New() + w, err := io.Copy(hash, f) + if err != nil { + return nil, w, err + } + return hash.Sum(nil), w, nil +}