From 776fa43eb20380f6a7a3164b942146d87c824deb Mon Sep 17 00:00:00 2001 From: Cole Wippern Date: Fri, 20 Dec 2019 18:13:56 -0800 Subject: [PATCH] refactor cache.Warm and add tests --- pkg/cache/cache.go | 34 ----------- pkg/cache/doc_test.go | 47 +++++++++++++++ pkg/cache/errors.go | 79 +++++++++++++++++++++++++ pkg/cache/warm.go | 128 ++++++++++++++++++++++++++++++++--------- pkg/cache/warm_test.go | 116 +++++++++++++++++++++++++++++++++++++ pkg/fakes/image.go | 60 +++++++++++++++++++ 6 files changed, 402 insertions(+), 62 deletions(-) create mode 100644 pkg/cache/doc_test.go create mode 100644 pkg/cache/errors.go create mode 100644 pkg/cache/warm_test.go create mode 100644 pkg/fakes/image.go diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index 0f3aa51f3e..893b1142da 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -35,40 +35,6 @@ import ( "github.com/sirupsen/logrus" ) -type NotFoundErr struct { - msg string -} - -func (e NotFoundErr) Error() string { - return e.msg -} - -type ExpiredErr struct { - msg string -} - -func (e ExpiredErr) Error() string { - return e.msg -} - -func IsNotFound(e error) bool { - switch e.(type) { - case NotFoundErr: - return true - } - - return false -} - -func IsExpired(e error) bool { - switch e.(type) { - case ExpiredErr: - return true - } - - return false -} - // LayerCache is the layer cache type LayerCache interface { RetrieveLayer(string) (v1.Image, error) diff --git a/pkg/cache/doc_test.go b/pkg/cache/doc_test.go new file mode 100644 index 0000000000..49443a3ef6 --- /dev/null +++ b/pkg/cache/doc_test.go @@ -0,0 +1,47 @@ +/* +Copyright 2019 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cache + +import ( + "bytes" + "log" + + "github.com/GoogleContainerTools/kaniko/pkg/config" + "github.com/google/go-containerregistry/pkg/v1/remote" +) + +func ExampleWarmer_Warm() { + tarBuf := new(bytes.Buffer) + manifestBuf := new(bytes.Buffer) + w := &Warmer{ + Remote: remote.Image, + Local: LocalSource, + TarWriter: tarBuf, + ManifestWriter: manifestBuf, + } + + options := &config.WarmerOptions{} + + digest, err := w.Warm("ubuntu:latest", options) + if err != nil { + if !IsAlreadyCached(err) { + log.Fatal(err) + } + } + + log.Printf("digest %v tar len %d\nmanifest:\n%s\n", digest, tarBuf.Len(), manifestBuf.String()) +} diff --git a/pkg/cache/errors.go b/pkg/cache/errors.go new file mode 100644 index 0000000000..806f438cef --- /dev/null +++ b/pkg/cache/errors.go @@ -0,0 +1,79 @@ +/* +Copyright 2019 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cache + +// IsAlreadyCached returns true if the supplied error is of the type AlreadyCachedErr +// otherwise it returns false. +func IsAlreadyCached(err error) bool { + switch err.(type) { + case AlreadyCachedErr: + return true + } + + return false +} + +// AlreadyCachedErr is returned when the Docker image requested for caching is already +// present in the cache. +type AlreadyCachedErr struct { + msg string +} + +func (a AlreadyCachedErr) Error() string { + return a.msg +} + +// IsNotFound returns true if the supplied error is of the type NotFoundErr +// otherwise it returns false. +func IsNotFound(e error) bool { + switch e.(type) { + case NotFoundErr: + return true + } + + return false +} + +// NotFoundErr is returned when the requested Docker image is not present in the cache. +type NotFoundErr struct { + msg string +} + +func (e NotFoundErr) Error() string { + return e.msg +} + +// IsExpired returns true if the supplied error is of the type ExpiredErr +// otherwise it returns false. +func IsExpired(e error) bool { + switch e.(type) { + case ExpiredErr: + return true + } + + return false +} + +// ExpiredErr is returned when the requested Docker image is present in the cache, but is +// expired according to the supplied TTL. +type ExpiredErr struct { + msg string +} + +func (e ExpiredErr) Error() string { + return e.msg +} diff --git a/pkg/cache/warm.go b/pkg/cache/warm.go index 254065ea06..c4e6a03fd5 100644 --- a/pkg/cache/warm.go +++ b/pkg/cache/warm.go @@ -17,12 +17,15 @@ limitations under the License. package cache import ( - "fmt" + "bytes" + "io" "io/ioutil" + "os" "path" "github.com/GoogleContainerTools/kaniko/pkg/config" "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/google/go-containerregistry/pkg/v1/tarball" "github.com/pkg/errors" @@ -37,42 +40,111 @@ func WarmCache(opts *config.WarmerOptions) error { logrus.Debugf("%s\n", images) for _, image := range images { - cacheRef, err := name.NewTag(image, name.WeakValidation) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("Failed to verify image name: %s", image)) - } - img, err := remote.Image(cacheRef) - if err != nil || img == nil { - return errors.Wrap(err, fmt.Sprintf("Failed to retrieve image: %s", image)) - } + tarBuf := new(bytes.Buffer) + manifestBuf := new(bytes.Buffer) - digest, err := img.Digest() - if err != nil { - return errors.Wrap(err, fmt.Sprintf("Failed to retrieve digest: %s", image)) + cw := &Warmer{ + Remote: remote.Image, + Local: LocalSource, + TarWriter: tarBuf, + ManifestWriter: manifestBuf, } - cachePath := path.Join(cacheDir, digest.String()) - if !opts.Force { - _, err := LocalSource(&opts.CacheOptions, digest.String()) - if err == nil || IsExpired(err) { - continue + digest, err := cw.Warm(image, opts) + if err != nil { + if !IsAlreadyCached(err) { + return err } - } - err = tarball.WriteToFile(cachePath, cacheRef, img) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("Failed to write %s to cache", image)) + continue } - mfst, err := img.RawManifest() - if err != nil { - return errors.Wrap(err, fmt.Sprintf("Failed to retrieve manifest for %s", image)) - } - mfstPath := cachePath + ".json" - if err := ioutil.WriteFile(mfstPath, mfst, 0666); err != nil { - return errors.Wrap(err, fmt.Sprintf("Failed to save manifest for %s", image)) + cachePath := path.Join(cacheDir, digest.String()) + + if err := writeBufsToFile(cachePath, tarBuf, manifestBuf); err != nil { + return err } + logrus.Debugf("Wrote %s to cache", image) } return nil } + +func writeBufsToFile(cachePath string, tarBuf, manifestBuf *bytes.Buffer) error { + f, err := os.Create(cachePath) + if err != nil { + return err + } + defer f.Close() + + if _, err := f.Write(tarBuf.Bytes()); err != nil { + return errors.Wrap(err, "Failed to save tar to file") + } + + mfstPath := cachePath + ".json" + if err := ioutil.WriteFile(mfstPath, manifestBuf.Bytes(), 0666); err != nil { + return errors.Wrap(err, "Failed to save manifest to file") + } + + return nil +} + +// FetchRemoteImage retrieves a Docker image manifest from a remote source. +// github.com/google/go-containerregistry/pkg/v1/remote.Image can be used as +// this type. +type FetchRemoteImage func(name.Reference, ...remote.Option) (v1.Image, error) + +// FetchLocalSource retrieves a Docker image manifest from a local source. +// github.com/GoogleContainerTools/kaniko/cache.LocalSource can be used as +// this type. +type FetchLocalSource func(*config.CacheOptions, string) (v1.Image, error) + +// Warmer is used to prepopulate the cache with a Docker image +type Warmer struct { + Remote FetchRemoteImage + Local FetchLocalSource + TarWriter io.Writer + ManifestWriter io.Writer +} + +// Warm retrieves a Docker image and populates the supplied buffer with the image content and manifest +// or returns an AlreadyCachedErr if the image is present in the cache. +func (w *Warmer) Warm(image string, opts *config.WarmerOptions) (v1.Hash, error) { + cacheRef, err := name.NewTag(image, name.WeakValidation) + if err != nil { + return v1.Hash{}, errors.Wrapf(err, "Failed to verify image name: %s", image) + } + + img, err := w.Remote(cacheRef) + if err != nil || img == nil { + return v1.Hash{}, errors.Wrapf(err, "Failed to retrieve image: %s", image) + } + + digest, err := img.Digest() + if err != nil { + return v1.Hash{}, errors.Wrapf(err, "Failed to retrieve digest: %s", image) + } + + if !opts.Force { + _, err := w.Local(&opts.CacheOptions, digest.String()) + if err == nil || IsExpired(err) { + return v1.Hash{}, AlreadyCachedErr{} + } + } + + err = tarball.Write(cacheRef, img, w.TarWriter) + if err != nil { + return v1.Hash{}, errors.Wrapf(err, "Failed to write %s to tar buffer", image) + } + + mfst, err := img.RawManifest() + if err != nil { + return v1.Hash{}, errors.Wrapf(err, "Failed to retrieve manifest for %s", image) + } + + if _, err := w.ManifestWriter.Write(mfst); err != nil { + return v1.Hash{}, errors.Wrapf(err, "Failed to save manifest to buffer for %s", image) + } + + return digest, nil +} diff --git a/pkg/cache/warm_test.go b/pkg/cache/warm_test.go new file mode 100644 index 0000000000..554210b4e0 --- /dev/null +++ b/pkg/cache/warm_test.go @@ -0,0 +1,116 @@ +/* +Copyright 2019 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cache + +import ( + "bytes" + "testing" + + "github.com/GoogleContainerTools/kaniko/pkg/config" + "github.com/GoogleContainerTools/kaniko/pkg/fakes" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/remote" +) + +const ( + image = "foo:latest" +) + +func Test_Warmer_Warm_not_in_cache(t *testing.T) { + tarBuf := new(bytes.Buffer) + manifestBuf := new(bytes.Buffer) + + cw := &Warmer{ + Remote: func(_ name.Reference, _ ...remote.Option) (v1.Image, error) { + return fakes.FakeImage{}, nil + }, + Local: func(_ *config.CacheOptions, _ string) (v1.Image, error) { + return nil, NotFoundErr{} + }, + TarWriter: tarBuf, + ManifestWriter: manifestBuf, + } + + opts := &config.WarmerOptions{} + + _, err := cw.Warm(image, opts) + if err != nil { + t.Errorf("expected error to be nil but was %v", err) + t.FailNow() + } + + if len(tarBuf.Bytes()) == 0 { + t.Error("expected image to be written but buffer was empty") + } +} + +func Test_Warmer_Warm_in_cache_not_expired(t *testing.T) { + tarBuf := new(bytes.Buffer) + manifestBuf := new(bytes.Buffer) + + cw := &Warmer{ + Remote: func(_ name.Reference, _ ...remote.Option) (v1.Image, error) { + return fakes.FakeImage{}, nil + }, + Local: func(_ *config.CacheOptions, _ string) (v1.Image, error) { + return fakes.FakeImage{}, nil + }, + TarWriter: tarBuf, + ManifestWriter: manifestBuf, + } + + opts := &config.WarmerOptions{} + + _, err := cw.Warm(image, opts) + if !IsAlreadyCached(err) { + t.Errorf("expected error to be already cached err but was %v", err) + t.FailNow() + } + + if len(tarBuf.Bytes()) != 0 { + t.Errorf("expected nothing to be written") + } +} + +func Test_Warmer_Warm_in_cache_expired(t *testing.T) { + tarBuf := new(bytes.Buffer) + manifestBuf := new(bytes.Buffer) + + cw := &Warmer{ + Remote: func(_ name.Reference, _ ...remote.Option) (v1.Image, error) { + return fakes.FakeImage{}, nil + }, + Local: func(_ *config.CacheOptions, _ string) (v1.Image, error) { + return fakes.FakeImage{}, ExpiredErr{} + }, + TarWriter: tarBuf, + ManifestWriter: manifestBuf, + } + + opts := &config.WarmerOptions{} + + _, err := cw.Warm(image, opts) + if !IsAlreadyCached(err) { + t.Errorf("expected error to be already cached err but was %v", err) + t.FailNow() + } + + if len(tarBuf.Bytes()) != 0 { + t.Errorf("expected nothing to be written") + } +} diff --git a/pkg/fakes/image.go b/pkg/fakes/image.go new file mode 100644 index 0000000000..af07adf3d4 --- /dev/null +++ b/pkg/fakes/image.go @@ -0,0 +1,60 @@ +/* +Copyright 2019 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fakes + +import ( + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +type FakeImage struct { + Hash v1.Hash +} + +func (f FakeImage) Layers() ([]v1.Layer, error) { + return nil, nil +} +func (f FakeImage) MediaType() (types.MediaType, error) { + return "", nil +} +func (f FakeImage) Size() (int64, error) { + return 0, nil +} +func (f FakeImage) ConfigName() (v1.Hash, error) { + return v1.Hash{}, nil +} +func (f FakeImage) ConfigFile() (*v1.ConfigFile, error) { + return &v1.ConfigFile{}, nil +} +func (f FakeImage) RawConfigFile() ([]byte, error) { + return []byte{}, nil +} +func (f FakeImage) Digest() (v1.Hash, error) { + return f.Hash, nil +} +func (f FakeImage) Manifest() (*v1.Manifest, error) { + return &v1.Manifest{}, nil +} +func (f FakeImage) RawManifest() ([]byte, error) { + return []byte{}, nil +} +func (f FakeImage) LayerByDigest(v1.Hash) (v1.Layer, error) { + return nil, nil +} +func (f FakeImage) LayerByDiffID(v1.Hash) (v1.Layer, error) { + return nil, nil +}