From 183b80d124f1f4c215ce38f934a5d1fb54253285 Mon Sep 17 00:00:00 2001 From: Seth Vargo Date: Thu, 29 Oct 2020 19:51:02 -0400 Subject: [PATCH] Add support for AR, allow running locally --- Dockerfile | 11 ++- README.md | 26 +++++-- cache.go | 86 ---------------------- cloudbuild/cloudbuild.yaml | 38 +++++++++- cmd/gcr-cleaner-cli/main.go | 90 +++++++++++++++++++++++ main.go => cmd/gcr-cleaner-server/main.go | 7 +- pkg/gcrcleaner/cache.go | 72 ++++++++++++++++++ pkg/gcrcleaner/cleaner.go | 1 + 8 files changed, 225 insertions(+), 106 deletions(-) delete mode 100644 cache.go create mode 100644 cmd/gcr-cleaner-cli/main.go rename main.go => cmd/gcr-cleaner-server/main.go (91%) diff --git a/Dockerfile b/Dockerfile index 689ebbe..34b45ae 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM golang:1.14 AS builder +FROM golang:1.15 AS builder + +ARG SERVICE RUN apt-get -qq update && apt-get -yqq install upx @@ -28,14 +30,11 @@ RUN go build \ -a \ -trimpath \ -ldflags "-s -w -extldflags '-static'" \ - -installsuffix cgo \ - -tags netgo \ - -mod vendor \ + -tags 'osusergo netgo static_build' \ -o /bin/gcrcleaner \ - . + ./cmd/${SERVICE} RUN strip /bin/gcrcleaner - RUN upx -q -9 /bin/gcrcleaner diff --git a/README.md b/README.md index b54ceae..31d7f2d 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,9 @@ # GCR Cleaner -GCR Cleaner deletes untagged images in Google Container Registry. This can help -reduce costs and keep your container images list in order. +GCR Cleaner deletes untagged images in Google Cloud [Container +Registry][container-registry] or Google Cloud [Artifact +Registry][artifact-registry]. This can help reduce costs and keep your container +images list in order. GCR Cleaner is designed to be deployed as a [Cloud Run][cloud-run] service and invoked periodically via [Cloud Scheduler][cloud-scheduler]. @@ -59,7 +61,7 @@ invoked periodically via [Cloud Scheduler][cloud-scheduler]. --project ${PROJECT_ID} \ --platform "managed" \ --service-account "gcr-cleaner@${PROJECT_ID}.iam.gserviceaccount.com" \ - --image "gcr.io/gcr-cleaner/gcr-cleaner" \ + --image "us-docker.pkg.dev/gcr-cleaner/gcr-cleaner/gcr-cleaner" \ --region "us-central1" \ --timeout "60s" ``` @@ -105,8 +107,9 @@ invoked periodically via [Cloud Scheduler][cloud-scheduler]. ```sh # Replace this with the full name of the repository for which you - # want to cleanup old references. + # want to cleanup old references, for example: export REPO="gcr.io/${PROJECT_ID}/my-image" + export REPO="us-docker-pkg.dev/${PROJECT_ID}/my-repo/my-image" ``` ```sh @@ -157,6 +160,14 @@ The payload is expected to be JSON with the following fields: - `keep` - If an integer is provided, it will always keep that minimum number of images. Note that it will not consider images inside the `grace` duration. +## Running locally + +In addition to the server, you can also run GCR Cleaner locally for one-off tasks using `cmd/gcr-cleaner-cli`: + +```text +docker run -it us-docker.pkg.dev/gcr-cleaner/gcr-cleaner/gcr-cleaner-cli +``` + ## I just want the container! You can build the container yourself using the included Dockerfile. @@ -170,7 +181,6 @@ europe-docker.pkg.dev/gcr-cleaner/gcr-cleaner/gcr-cleaner us-docker.pkg.dev/gcr-cleaner/gcr-cleaner/gcr-cleaner ``` - ## FAQ **How do I clean up multiple Google Container Registry repos at once?** @@ -193,11 +203,13 @@ service. This library is licensed under Apache 2.0. Full license text is available in [LICENSE](https://github.com/sethvargo/gcr-cleaner/tree/master/LICENSE). +[artifact-registry]: https://cloud.google.com/artifact-registry [cloud-build]: https://cloud.google.com/build/ [cloud-pubsub]: https://cloud.google.com/pubsub/ [cloud-run]: https://cloud.google.com/run/ [cloud-scheduler]: https://cloud.google.com/scheduler/ -[cloud-shell]: https://cloud.google.com/shell [cloud-sdk]: https://cloud.google.com/sdk -[gcrgc.sh]: https://gist.github.com/ahmetb/7ce6d741bd5baa194a3fac6b1fec8bb7 +[cloud-shell]: https://cloud.google.com/shell +[container-registry]: https://cloud.google.com/container-registry [gcr-cleaner-godoc]: https://godoc.org/github.com/sethvargo/gcr-cleaner/pkg/gcrcleaner +[gcrgc.sh]: https://gist.github.com/ahmetb/7ce6d741bd5baa194a3fac6b1fec8bb7 diff --git a/cache.go b/cache.go deleted file mode 100644 index c660601..0000000 --- a/cache.go +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright 2019 The GCR Cleaner Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "sync" - "time" -) - -// timerCache is a Cache implementation that caches items for a configurable -// period of time. -type timerCache struct { - lock sync.RWMutex - data map[string]struct{} - lifetime time.Duration - - stopCh chan struct{} - stopped bool -} - -func newTimerCache(lifetime time.Duration) *timerCache { - return &timerCache{ - data: make(map[string]struct{}), - lifetime: lifetime, - stopCh: make(chan struct{}), - } -} - -// Stop stops the cache. -func (c *timerCache) Stop() { - c.lock.Lock() - if !c.stopped { - close(c.stopCh) - c.stopped = true - } - c.lock.Unlock() -} - -// Insert adds the item to the cache. If the item already existed in the cache, -// this function returns false. -func (c *timerCache) Insert(s string) bool { - // Read only - c.lock.RLock() - if _, ok := c.data[s]; ok { - c.lock.RUnlock() - return true - } - c.lock.RUnlock() - - // Full insert - c.lock.Lock() - if _, ok := c.data[s]; ok { - c.lock.Unlock() - return true - } - - c.data[s] = struct{}{} - c.lock.Unlock() - - // Start a timeout to delete the item from the cache. - go c.timeout(s) - - return false -} - -func (c *timerCache) timeout(s string) { - select { - case <-time.After(c.lifetime): - c.lock.Lock() - delete(c.data, s) - c.lock.Unlock() - case <-c.stopCh: - } -} diff --git a/cloudbuild/cloudbuild.yaml b/cloudbuild/cloudbuild.yaml index 91c06bc..5e919c0 100644 --- a/cloudbuild/cloudbuild.yaml +++ b/cloudbuild/cloudbuild.yaml @@ -13,19 +13,48 @@ # limitations under the License. steps: -- id: 'build' - name: 'docker:18' +- id: 'build-cli' + name: 'docker:19' args: [ 'build', + '--build-arg', 'SERVICE=gcr-cleaner-cli', + '--tag', 'gcr.io/${PROJECT_ID}/gcr-cleaner-cli', + '--tag', 'asia-docker.pkg.dev/${PROJECT_ID}/gcr-cleaner/gcr-cleaner-cli', + '--tag', 'europe-docker.pkg.dev/${PROJECT_ID}/gcr-cleaner/gcr-cleaner-cli', + '--tag', 'us-docker.pkg.dev/${PROJECT_ID}/gcr-cleaner/gcr-cleaner-cli', + '.' + ] + waitFor: ['-'] + +- id: 'build-server' + name: 'docker:19' + args: [ + 'build', + '--build-arg', 'SERVICE=gcr-cleaner-server', '--tag', 'gcr.io/${PROJECT_ID}/gcr-cleaner', '--tag', 'asia-docker.pkg.dev/${PROJECT_ID}/gcr-cleaner/gcr-cleaner', '--tag', 'europe-docker.pkg.dev/${PROJECT_ID}/gcr-cleaner/gcr-cleaner', '--tag', 'us-docker.pkg.dev/${PROJECT_ID}/gcr-cleaner/gcr-cleaner', '.' ] + waitFor: ['-'] + +- id: 'push-cli' + name: 'docker:19' + entrypoint: '/bin/sh' + args: + - '-euo' + - 'pipefail' + - '-c' + - |- + docker push gcr.io/${PROJECT_ID}/gcr-cleaner-cli + docker push asia-docker.pkg.dev/${PROJECT_ID}/gcr-cleaner/gcr-cleaner-cli + docker push europe-docker.pkg.dev/${PROJECT_ID}/gcr-cleaner/gcr-cleaner-cli + docker push us-docker.pkg.dev/${PROJECT_ID}/gcr-cleaner/gcr-cleaner-cli + waitFor: ['build-cli'] -- id: 'push' - name: 'docker:18' +- id: 'push-server' + name: 'docker:19' entrypoint: '/bin/sh' args: - '-euo' @@ -36,3 +65,4 @@ steps: docker push asia-docker.pkg.dev/${PROJECT_ID}/gcr-cleaner/gcr-cleaner docker push europe-docker.pkg.dev/${PROJECT_ID}/gcr-cleaner/gcr-cleaner docker push us-docker.pkg.dev/${PROJECT_ID}/gcr-cleaner/gcr-cleaner + waitFor: ['build-server'] diff --git a/cmd/gcr-cleaner-cli/main.go b/cmd/gcr-cleaner-cli/main.go new file mode 100644 index 0000000..e5acc9d --- /dev/null +++ b/cmd/gcr-cleaner-cli/main.go @@ -0,0 +1,90 @@ +// Copyright 2019 The GCR Cleaner Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package main defines the CLI interface for GCR Cleaner. +package main + +import ( + "flag" + "fmt" + "os" + "runtime" + "time" + + gcrauthn "github.com/google/go-containerregistry/pkg/authn" + gcrgoogle "github.com/google/go-containerregistry/pkg/v1/google" + "github.com/sethvargo/gcr-cleaner/pkg/gcrcleaner" +) + +var ( + stdout = os.Stdout + stderr = os.Stderr + + tokenPtr = flag.String("token", os.Getenv("GCRCLEANER_TOKEN"), "Authentication token") + repoPtr = flag.String("repo", "", "Repository name") + gracePtr = flag.Duration("grace", 0, "Grace period") + allowTaggedPtr = flag.Bool("allow-tagged", false, "Delete tagged images") + keepPtr = flag.Int("keep", 0, "Minimum to keep") +) + +func main() { + flag.Parse() + + if err := realMain(); err != nil { + fmt.Fprintf(stderr, "%s\n", err) + os.Exit(1) + } +} + +func realMain() error { + if *repoPtr == "" { + return fmt.Errorf("missing -repo") + } + + // Try to find the "best" authentication. + var auther gcrauthn.Authenticator + if *tokenPtr != "" { + auther = &gcrauthn.Bearer{Token: *tokenPtr} + } else { + var err error + auther, err = gcrgoogle.NewEnvAuthenticator() + if err != nil { + return fmt.Errorf("failed to setup auther: %w", err) + } + } + + concurrency := runtime.NumCPU() + cleaner, err := gcrcleaner.NewCleaner(auther, concurrency) + if err != nil { + return fmt.Errorf("failed to create cleaner: %w", err) + } + + // Convert duration to a negative value, since we're about to "add" it to the + // since time. + sub := time.Duration(*gracePtr) + if *gracePtr > 0 { + sub = sub * -1 + } + since := time.Now().UTC().Add(sub) + + // Do the deletion. + fmt.Fprintf(stdout, "%s: deleting refs since %s\n", *repoPtr, since) + deleted, err := cleaner.Clean(*repoPtr, since, *allowTaggedPtr, *keepPtr) + if err != nil { + return err + } + fmt.Fprintf(stdout, "%s: successfully deleted %d refs", *repoPtr, len(deleted)) + + return nil +} diff --git a/main.go b/cmd/gcr-cleaner-server/main.go similarity index 91% rename from main.go rename to cmd/gcr-cleaner-server/main.go index 9ce55cb..b2d148b 100644 --- a/main.go +++ b/cmd/gcr-cleaner-server/main.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Package main defines the server interface for GCR Cleaner. package main import ( @@ -36,7 +37,7 @@ func main() { if port == "" { port = "8080" } - addr := "0.0.0.0:" + port + addr := ":" + port var auther gcrauthn.Authenticator if token := os.Getenv("GCRCLEANER_TOKEN"); token != "" { @@ -60,7 +61,7 @@ func main() { log.Fatalf("failed to create server: %s", err) } - cache := newTimerCache(30 * time.Minute) + cache := gcrcleaner.NewTimerCache(5 * time.Minute) mux := http.NewServeMux() mux.Handle("/http", cleanerServer.HTTPHandler()) @@ -73,7 +74,7 @@ func main() { go func() { log.Printf("server is listening on %s\n", port) - if err := server.ListenAndServe(); err != http.ErrServerClosed { + if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { log.Fatalf("server exited: %s", err) } }() diff --git a/pkg/gcrcleaner/cache.go b/pkg/gcrcleaner/cache.go index e95895b..52f559e 100644 --- a/pkg/gcrcleaner/cache.go +++ b/pkg/gcrcleaner/cache.go @@ -14,6 +14,11 @@ package gcrcleaner +import ( + "sync" + "time" +) + // Cache is an interface used by the PubSub() function to prevent duplicate // messages from being processed. type Cache interface { @@ -25,3 +30,70 @@ type Cache interface { // additionally processing. Stop() } + +// timerCache is a Cache implementation that caches items for a configurable +// period of time. +type timerCache struct { + lock sync.RWMutex + data map[string]struct{} + lifetime time.Duration + + stopCh chan struct{} + stopped bool +} + +// NewTimerCache creates a new timer-based cache. +func NewTimerCache(lifetime time.Duration) *timerCache { + return &timerCache{ + data: make(map[string]struct{}), + lifetime: lifetime, + stopCh: make(chan struct{}), + } +} + +// Stop stops the cache. +func (c *timerCache) Stop() { + c.lock.Lock() + if !c.stopped { + close(c.stopCh) + c.stopped = true + } + c.lock.Unlock() +} + +// Insert adds the item to the cache. If the item already existed in the cache, +// this function returns false. +func (c *timerCache) Insert(s string) bool { + // Read only + c.lock.RLock() + if _, ok := c.data[s]; ok { + c.lock.RUnlock() + return true + } + c.lock.RUnlock() + + // Full insert + c.lock.Lock() + if _, ok := c.data[s]; ok { + c.lock.Unlock() + return true + } + + c.data[s] = struct{}{} + c.lock.Unlock() + + // Start a timeout to delete the item from the cache. + go c.timeout(s) + + return false +} + +func (c *timerCache) timeout(s string) { + select { + case <-time.After(c.lifetime): + c.lock.Lock() + delete(c.data, s) + c.lock.Unlock() + case <-c.stopCh: + } +} diff --git a/pkg/gcrcleaner/cleaner.go b/pkg/gcrcleaner/cleaner.go index 7fdf9f1..8b209b2 100644 --- a/pkg/gcrcleaner/cleaner.go +++ b/pkg/gcrcleaner/cleaner.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Package gcrcleaner cleans up stale images from a container registry. package gcrcleaner import (