feat: add Google Cloud Storage cache adapter

atlas/cache_gcs.go

@@ -0,0 +1,12 @@
+// +build !noGCSCache
+
+package atlas
+
+// The point of this file is to load and register the GCS cache backend.
+// the GCS cache can be excluded during the build with the `noGCSCache` build flag
+// for example from the cmd/tegola directory:
+//
+// go build -tags 'noGCSCache'
+import (
+	_ "github.com/go-spatial/tegola/cache/gcs"
+)

atlas/cache_init_test.go

@@ -10,7 +10,7 @@ import (
 
 func TestCheckCacheTypes(t *testing.T) {
 	c := cache.Registered()
-	exp := []string{"azblob", "file", "redis", "s3"}
+	exp := []string{"azblob", "file", "redis", "s3", "gcs"}
 	sort.Strings(exp)
 	if !reflect.DeepEqual(c, exp) {
 		t.Errorf("registered cachés, expected %v got %v", exp, c)

cache/gcs/README.md

@@ -0,0 +1,16 @@
+# Google Cloud Storage Cache
+
+GCS cache is an abstraction on top of Google Cloud Storage (GCS) which implements the tegola cache interface. To use it, you need to configure cache as the example below:
+
+```toml
+[cache]
+# required
+type="gcs"
+bucket="your_bucket_name"   # Bucket is the name of the GCS bucket to operate on
+
+# optional
+basepath="tegola"           # Basepath is a path prefix added to all cache operations inside of the GCS bucket
+max_zoom=8                  # MaxZoom determines the max zoom the cache to persist.
+```
+
+The credentials (service account and project_id) are handled by the `GOOGLE_APPLICATION_CREDENTIALS` environment variable.

cache/gcs/gcs.go

@@ -0,0 +1,186 @@
+package gcs
+
+import (
+	"context"
+	"errors"
+	"io"
+	"path/filepath"
+
+	"github.com/go-spatial/tegola"
+	"github.com/go-spatial/tegola/cache"
+	"github.com/go-spatial/tegola/dict"
+	"github.com/go-spatial/tegola/internal/log"
+
+	"cloud.google.com/go/storage"
+)
+
+const CacheType = "gcs"
+
+var (
+	ErrMissingBucket = errors.New("cache_gcs: missing required param 'bucket'")
+)
+
+const (
+	// required
+	ConfigKeyBucketName = "bucket"
+
+	// optional
+	ConfigKeyBasepath = "basepath"
+	ConfigKeyMaxZoom  = "max_zoom"
+)
+
+// testData is used during New() to confirm the ability to write, read and purge the cache
+var testData = []byte{0x1f, 0x8b, 0x8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x2a, 0xce, 0xcc, 0x49, 0x2c, 0x6, 0x4, 0x0, 0x0, 0xff, 0xff, 0xaf, 0x9d, 0x59, 0xca, 0x5, 0x0, 0x0, 0x0}
+
+func init() {
+	cache.Register(CacheType, New)
+}
+
+func New(config dict.Dicter) (cache.Interface, error) {
+	var err error
+
+	gcsCache := GCSCache{}
+
+	gcsCache.BucketName, err = config.String(ConfigKeyBucketName, nil)
+	if err != nil {
+		return nil, ErrMissingBucket
+	}
+
+	gcsCache.Basepath, err = config.String(ConfigKeyBasepath, nil)
+	defaultMaxZoom := uint(tegola.MaxZ)
+	gcsCache.MaxZoom, err = config.Uint(ConfigKeyMaxZoom, &defaultMaxZoom)
+
+	gcsCache.Ctx = context.Background()
+	client, err := storage.NewClient(gcsCache.Ctx)
+	if err != nil {
+		log.Fatal(err)
+	}
+	gcsCache.Client = client
+	gcsCache.Bucket = client.Bucket(gcsCache.BucketName)
+
+	// in order to confirm we have the correct permissions on the bucket create a small file
+	// and test a PUT, GET and DELETE to the bucket
+	key := cache.Key{
+		MapName:   "tegola-test-map",
+		LayerName: "test-layer",
+		Z:         0,
+		X:         0,
+		Y:         0,
+	}
+
+	// write gzip encoded test file
+	if err := gcsCache.Set(&key, testData); err != nil {
+		e := cache.ErrSettingToCache{
+			CacheType: CacheType,
+			Err:       err,
+		}
+
+		return nil, e
+	}
+
+	// read the test file
+	_, hit, err := gcsCache.Get(&key)
+	if err != nil {
+		e := cache.ErrGettingFromCache{
+			CacheType: CacheType,
+			Err:       err,
+		}
+
+		return nil, e
+	}
+	if !hit {
+		// return an error?
+	}
+
+	// purge the test file
+	if err := gcsCache.Purge(&key); err != nil {
+		e := cache.ErrPurgingCache{
+			CacheType: CacheType,
+			Err:       err,
+		}
+
+		return nil, e
+	}
+
+	return &gcsCache, nil
+}
+
+type GCSCache struct {
+
+	// Context
+	Ctx context.Context
+
+	// Bucket is the name of the GCS bucket to operate on
+	BucketName string
+
+	// Basepath is a path prefix added to all cache operations inside of the GCS bucket
+	// helpful so a bucket does not need to be dedicated to only this cache
+	Basepath string
+
+	// MaxZoom determines the max zoom the cache to persist. Beyond this
+	// zoom, cache Set() calls will be ignored. This is useful if the cache
+	// should not be leveraged for higher zooms when data changes often.
+	MaxZoom uint
+
+	// client holds a reference to the storage client. it's expected the client
+	// has an active session and read, write, delete permissions have been checked
+	Client *storage.Client
+
+	// bucket holds a reference to the bucket handle.
+	Bucket *storage.BucketHandle
+}
+
+func (gcsCache *GCSCache) Get(key *cache.Key) ([]byte, bool, error) {
+	k := filepath.Join(gcsCache.Basepath, key.String())
+	obj := gcsCache.Bucket.Object(k)
+
+	r, err := obj.NewReader(gcsCache.Ctx)
+	if err != nil {
+		return nil, false, nil
+	}
+	defer r.Close()
+
+	val, err := io.ReadAll(r)
+	if err != nil {
+		return nil, false, err
+	}
+
+	log.Infof("GET %s: %d bytes\n", k, len(val))
+
+	return val, true, nil
+}
+
+func (gcsCache *GCSCache) Set(key *cache.Key, val []byte) error {
+	k := filepath.Join(gcsCache.Basepath, key.String())
+	obj := gcsCache.Bucket.Object(k)
+
+	// check for maxzoom
+	if key.Z > gcsCache.MaxZoom {
+		return nil
+	}
+
+	w := obj.NewWriter(gcsCache.Ctx)
+	if _, err := w.Write(val); err != nil {
+		return err
+	}
+	if err := w.Close(); err != nil {
+		return err
+	}
+
+	log.Infof("SET %s: %d bytes\n", k, len(val))
+
+	return nil
+}
+
+func (gcsCache *GCSCache) Purge(key *cache.Key) error {
+	k := filepath.Join(gcsCache.Basepath, key.String())
+	obj := gcsCache.Bucket.Object(k)
+
+	if err := obj.Delete(gcsCache.Ctx); err != nil {
+		return err
+	}
+
+	log.Infof("PURGE %s\n", k)
+
+	return nil
+}

go.mod

@@ -3,6 +3,7 @@ module github.com/go-spatial/tegola
 go 1.17
 
 require (
+	cloud.google.com/go/storage v1.28.0
 	github.com/Azure/azure-storage-blob-go v0.0.0-20180706173141-f0a732ea9441
 	github.com/BurntSushi/toml v0.4.1
 	github.com/ajstarks/svgo v0.0.0-20170507103333-2489f1e6d405
@@ -14,7 +15,7 @@ require (
 	github.com/go-spatial/cobra v0.0.3-0.20181105183926-68194e4fbcc6
 	github.com/go-spatial/geom v0.0.0-20190821234737-802ab2533ab4
 	github.com/go-test/deep v0.0.0-20170429201529-f49763a6ea0a
-	github.com/golang/protobuf v1.4.3
+	github.com/golang/protobuf v1.5.2
 	github.com/jackc/pgproto3/v2 v2.2.0
 	github.com/jackc/pgtype v1.9.1
 	github.com/jackc/pgx/v4 v4.14.1
@@ -28,13 +29,21 @@ require (
 )
 
 require (
+	cloud.google.com/go v0.104.0 // indirect
+	cloud.google.com/go/compute v1.12.1 // indirect
+	cloud.google.com/go/compute/metadata v0.2.1 // indirect
+	cloud.google.com/go/iam v0.5.0 // indirect
 	github.com/Azure/azure-pipeline-go v0.0.0-20180607212504-7571e8eb0876 // indirect
 	github.com/arolek/p v0.0.0-20191103215535-df3c295ed582 // indirect
 	github.com/aws/aws-lambda-go v1.13.3 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.1.1 // indirect
 	github.com/gofrs/uuid v4.0.0+incompatible // indirect
-	github.com/google/uuid v1.0.0 // indirect
+	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect
+	github.com/googleapis/gax-go/v2 v2.6.0 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
 	github.com/jackc/pgconn v1.10.1 // indirect
@@ -48,11 +57,19 @@ require (
 	github.com/prometheus/common v0.15.0 // indirect
 	github.com/prometheus/procfs v0.2.0 // indirect
 	github.com/spf13/pflag v1.0.1 // indirect
+	go.opencensus.io v0.23.0 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
 	golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 // indirect
-	golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 // indirect
-	golang.org/x/text v0.3.6 // indirect
-	golang.org/x/tools v0.1.5 // indirect
-	google.golang.org/protobuf v1.23.0 // indirect
+	golang.org/x/net v0.0.0-20221014081412-f15817d10f9b // indirect
+	golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 // indirect
+	golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect
+	golang.org/x/text v0.4.0 // indirect
+	golang.org/x/tools v0.1.12 // indirect
+	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
+	google.golang.org/api v0.102.0 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto v0.0.0-20221024183307-1bc688fe9f3e // indirect
+	google.golang.org/grpc v1.50.1 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
 )