Run local KES for S3 integration tests (cortexproject#4547)

Instead of calling out to play.min.io, run the KES service locally in a
container. Generate its key and cert instead of downloading them from
GitHub.

By depending on fewer third-party services we reduce test failures.

Extract function to generate PKI certificates.
Update to more recent Minio image; not using the very latest KES because
it removed `--root` which is useful.
Re-enable the SSE tests (see cortexproject#4545)

Signed-off-by: Bryan Boreham <bjboreham@gmail.com>

Author: bboreham

integration/certs.go

@@ -0,0 +1,51 @@
+//go:build requires_docker
+// +build requires_docker
+
+package integration
+
+import (
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"os"
+	"path/filepath"
+
+	"github.com/cortexproject/cortex/integration/ca"
+)
+
+func writeCerts(dir string, dnsNames ...string) error {
+	// set the ca
+	cert := ca.New("Test")
+
+	// Ensure the entire path of directories exist.
+	if err := os.MkdirAll(filepath.Join(dir, "certs"), os.ModePerm); err != nil {
+		return err
+	}
+
+	if err := cert.WriteCACertificate(filepath.Join(dir, caCertFile)); err != nil {
+		return err
+	}
+
+	// server certificate
+	if err := cert.WriteCertificate(
+		&x509.Certificate{
+			Subject:     pkix.Name{CommonName: "client"},
+			ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
+		},
+		filepath.Join(dir, clientCertFile),
+		filepath.Join(dir, clientKeyFile),
+	); err != nil {
+		return err
+	}
+	if err := cert.WriteCertificate(
+		&x509.Certificate{
+			Subject:     pkix.Name{CommonName: "server"},
+			DNSNames:    dnsNames,
+			ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		},
+		filepath.Join(dir, serverCertFile),
+		filepath.Join(dir, serverKeyFile),
+	); err != nil {
+		return err
+	}
+	return nil
+}

integration/e2e/db/db.go

@@ -2,6 +2,7 @@ package e2edb
 
 import (
 	"fmt"
+	"path/filepath"
 	"strings"
 
 	"github.com/cortexproject/cortex/integration/e2e"
@@ -15,11 +16,23 @@ const (
 
 // NewMinio returns minio server, used as a local replacement for S3.
 func NewMinio(port int, bktNames ...string) *e2e.HTTPService {
-	minioKESGithubContent := "https://raw.githubusercontent.com/minio/kes/master"
-	commands := []string{
-		fmt.Sprintf("curl -sSL --tlsv1.2 -O '%s/root.key' -O '%s/root.cert'", minioKESGithubContent, minioKESGithubContent),
+	return newMinio(port, map[string]string{}, bktNames...)
+}
+
+// NewMinioWithKES returns minio server, configured to talk to a KES service.
+func NewMinioWithKES(port int, kesEndpoint, rootKeyFile, rootCertFile, caCertFile string, bktNames ...string) *e2e.HTTPService {
+	kesEnvVars := map[string]string{
+		"MINIO_KMS_KES_ENDPOINT":  kesEndpoint,
+		"MINIO_KMS_KES_KEY_FILE":  filepath.Join(e2e.ContainerSharedDir, rootKeyFile),
+		"MINIO_KMS_KES_CERT_FILE": filepath.Join(e2e.ContainerSharedDir, rootCertFile),
+		"MINIO_KMS_KES_CAPATH":    filepath.Join(e2e.ContainerSharedDir, caCertFile),
+		"MINIO_KMS_KES_KEY_NAME":  "my-minio-key",
 	}
+	return newMinio(port, kesEnvVars, bktNames...)
+}
 
+func newMinio(port int, envVars map[string]string, bktNames ...string) *e2e.HTTPService {
+	commands := []string{}
 	for _, bkt := range bktNames {
 		commands = append(commands, fmt.Sprintf("mkdir -p /data/%s", bkt))
 	}
@@ -33,17 +46,27 @@ func NewMinio(port int, bktNames ...string) *e2e.HTTPService {
 		e2e.NewHTTPReadinessProbe(port, "/minio/health/ready", 200, 200),
 		port,
 	)
-	m.SetEnvVars(map[string]string{
-		"MINIO_ACCESS_KEY": MinioAccessKey,
-		"MINIO_SECRET_KEY": MinioSecretKey,
-		"MINIO_BROWSER":    "off",
-		"ENABLE_HTTPS":     "0",
-		// https://docs.min.io/docs/minio-kms-quickstart-guide.html
-		"MINIO_KMS_KES_ENDPOINT":  "https://play.min.io:7373",
-		"MINIO_KMS_KES_KEY_FILE":  "root.key",
-		"MINIO_KMS_KES_CERT_FILE": "root.cert",
-		"MINIO_KMS_KES_KEY_NAME":  "my-minio-key",
-	})
+	envVars["MINIO_ACCESS_KEY"] = MinioAccessKey
+	envVars["MINIO_SECRET_KEY"] = MinioSecretKey
+	envVars["MINIO_BROWSER"] = "off"
+	envVars["ENABLE_HTTPS"] = "0"
+	m.SetEnvVars(envVars)
+	return m
+}
+
+// NewKES returns KES server, used as a local key management store
+func NewKES(port int, serverKeyFile, serverCertFile, rootCertFile string) *e2e.HTTPService {
+	// Run this as a shell command, so sub-shell can evaluate 'identity' of root user.
+	command := fmt.Sprintf("/kes server --addr 0.0.0.0:%d --key=%s --cert=%s --root=$(/kes tool identity of %s) --auth=off --quiet",
+		port, filepath.Join(e2e.ContainerSharedDir, serverKeyFile), filepath.Join(e2e.ContainerSharedDir, serverCertFile), filepath.Join(e2e.ContainerSharedDir, rootCertFile))
+
+	m := e2e.NewHTTPService(
+		"kes",
+		images.KES,
+		e2e.NewCommandWithoutEntrypoint("sh", "-c", command),
+		nil, // KES only supports https calls - TODO make Scenario able to call https or poll plain TCP socket.
+		port,
+	)
 	return m
 }
 

integration/e2e/images/images.go

@@ -7,7 +7,8 @@ package images
 
 var (
 	Memcached        = "memcached:1.6.1"
-	Minio            = "minio/minio:RELEASE.2019-12-30T05-45-39Z"
+	Minio            = "minio/minio:RELEASE.2021-10-13T00-23-17Z"
+	KES              = "minio/kes:v0.17.1"
 	Consul           = "consul:1.8.4"
 	ETCD             = "gcr.io/etcd-development/etcd:v3.4.7"
 	DynamoDB         = "amazon/dynamodb-local:1.11.477"

integration/s3_storage_client_test.go

@@ -1,3 +1,4 @@
+//go:build requires_docker
 // +build requires_docker
 
 package integration
@@ -24,7 +25,13 @@ func TestS3Client(t *testing.T) {
 	defer s.Close()
 
 	// Start dependencies.
-	minio := e2edb.NewMinio(9000, bucketName)
+	// We use KES to emulate a Key Management Store for use with Minio
+	kesDNSName := networkName + "-kes"
+	require.NoError(t, writeCerts(s.SharedDir(), kesDNSName))
+	// Start dependencies.
+	kes := e2edb.NewKES(7373, serverKeyFile, serverCertFile, clientCertFile)
+	require.NoError(t, s.Start(kes)) // TODO: wait for it to be ready, but currently there is no way to probe.
+	minio := e2edb.NewMinioWithKES(9000, "https://"+kesDNSName+":7373", clientKeyFile, clientCertFile, caCertFile, bucketName)
 	require.NoError(t, s.StartAndWaitReady(minio))
 
 	tests := []struct {
@@ -94,11 +101,6 @@ func TestS3Client(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			switch tt.name {
-			case "config-with-deprecated-sse", "config-with-sse-s3":
-				t.Skip("TODO: Issue #4543")
-			}
-
 			client, err := s3.NewS3ObjectClient(tt.cfg)
 
 			require.NoError(t, err)