Add Secrets Manager KV store (TykTechnologies#6563)

cli/linter/schema.json

@@ -1135,6 +1135,20 @@
             }
           }
         },
+        "secrets_manager": {
+          "type": ["object", "null"],
+          "properties": {
+            "access_key_id": {
+              "type": "string"
+            },
+            "secret_access_key": {
+              "type": "string"
+            },
+            "region": {
+              "type": "string"
+            }
+          }
+        },
         "vault": {
           "type": ["object", "null"],
           "properties": {

config/config.go

@@ -1069,8 +1069,9 @@ type Config struct {
 	// This section enables the use of the KV capabilities to substitute configuration values.
 	// See more details https://tyk.io/docs/tyk-configuration-reference/kv-store/
 	KV struct {
-		Consul ConsulConfig `json:"consul"`
-		Vault  VaultConfig  `json:"vault"`
+		Consul         ConsulConfig         `json:"consul"`
+		SecretsManager SecretsManagerConfig `json:"secrets_manager"`
+		Vault          VaultConfig          `json:"vault"`
 	} `json:"kv"`
 
 	// Secrets are key-value pairs that can be accessed in the dashboard via "secrets://"
@@ -1213,6 +1214,18 @@ type ConsulConfig struct {
 	} `json:"tls_config"`
 }
 
+// SecretsManagerConfig is used to configure the AWS Secrets Manager client.
+type SecretsManagerConfig struct {
+	// AccessKeyID is the AWS access key ID.
+	AccessKeyID string `json:"access_key_id"`
+
+	// SecretAccessKey is the AWS secret access key.
+	SecretAccessKey string `json:"secret_access_key"`
+
+	// Region is the AWS region.
+	Region string `json:"region"`
+}
+
 // GetEventTriggers returns event triggers. There was a typo in the json tag.
 // To maintain backward compatibility, this solution is chosen.
 func (c Config) GetEventTriggers() map[apidef.TykEvent][]TykEventHandler {

config/config_test.go

@@ -353,3 +353,23 @@ func TestPortsWhiteListDecoder(t *testing.T) {
 	assert.Contains(t, tlsWhiteList.Ports, 6000, "tls should have 6000 port")
 	assert.Contains(t, tlsWhiteList.Ports, 6015, "tls should have 6015 port")
 }
+
+func TestConfigKVSecretsManager(t *testing.T) {
+	var c Config
+
+	err := os.Setenv("TYK_GW_KV_SECRETSMANAGER_ACCESSKEYID", "AKIAIOSFODNN7EXAMPLE")
+	assert.NoError(t, err)
+
+	err = os.Setenv("TYK_GW_KV_SECRETSMANAGER_SECRETACCESSKEY", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
+	assert.NoError(t, err)
+
+	err = os.Setenv("TYK_GW_KV_SECRETSMANAGER_REGION", "us-east-1")
+	assert.NoError(t, err)
+
+	err = envconfig.Process("TYK_GW", &c)
+	assert.NoError(t, err)
+
+	assert.Equal(t, "AKIAIOSFODNN7EXAMPLE", c.KV.SecretsManager.AccessKeyID)
+	assert.Equal(t, "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", c.KV.SecretsManager.SecretAccessKey)
+	assert.Equal(t, "us-east-1", c.KV.SecretsManager.Region)
+}

gateway/api_definition.go

@@ -34,7 +34,7 @@ import (
 
 	"github.com/cenk/backoff"
 
-	sprig "github.com/Masterminds/sprig/v3"
+	"github.com/Masterminds/sprig/v3"
 
 	"github.com/sirupsen/logrus"
 
@@ -558,12 +558,13 @@ func (a APIDefinitionLoader) FromDashboardService(endpoint string) ([]*APISpec,
 var envRegex = regexp.MustCompile(`env://([^"]+)`)
 
 const (
-	prefixEnv       = "env://"
-	prefixSecrets   = "secrets://"
-	prefixConsul    = "consul://"
-	prefixVault     = "vault://"
-	prefixKeys      = "tyk-apis"
-	vaultSecretPath = "secret/data/"
+	prefixEnv            = "env://"
+	prefixSecrets        = "secrets://"
+	prefixConsul         = "consul://"
+	prefixSecretsManager = "secretsmanager://"
+	prefixVault          = "vault://"
+	prefixKeys           = "tyk-apis"
+	vaultSecretPath      = "secret/data/"
 )
 
 func (a APIDefinitionLoader) replaceSecrets(in []byte) []byte {
@@ -597,6 +598,12 @@ func (a APIDefinitionLoader) replaceSecrets(in []byte) []byte {
 		}
 	}
 
+	if strings.Contains(input, prefixSecretsManager) {
+		if err := a.replaceSecretsManagerSecrets(&input); err != nil {
+			log.WithError(err).Error("Couldn't replace Secrets Manager secrets")
+		}
+	}
+
 	if strings.Contains(input, prefixVault) {
 		if err := a.replaceVaultSecrets(&input); err != nil {
 			log.WithError(err).Error("Couldn't replace vault secrets")
@@ -624,6 +631,29 @@ func (a APIDefinitionLoader) replaceConsulSecrets(input *string) error {
 	return nil
 }
 
+func (a APIDefinitionLoader) replaceSecretsManagerSecrets(input *string) error {
+	if err := a.Gw.setUpSecretsManager(); err != nil {
+		return err
+	}
+
+	value, err := a.Gw.secretsManagerKVStore.Get(prefixKeys)
+	if err != nil {
+		return err
+	}
+
+	jsonValue := make(map[string]interface{})
+	err = json.Unmarshal([]byte(value), &jsonValue)
+	if err != nil {
+		return fmt.Errorf("error unmarshalling secret string: %w", err)
+	}
+
+	for k, v := range jsonValue {
+		*input = strings.Replace(*input, prefixSecretsManager+k, fmt.Sprintf("%v", v), -1)
+	}
+
+	return nil
+}
+
 func (a APIDefinitionLoader) replaceVaultSecrets(input *string) error {
 	if err := a.Gw.setUpVault(); err != nil {
 		return err
@@ -651,7 +681,7 @@ func (a APIDefinitionLoader) replaceVaultSecrets(input *string) error {
 	return nil
 }
 
-// FromCloud will connect and download ApiDefintions from a Mongo DB instance.
+// FromRPC will connect and download API definitions.
 func (a APIDefinitionLoader) FromRPC(store RPCDataLoader, orgId string, gw *Gateway) ([]*APISpec, error) {
 	if rpc.IsEmergencyMode() {
 		return gw.LoadDefinitionsFromRPCBackup()
@@ -759,6 +789,7 @@ func (a APIDefinitionLoader) FromDir(dir string) []*APISpec {
 	// Grab json files from directory
 	paths, _ := filepath.Glob(filepath.Join(dir, "*.json"))
 	for _, path := range paths {
+		// TODO(#6563): Why aren't OAS definitions loaded here? This prevents secrets from being replaced in OAS definitions.
 		if strings.HasSuffix(path, "-oas.json") {
 			continue
 		}
@@ -773,6 +804,7 @@ func (a APIDefinitionLoader) FromDir(dir string) []*APISpec {
 	}
 	return specs
 }
+
 func (a APIDefinitionLoader) loadDefFromFilePath(filePath string) (*APISpec, error) {
 	log.Info("Loading API Specification from ", filePath)
 

gateway/api_definition_test.go

@@ -18,10 +18,12 @@ import (
 	"github.com/stretchr/testify/assert"
 
 	persistentmodel "github.com/TykTechnologies/storage/persistent/model"
+
 	"github.com/TykTechnologies/tyk/apidef"
 	"github.com/TykTechnologies/tyk/apidef/oas"
 	"github.com/TykTechnologies/tyk/config"
 	"github.com/TykTechnologies/tyk/rpc"
+	"github.com/TykTechnologies/tyk/storage/kv"
 	"github.com/TykTechnologies/tyk/test"
 	"github.com/TykTechnologies/tyk/user"
 )
@@ -1602,3 +1604,64 @@ func TestInternalEndpointMW_TT_11126(t *testing.T) {
 		{Path: "/headers", Code: http.StatusForbidden},
 	}...)
 }
+
+func TestAPIDefinitionReplaceSecrets(t *testing.T) {
+	ts := StartTest(func(globalConf *config.Config) {
+		globalConf.Secrets = map[string]string{
+			"jwt_signing_method": "secrets::jwt_signing_method::value",
+			"jwt_source":         "secrets::jwt_source::value",
+		}
+	})
+	defer ts.Close()
+
+	t.Setenv("JWT_SIGNING_METHOD", "env::jwt_signing_method::value")
+	t.Setenv("JWT_SOURCE", "env::jwt_source::value")
+
+	ts.Gw.secretsManagerKVStore = kv.NewSecretsManagerWithClient(kv.NewDummySecretsManagerClient(map[string]string{
+		"tyk-apis": "{" +
+			"\"jwt_signing_method\":\"secretsmanager::jwt_signing_method::value\"," +
+			"\"jwt_source\":\"secretsmanager::jwt_source::value\"" +
+			"}",
+	}))
+
+	tests := []struct {
+		name   string
+		scheme string
+		specs  []*APISpec
+	}{
+		{
+			name:   "Env",
+			scheme: "env",
+			specs: BuildAPI(func(spec *APISpec) {
+				spec.JWTSigningMethod = "env://JWT_SIGNING_METHOD"
+				spec.JWTSource = "env://JWT_SOURCE"
+			}),
+		},
+		{
+			name:   "Secrets",
+			scheme: "secrets",
+			specs: BuildAPI(func(spec *APISpec) {
+				spec.JWTSigningMethod = "secrets://jwt_signing_method"
+				spec.JWTSource = "secrets://jwt_source"
+			}),
+		},
+		{
+			name:   "SecretsManager",
+			scheme: "secretsmanager",
+			specs: BuildAPI(func(spec *APISpec) {
+				spec.JWTSigningMethod = "secretsmanager://jwt_signing_method"
+				spec.JWTSource = "secretsmanager://jwt_source"
+			}),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			specs := ts.Gw.LoadAPI(tt.specs...)
+			for _, spec := range specs {
+				assert.Equal(t, tt.scheme+"::jwt_signing_method::value", spec.JWTSigningMethod)
+				assert.Equal(t, tt.scheme+"::jwt_source::value", spec.JWTSource)
+			}
+		})
+	}
+}

gateway/mw_url_rewrite.go

@@ -20,19 +20,21 @@ import (
 )
 
 const (
-	metaLabel        = "$tyk_meta."
-	contextLabel     = "$tyk_context."
-	consulLabel      = "$secret_consul."
-	vaultLabel       = "$secret_vault."
-	envLabel         = "$secret_env."
-	secretsConfLabel = "$secret_conf."
-	triggerKeyPrefix = "trigger"
-	triggerKeySep    = "-"
+	metaLabel           = "$tyk_meta."
+	contextLabel        = "$tyk_context."
+	consulLabel         = "$secret_consul."
+	secretsManagerLabel = "$secret_secretsmanager."
+	vaultLabel          = "$secret_vault."
+	envLabel            = "$secret_env."
+	secretsConfLabel    = "$secret_conf."
+	triggerKeyPrefix    = "trigger"
+	triggerKeySep       = "-"
 )
 
 var dollarMatch = regexp.MustCompile(`\$\d+`)
 var contextMatch = regexp.MustCompile(`\$tyk_context.([A-Za-z0-9_\-\.]+)`)
 var consulMatch = regexp.MustCompile(`\$secret_consul.([A-Za-z0-9\/\-\.]+)`)
+var secretsManagerMatch = regexp.MustCompile(`\$secret_secretsmanager\.([A-Za-z0-9\/\-\._]+)`)
 var vaultMatch = regexp.MustCompile(`\$secret_vault.([A-Za-z0-9\/\-\.]+)`)
 var envValueMatch = regexp.MustCompile(`\$secret_env.([A-Za-z0-9_\-\.]+)`)
 var metaMatch = regexp.MustCompile(`\$tyk_meta.([A-Za-z0-9_\-\.]+)`)
@@ -211,7 +213,6 @@ func (gw *Gateway) urlRewrite(meta *apidef.URLRewriteMeta, r *http.Request) (str
 }
 
 func (gw *Gateway) replaceTykVariables(r *http.Request, in string, escape bool) string {
-
 	if strings.Contains(in, secretsConfLabel) {
 		contextData := ctxGetData(r)
 		vars := secretsConfMatch.FindAllString(in, -1)
@@ -251,12 +252,18 @@ func (gw *Gateway) replaceTykVariables(r *http.Request, in string, escape bool)
 			in = gw.replaceVariables(in, vars, session.MetaData, metaLabel, escape)
 		}
 	}
+
+	if strings.Contains(in, secretsManagerLabel) {
+		contextData := ctxGetData(r)
+		vars := secretsManagerMatch.FindAllString(in, -1)
+		in = gw.replaceVariables(in, vars, contextData, secretsManagerLabel, escape)
+	}
+
 	//todo add config_data
 	return in
 }
 
 func (gw *Gateway) replaceVariables(in string, vars []string, vals map[string]interface{}, label string, escape bool) string {
-
 	emptyStringFn := func(key, in, val string) string {
 		in = strings.Replace(in, val, "", -1)
 		log.WithFields(logrus.Fields{
@@ -272,9 +279,7 @@ func (gw *Gateway) replaceVariables(in string, vars []string, vals map[string]in
 		key := strings.Replace(v, label, "", 1)
 
 		switch label {
-
 		case secretsConfLabel:
-
 			secrets := gw.GetConfig().Secrets
 
 			val, ok := secrets[key]
@@ -284,19 +289,15 @@ func (gw *Gateway) replaceVariables(in string, vars []string, vals map[string]in
 			}
 
 			in = strings.Replace(in, v, val, -1)
-
 		case envLabel:
-
 			val := os.Getenv(fmt.Sprintf("TYK_SECRET_%s", strings.ToUpper(key)))
 			if val == "" {
 				in = emptyStringFn(key, in, v)
 				continue
 			}
 
 			in = strings.Replace(in, v, val, -1)
-
 		case vaultLabel:
-
 			if err := gw.setUpVault(); err != nil {
 				in = emptyStringFn(key, in, v)
 				continue
@@ -309,9 +310,7 @@ func (gw *Gateway) replaceVariables(in string, vars []string, vals map[string]in
 			}
 
 			in = strings.Replace(in, v, val, -1)
-
 		case consulLabel:
-
 			if err := gw.setUpConsul(); err != nil {
 				in = emptyStringFn(key, in, v)
 				continue
@@ -324,9 +323,20 @@ func (gw *Gateway) replaceVariables(in string, vars []string, vals map[string]in
 			}
 
 			in = strings.Replace(in, v, val, -1)
+		case secretsManagerLabel:
+			if err := gw.setUpSecretsManager(); err != nil {
+				in = emptyStringFn(key, in, v)
+				continue
+			}
 
-		default:
+			val, err := gw.secretsManagerKVStore.Get(key)
+			if err != nil {
+				in = strings.Replace(in, v, "", -1)
+				continue
+			}
 
+			in = strings.Replace(in, v, val, -1)
+		default:
 			val, ok := vals[key]
 			if ok {
 				valStr := valToStr(val)