Skip to content

Commit

Permalink
Read emulatorEnabled value for storage emulators from mounted secre…
Browse files Browse the repository at this point in the history 
…t instead of environment variables (#819)

* Read `emulatorEnabled` value for storage emulators from mounted secret instead of environment variables

* Address review comments from @anveshreddy18; introduce `SnapstoreConfig.IsEmulatorEnabled`

* Address review comments from @anveshreddy18 and @renormalize - throw error if emulator is enabled but `storageAPIEndpoint`/`domain` is not provided
  • Loading branch information
@shreyas-s-rao
shreyas-s-rao authored Dec 30, 2024
1 parent 2438610 commit 2d7dab8
Show file tree
Hide file tree
Showing 6 changed files with 86 additions and 109 deletions.
16 changes: 0 additions & 16 deletions chart/etcd-backup-restore/templates/etcd-statefulset.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -191,25 +191,9 @@ spec:
secretKeyRef:
name: {{ .Release.Name }}-etcd-backup
key: "storageKey"
{{- if .Values.backup.abs.emulatorEnabled }}
- name: "AZURE_EMULATOR_ENABLED"
valueFrom:
secretKeyRef:
name: {{ .Release.Name }}-etcd-backup
key: "emulatorEnabled"
optional: true
{{- end }}
{{- else if eq .Values.backup.storageProvider "GCS" }}
- name: "GOOGLE_APPLICATION_CREDENTIALS"
value: "/root/.gcp/serviceaccount.json"
{{- if .Values.backup.gcs.emulatorEnabled }}
- name: "GOOGLE_EMULATOR_ENABLED"
valueFrom:
secretKeyRef:
name: {{ .Release.Name }}-etcd-backup
key: "emulatorEnabled"
optional: true
{{- end }}
{{- else if eq .Values.backup.storageProvider "Swift" }}
- name: "OS_AUTH_URL"
valueFrom:
Expand Down
60 changes: 33 additions & 27 deletions pkg/snapstore/abs_snapstore.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,10 +91,11 @@ type ABSSnapStore struct {
}

type absCredentials struct {
BucketName string `json:"bucketName"`
StorageAccount string `json:"storageAccount"`
StorageKey string `json:"storageKey"`
Domain *string `json:"domain,omitempty"`
BucketName string `json:"bucketName"`
StorageAccount string `json:"storageAccount"`
StorageKey string `json:"storageKey"`
Domain *string `json:"domain,omitempty"`
EmulatorEnabled bool `json:"emulatorEnabled,omitempty"`
}

// NewABSSnapStore creates a new ABSSnapStore using a shared configuration and a specified bucket
Expand All @@ -109,12 +110,18 @@ func NewABSSnapStore(config *brtypes.SnapstoreConfig) (*ABSSnapStore, error) {
return nil, fmt.Errorf("failed to create sharedKeyCredential: %w", err)
}

emulatorEnabled := config.IsEmulatorEnabled || absCreds.EmulatorEnabled
domain := brtypes.AzureBlobStorageGlobalDomain
if absCreds.Domain != nil {
domain = *absCreds.Domain
} else {
// if emulator is enabled, but custom domain for the emulator is not provided, throw error
if emulatorEnabled {
return nil, fmt.Errorf("emulator enabled, but `domain` not provided")
}
}

blobServiceURL, err := ConstructBlobServiceURL(absCreds.StorageAccount, domain)
blobServiceURL, err := ConstructBlobServiceURL(absCreds.StorageAccount, domain, emulatorEnabled)
if err != nil {
return nil, fmt.Errorf("failed to construct the blob service URL with error: %w", err)
}
Expand Down Expand Up @@ -145,25 +152,14 @@ func NewABSSnapStore(config *brtypes.SnapstoreConfig) (*ABSSnapStore, error) {
}

// ConstructBlobServiceURL constructs the Blob Service URL based on the activation status of the Azurite Emulator.
// It checks the environment variable for emulator configuration and constructs the URL accordingly.
// The function expects the following environment variable:
// - AZURE_EMULATOR_ENABLED: Indicates whether the Azurite Emulator is enabled (expects "true" or "false").
func ConstructBlobServiceURL(storageAccount, domain string) (string, error) {
scheme := "https"

emulatorEnabled, ok := os.LookupEnv(EnvAzureEmulatorEnabled)
if ok {
isEmulator, err := strconv.ParseBool(emulatorEnabled)
if err != nil {
return "", fmt.Errorf("invalid value for %s: %s, error: %w", EnvAzureEmulatorEnabled, emulatorEnabled, err)
}
if isEmulator {
// TODO: going forward, use Azurite with HTTPS (TLS) communication
scheme = "http"
}
}

return fmt.Sprintf("%s://%s.%s", scheme, storageAccount, domain), nil
// The `domain` must either be the default Azure global blob storage domain, or a specific domain for Azurite (without HTTP scheme).
func ConstructBlobServiceURL(storageAccount, domain string, emulatorEnabled bool) (string, error) {
if emulatorEnabled {
// TODO: going forward, use Azurite with HTTPS (TLS) communication
// by using [production-style URLs](https://github.com/Azure/Azurite?tab=readme-ov-file#production-style-url)
return fmt.Sprintf("http://%s/%s", domain, storageAccount), nil
}
return fmt.Sprintf("https://%s.%s", storageAccount, domain), nil
}

func getCredentials(prefixString string) (*absCredentials, error) {
Expand Down Expand Up @@ -231,23 +227,33 @@ func readABSCredentialFiles(dirname string) (*absCredentials, error) {

for _, file := range files {
if file.Name() == "storageAccount" {
data, err := os.ReadFile(path.Join(dirname, file.Name())) // #nosec G304 -- this is a trusted file, obtained via user input.
data, err := os.ReadFile(path.Join(dirname, file.Name())) // #nosec G304 -- this is a trusted file, obtained via mounted secret.
if err != nil {
return nil, err
}
absConfig.StorageAccount = string(data)
} else if file.Name() == "storageKey" {
data, err := os.ReadFile(path.Join(dirname, file.Name())) // #nosec G304 -- this is a trusted file, obtained via user input.
data, err := os.ReadFile(path.Join(dirname, file.Name())) // #nosec G304 -- this is a trusted file, obtained via mounted secret.
if err != nil {
return nil, err
}
absConfig.StorageKey = string(data)
} else if file.Name() == "domain" {
data, err := os.ReadFile(path.Join(dirname, file.Name())) // #nosec G304 -- this is a trusted file, obtained via user input.
data, err := os.ReadFile(path.Join(dirname, file.Name())) // #nosec G304 -- this is a trusted file, obtained via mounted secret.
if err != nil {
return nil, err
}
absConfig.Domain = ptr.To(string(data))
} else if file.Name() == "emulatorEnabled" {
data, err := os.ReadFile(path.Join(dirname, file.Name())) // #nosec G304 -- this is a trusted file, obtained via mounted secret.
if err != nil {
return nil, err
}
emulatorEnabled, err := strconv.ParseBool(string(data))
if err != nil {
return nil, err
}
absConfig.EmulatorEnabled = emulatorEnabled
}
}

Expand Down
81 changes: 43 additions & 38 deletions pkg/snapstore/gcs_snapstore.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,10 @@ import (

const (
envStoreCredentials = "GOOGLE_APPLICATION_CREDENTIALS" // #nosec G101 -- This is not a hardcoded password, but only the environment variable to the credentials.
envStorageAPIEndpoint = "GOOGLE_STORAGE_API_ENDPOINT"
envSourceStoreCredentials = "SOURCE_GOOGLE_APPLICATION_CREDENTIALS"

storageAPIEndpointFileName = "storageAPIEndpoint"
fileNameEmulatorEnabled = "emulatorEnabled"
fileNameStorageAPIEndpoint = "storageAPIEndpoint"
)

// GCSSnapStore is snapstore with GCS object store as backend.
Expand Down Expand Up @@ -61,21 +61,23 @@ const (
// NewGCSSnapStore create new GCSSnapStore from shared configuration with specified bucket.
func NewGCSSnapStore(config *brtypes.SnapstoreConfig) (*GCSSnapStore, error) {
ctx := context.TODO()
var emulatorConfig gcsEmulatorConfig
emulatorConfig.enabled = isEmulatorEnabled()
var opts []option.ClientOption // no need to explicitly set store credentials here since the Google SDK picks it up from the standard environment variable

if gcsApplicationCredentialsPath, isSet := os.LookupEnv(getEnvPrefixString(config.IsSource) + envStoreCredentials); isSet {
storageAPIEndpointFilePath := path.Join(path.Dir(gcsApplicationCredentialsPath), storageAPIEndpointFileName)
endpoint, err := getGCSStorageAPIEndpoint(storageAPIEndpointFilePath)
if err != nil {
return nil, fmt.Errorf("error getting storage API endpoint from %v", storageAPIEndpointFilePath)
var opts []option.ClientOption // no need to explicitly set store credentials here since the Google SDK picks it up from the standard environment variable
var emulatorConfig gcsEmulatorConfig
emulatorConfig.enabled = config.IsEmulatorEnabled || isEmulatorEnabled(config)
endpoint, err := getGCSStorageAPIEndpoint(config)
if err != nil {
return nil, err
}
if endpoint != "" {
opts = append(opts, option.WithEndpoint(endpoint))
if emulatorConfig.enabled {
emulatorConfig.endpoint = endpoint
}
if endpoint != "" {
opts = append(opts, option.WithEndpoint(endpoint))
if emulatorConfig.enabled {
emulatorConfig.endpoint = endpoint
}
} else {
// if emulator is enabled, but custom storage API endpoint for the emulator is not provided, throw error
if emulatorConfig.enabled {
return nil, fmt.Errorf("emulator enabled, but `storageAPIEndpoint` not provided")
}
}

Expand Down Expand Up @@ -104,20 +106,36 @@ func NewGCSSnapStore(config *brtypes.SnapstoreConfig) (*GCSSnapStore, error) {
return NewGCSSnapStoreFromClient(config.Container, config.Prefix, config.TempDir, config.MaxParallelChunkUploads, config.MinChunkSize, chunkDirSuffix, gcsClient), nil
}

func getGCSStorageAPIEndpoint(path string) (string, error) {
if _, err := os.Stat(path); err == nil {
data, err := os.ReadFile(path) // #nosec G304 -- this is a trusted file, obtained via user input.
if err != nil {
return "", err
func getGCSStorageAPIEndpoint(config *brtypes.SnapstoreConfig) (string, error) {
if gcsApplicationCredentialsPath, isSet := os.LookupEnv(getEnvPrefixString(config.IsSource) + envStoreCredentials); isSet {
storageAPIEndpointFilePath := path.Join(path.Dir(gcsApplicationCredentialsPath), fileNameStorageAPIEndpoint)
if _, err := os.Stat(storageAPIEndpointFilePath); err != nil {
// if the file does not exist, then there is no override for the storage API endpoint
return "", nil
}
if len(data) == 0 {
return "", fmt.Errorf("file %s is empty", path)
endpoint, err := os.ReadFile(storageAPIEndpointFilePath) // #nosec G304 -- this is a trusted file, obtained from mounted secret.
if err != nil {
return "", fmt.Errorf("error getting storage API endpoint from %v", storageAPIEndpointFilePath)
}
return strings.TrimSpace(string(data)), nil
return string(endpoint), nil
}
return "", nil
}

// support falling back to environment variable `GOOGLE_STORAGE_API_ENDPOINT`
return strings.TrimSpace(os.Getenv(envStorageAPIEndpoint)), nil
func isEmulatorEnabled(config *brtypes.SnapstoreConfig) bool {
if gcsApplicationCredentialsPath, isSet := os.LookupEnv(getEnvPrefixString(config.IsSource) + envStoreCredentials); isSet {
emulatorEnabledFilePath := path.Join(path.Dir(gcsApplicationCredentialsPath), fileNameEmulatorEnabled)
emulatorEnabledString, err := os.ReadFile(emulatorEnabledFilePath) // #nosec G304 -- this is a trusted file, obtained from mounted secret.
if err != nil {
return false
}
emulatorEnabled, err := strconv.ParseBool(string(emulatorEnabledString))
if err != nil {
return false
}
return emulatorEnabled
}
return false
}

// NewGCSSnapStoreFromClient create new GCSSnapStore from shared configuration with specified bucket.
Expand All @@ -133,19 +151,6 @@ func NewGCSSnapStoreFromClient(bucket, prefix, tempDir string, maxParallelChunkU
}
}

// isEmulatorEnabled checks if the fake GCS emulator is enabled
func isEmulatorEnabled() bool {
isFakeGCSEnabled, ok := os.LookupEnv(EnvGCSEmulatorEnabled)
if !ok {
return false
}
emulatorEnabled, err := strconv.ParseBool(isFakeGCSEnabled)
if err != nil {
return false
}
return emulatorEnabled
}

// configureClient configures the fake gcs emulator
func (e *gcsEmulatorConfig) configureClient(opts []option.ClientOption) error {
err := os.Setenv("STORAGE_EMULATOR_HOST", strings.TrimPrefix(e.endpoint, "http://"))
Expand Down
4 changes: 0 additions & 4 deletions pkg/snapstore/snapstore.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,6 @@ const (

backupVersionV1 = "v1"
backupVersionV2 = "v2"
// EnvAzureEmulatorEnabled is the environment variable which indicates whether the Azurite emulator is enabled
EnvAzureEmulatorEnabled = "AZURE_EMULATOR_ENABLED"
// EnvGCSEmulatorEnabled is the environment variable which indicates whether the GCS emulator is enabled
EnvGCSEmulatorEnabled = "GOOGLE_EMULATOR_ENABLED"
)

type chunk struct {
Expand Down
32 changes: 8 additions & 24 deletions pkg/snapstore/snapstore_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -558,34 +558,18 @@ var _ = Describe("Blob Service URL construction for Azure", func() {
credentials, err = container.NewSharedKeyCredential(storageAccount, storageKey)
Expect(err).ShouldNot(HaveOccurred())
})
Context(fmt.Sprintf("when the environment variable %q is not set", EnvAzureEmulatorEnabled), func() {
It("should return the default blob service URL", func() {
blobServiceURL, err := ConstructBlobServiceURL(credentials.AccountName(), domain)
Context("when emulatorEnabled field is not set or set to false", func() {
It("should return the default blob service URL with HTTPS scheme", func() {
blobServiceURL, err := ConstructBlobServiceURL(credentials.AccountName(), domain, false)
Expect(err).ShouldNot(HaveOccurred())
Expect(blobServiceURL).Should(Equal(fmt.Sprintf("https://%s.%s", credentials.AccountName(), domain)))
})
})
Context(fmt.Sprintf("when the environment variable %q is set", EnvAzureEmulatorEnabled), func() {
Context("to values which are not \"true\"", func() {
It("should error when the environment variable is not \"true\" or \"false\"", func() {
GinkgoT().Setenv(EnvAzureEmulatorEnabled, "")
_, err := ConstructBlobServiceURL(credentials.AccountName(), domain)
Expect(err).Should(HaveOccurred())
})
It("should return the default blob service URL when the environment variable is set to \"false\"", func() {
GinkgoT().Setenv(EnvAzureEmulatorEnabled, "false")
blobServiceURL, err := ConstructBlobServiceURL(credentials.AccountName(), domain)
Expect(err).ShouldNot(HaveOccurred())
Expect(blobServiceURL).Should(Equal(fmt.Sprintf("https://%s.%s", credentials.AccountName(), domain)))
})
})
Context("to \"true\"", func() {
It("should return the Azurite blob service URL with HTTP scheme", func() {
GinkgoT().Setenv(EnvAzureEmulatorEnabled, "true")
blobServiceURL, err := ConstructBlobServiceURL(credentials.AccountName(), domain)
Expect(err).ShouldNot(HaveOccurred())
Expect(blobServiceURL).Should(Equal(fmt.Sprintf("http://%s.%s", credentials.AccountName(), domain)))
})
Context("when emulatorEnabled field is set to true", func() {
It("should return the Azurite blob service URL with HTTP scheme", func() {
blobServiceURL, err := ConstructBlobServiceURL(credentials.AccountName(), domain, true)
Expect(err).ShouldNot(HaveOccurred())
Expect(blobServiceURL).Should(Equal(fmt.Sprintf("http://%s/%s", domain, credentials.AccountName())))
})
})
})
Expand Down
2 changes: 2 additions & 0 deletions pkg/types/snapstore.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,8 @@ type SnapstoreConfig struct {
TempDir string `json:"tempDir,omitempty"`
// IsSource determines if this SnapStore is the source for a copy operation
IsSource bool `json:"isSource,omitempty"`
// IsEmulatorEnabled indicates whether a storage emulator is being used for the snapstore.
IsEmulatorEnabled bool `json:"isEmulatorEnabled,omitempty"`
}

// AddFlags adds the flags to flagset.
Expand Down

0 comments on commit 2d7dab8

Please sign in to comment.