test: add unittests for ns scoped obj store config

Signed-off-by: Humair Khan <HumairAK@users.noreply.github.com>
kubeflow · Mar 27, 2024 · e43914c · e43914c
1 parent 52192c1
commit e43914c
Show file tree

Hide file tree

Showing 5 changed files with 545 additions and 26 deletions.
diff --git a/backend/src/v2/component/launcher_v2_test.go b/backend/src/v2/component/launcher_v2_test.go
@@ -77,7 +77,7 @@ func Test_executeV2_Parameters(t *testing.T) {
 			fakeMetadataClient := metadata.NewFakeClient()
 			bucket, err := blob.OpenBucket(context.Background(), "gs://test-bucket")
 			assert.Nil(t, err)
-			bucketConfig, err := objectstore.ParseBucketConfig("gs://test-bucket/pipeline-root/")
+			bucketConfig, err := objectstore.ParseBucketConfig("gs://test-bucket/pipeline-root/", nil)
 			assert.Nil(t, err)
 			_, _, err = executeV2(context.Background(), test.executorInput, addNumbersComponent, "sh", test.executorArgs, bucket, bucketConfig, fakeMetadataClient, "namespace", fakeKubernetesClientset)
 

diff --git a/backend/src/v2/config/env_test.go b/backend/src/v2/config/env_test.go
@@ -0,0 +1,377 @@
+// Copyright 2024 The Kubeflow Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package config
+
+import (
+	"github.com/kubeflow/pipelines/backend/src/v2/objectstore"
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func Test_getDefaultMinioSessionInfo(t *testing.T) {
+	actualDefaultSession := getDefaultMinioSessionInfo()
+	expectedDefaultSession := objectstore.SessionInfo{
+		Region:       "minio",
+		Endpoint:     "minio-service.kubeflow:9000",
+		DisableSSL:   true,
+		SecretName:   "mlpipeline-minio-artifact",
+		AccessKeyKey: "accesskey",
+		SecretKeyKey: "secretkey",
+	}
+	assert.Equal(t, actualDefaultSession, expectedDefaultSession)
+}
+
+func TestGetBucketSessionInfo(t *testing.T) {
+	tt := []struct {
+		msg                 string
+		config              Config
+		expectedSessionInfo objectstore.SessionInfo
+		pipelineroot        string
+		// optional
+		shouldError bool
+		// optional
+		errorMsg string
+	}{
+		{
+			msg:                 "invalid_unsupported_object_store_protocol",
+			pipelineroot:        "unsupported://my-bucket/v2/artifacts",
+			config:              Config{},
+			expectedSessionInfo: objectstore.SessionInfo{},
+			shouldError:         true,
+			errorMsg:            "unsupported Cloud bucket",
+		},
+		{
+			msg:                 "invalid_unsupported_pipeline_root_format",
+			pipelineroot:        "minio.unsupported.format",
+			config:              Config{},
+			expectedSessionInfo: objectstore.SessionInfo{},
+			shouldError:         true,
+			errorMsg:            "unrecognized pipeline root format",
+		},
+		{
+			msg:          "valid_no_providers",
+			pipelineroot: "minio://my-bucket/v2/artifacts",
+			config:       Config{},
+			expectedSessionInfo: objectstore.SessionInfo{
+				Region:       "minio",
+				Endpoint:     "minio-service.kubeflow:9000",
+				DisableSSL:   true,
+				SecretName:   "mlpipeline-minio-artifact",
+				AccessKeyKey: "accesskey",
+				SecretKeyKey: "secretkey",
+			},
+		},
+		{
+			msg:          "invalid_one_empty_minio_provider",
+			pipelineroot: "minio://my-bucket/v2/artifacts",
+			config: Config{
+				data: map[string]string{
+					"providers": `
+minio: {}
+`,
+				},
+			},
+			expectedSessionInfo: objectstore.SessionInfo{},
+			shouldError:         true,
+			errorMsg:            "Invalid provider config",
+		},
+		{
+			msg:          "invalid_one_empty_minio_provider_no_authconfigs",
+			pipelineroot: "minio://my-bucket/v2/artifacts",
+			config: Config{
+				data: map[string]string{
+					"providers": `
+minio:
+  authConfigs: []
+`,
+				},
+			},
+			expectedSessionInfo: objectstore.SessionInfo{},
+			shouldError:         true,
+			errorMsg:            "Invalid provider config",
+		},
+		{
+			msg:          "invalid_one_minio_provider_endpoint_only",
+			pipelineroot: "minio://my-bucket/v2/artifacts",
+			config: Config{
+				data: map[string]string{
+					"providers": `
+minio:
+  endpoint: some-endpoint.com
+`,
+				},
+			},
+			expectedSessionInfo: objectstore.SessionInfo{},
+			shouldError:         true,
+			errorMsg:            "Invalid provider config",
+		},
+		{
+			msg:          "valid_one_minio_provider",
+			pipelineroot: "minio://my-bucket/v2/artifacts",
+			config: Config{
+				data: map[string]string{
+					"providers": `
+minio:
+  endpoint: some-endpoint.com
+  region: minio
+  authConfigs: []
+`,
+				},
+			},
+			expectedSessionInfo: objectstore.SessionInfo{
+				Region:       "minio",
+				Endpoint:     "some-endpoint.com",
+				DisableSSL:   false,
+				SecretName:   "mlpipeline-minio-artifact",
+				AccessKeyKey: "accesskey",
+				SecretKeyKey: "secretkey",
+			},
+		},
+		{
+			msg:          "valid_pick_matching_provider",
+			pipelineroot: "minio://my-bucket/v2/artifacts",
+			config: Config{
+				data: map[string]string{
+					"providers": `
+gcs:
+  endpoint: storage.cloud.google.com
+  region: gcs
+  authConfigs: []
+minio:
+  endpoint: some-endpoint.com
+  region: minio
+  authConfigs: []
+aws:
+  endpoint: s3.amazonaws.com
+  region: s3
+  authConfigs: []
+`,
+				},
+			},
+			expectedSessionInfo: objectstore.SessionInfo{
+				Region:       "minio",
+				Endpoint:     "some-endpoint.com",
+				DisableSSL:   false,
+				SecretName:   "mlpipeline-minio-artifact",
+				AccessKeyKey: "accesskey",
+				SecretKeyKey: "secretkey",
+			},
+		},
+		{
+			msg:          "invalid_non_minio_should_require_secret",
+			pipelineroot: "s3://my-bucket/v2/artifacts",
+			config: Config{
+				data: map[string]string{
+					"providers": `
+s3:
+  endpoint: s3.amazonaws.com
+  region: us-east-1
+  authConfigs: []
+`,
+				},
+			},
+			expectedSessionInfo: objectstore.SessionInfo{},
+			shouldError:         true,
+			errorMsg:            "Invalid provider config",
+		},
+		{
+			msg:          "invalid_matching_prefix_should_require_secretref",
+			pipelineroot: "minio://my-bucket/v2/artifacts/pick/this",
+			config: Config{
+				data: map[string]string{
+					"providers": `
+minio:
+  endpoint: minio.endpoint.com
+  region: minio
+  authConfigs:
+    - bucketName: my-bucket
+      keyPrefix: v2/artifacts/skip/this
+      secretRef:
+        secretName: minio-skip-this-secret
+        accessKeyKey: minio_skip_this_access_key
+        secretKeyKey: minio_skip_this_secret_key
+    - bucketName: my-bucket
+      keyPrefix: v2/artifacts/pick/this
+`,
+				},
+			},
+			expectedSessionInfo: objectstore.SessionInfo{},
+			shouldError:         true,
+			errorMsg:            "Invalid provider config",
+		},
+		{
+			msg:          "valid_s3_with_secret",
+			pipelineroot: "s3://my-bucket/v2/artifacts",
+			config: Config{
+				data: map[string]string{
+					"providers": `
+s3:
+  endpoint: s3.amazonaws.com
+  region: us-east-1
+  defaultProviderSecretRef:
+    secretName: "s3-provider-secret"
+    accessKeyKey: "different_access_key"
+    secretKeyKey: "different_secret_key"
+`,
+				},
+			},
+			expectedSessionInfo: objectstore.SessionInfo{
+				Region:       "us-east-1",
+				Endpoint:     "s3.amazonaws.com",
+				DisableSSL:   false,
+				SecretName:   "s3-provider-secret",
+				SecretKeyKey: "different_secret_key",
+				AccessKeyKey: "different_access_key",
+			},
+		},
+		{
+			msg:          "valid_minio_first_matching_auth_config",
+			pipelineroot: "minio://my-bucket/v2/artifacts/pick/this",
+			config: Config{
+				data: map[string]string{
+					"providers": `
+minio:
+  endpoint: minio.endpoint.com
+  region: minio
+  defaultProviderSecretRef:
+      secretName: minio-default-provider-secret
+      accessKeyKey: minio_default_different_access_key
+      secretKeyKey: minio_default_different_secret_key
+  authConfigs:
+    - bucketName: my-bucket
+      keyPrefix: v2/artifacts/skip/this
+      secretRef:
+        secretName: minio-skip-this-secret
+        accessKeyKey: minio_skip_this_access_key
+        secretKeyKey: minio_skip_this_secret_key
+    - bucketName: my-bucket
+      keyPrefix: v2/artifacts/pick/this
+      secretRef:
+        secretName: minio-pick-this-secret
+        accessKeyKey: minio_pick_this_access_key
+        secretKeyKey: minio_pick_this_secret_key
+    - bucketName: my-bucket
+      keyPrefix: v2/artifacts
+      secretRef:
+        secretName: minio-not-reached-secret
+        accessKeyKey: minio_not_reached_access_key
+        secretKeyKey: minio_not_reached_secret_key
+`,
+				},
+			},
+			expectedSessionInfo: objectstore.SessionInfo{
+				Region:       "minio",
+				Endpoint:     "minio.endpoint.com",
+				DisableSSL:   false,
+				SecretName:   "minio-pick-this-secret",
+				SecretKeyKey: "minio_pick_this_secret_key",
+				AccessKeyKey: "minio_pick_this_access_key",
+			},
+		},
+		{
+			msg:          "valid_gs_first_matching_auth_config",
+			pipelineroot: "gs://my-bucket/v2/artifacts/pick/this",
+			config: Config{
+				data: map[string]string{
+					"providers": `
+minio:
+  endpoint: minio.endpoint.com
+  region: minio
+  defaultProviderSecretRef:
+      secretName: minio-default-provider-secret
+      accessKeyKey: minio_default_different_access_key
+      secretKeyKey: minio_default_different_secret_key
+  authConfigs:
+    - bucketName: my-bucket
+      keyPrefix: v2/artifacts/skip/this
+      secretRef:
+        secretName: minio-skip-this-secret
+        accessKeyKey: minio_skip_this_access_key
+        secretKeyKey: minio_skip_this_secret_key
+gcs: 
+  endpoint: storage.googleapis.com
+  region: gcs
+  defaultProviderSecretRef:
+    secretName: gcs-default-provider-secret
+    accessKeyKey: gcs_default_different_access_key
+    secretKeyKey: gcs_default_different_secret_key
+  authConfigs:
+    - bucketName: my-bucket
+      keyPrefix: v2/artifacts/pick/this
+      secretRef:
+        secretName: gcs-pick-this-secret
+        accessKeyKey: gcs_pick_this_access_key
+        secretKeyKey: gcs_pick_this_secret_key
+`,
+				},
+			},
+			expectedSessionInfo: objectstore.SessionInfo{
+				Region:       "gcs",
+				Endpoint:     "storage.googleapis.com",
+				DisableSSL:   false,
+				SecretName:   "gcs-pick-this-secret",
+				SecretKeyKey: "gcs_pick_this_secret_key",
+				AccessKeyKey: "gcs_pick_this_access_key",
+			},
+		},
+		{
+			msg:          "valid_pick_default_when_no_matching_prefix",
+			pipelineroot: "gs://my-bucket/v2/artifacts/pick/default",
+			config: Config{
+				data: map[string]string{
+					"providers": `
+gcs: 
+  endpoint: storage.googleapis.com
+  region: gcs
+  defaultProviderSecretRef:
+    secretName: gcs-default-provider-secret
+    accessKeyKey: gcs_default_different_access_key
+    secretKeyKey: gcs_default_different_secret_key
+  authConfigs:
+    - bucketName: my-bucket
+      keyPrefix: v2/artifacts/skip/this
+      secretRef:
+        secretName: gcs-skip-this-secret
+        accessKeyKey: gcs_skip_this_access_key
+        secretKeyKey: gcs_skip_this_secret_key
+`,
+				},
+			},
+			expectedSessionInfo: objectstore.SessionInfo{
+				Region:       "gcs",
+				Endpoint:     "storage.googleapis.com",
+				DisableSSL:   false,
+				SecretName:   "gcs-default-provider-secret",
+				SecretKeyKey: "gcs_default_different_secret_key",
+				AccessKeyKey: "gcs_default_different_access_key",
+			},
+		},
+	}
+
+	for _, test := range tt {
+		t.Run(test.msg, func(t *testing.T) {
+			actualSession, err := test.config.GetBucketSessionInfo(test.pipelineroot)
+			if test.shouldError {
+				assert.Error(t, err)
+				if err != nil && test.errorMsg != "" {
+					assert.Contains(t, err.Error(), test.errorMsg)
+				}
+			} else {
+				assert.Nil(t, err)
+			}
+			assert.Equal(t, test.expectedSessionInfo, actualSession)
+		})
+	}
+}
diff --git a/backend/src/v2/metadata/client_fake.go b/backend/src/v2/metadata/client_fake.go
@@ -32,7 +32,7 @@ func NewFakeClient() *FakeClient {
 	return &FakeClient{}
 }
 
-func (c *FakeClient) GetPipeline(ctx context.Context, pipelineName, runID, namespace, runResource, pipelineRoot string) (*Pipeline, error) {
+func (c *FakeClient) GetPipeline(ctx context.Context, pipelineName, runID, namespace, runResource, pipelineRoot string, bucketSessionInfo string) (*Pipeline, error) {
 	return nil, nil
 }