feat(NODE-5050): support GCP automatic credential fetch for CSFLE (#3574)

nbbeeken · baileympearson · web-flow · commit 722a4a6c613e · 2023-02-22T09:31:34.000-05:00
Co-authored-by: Bailey Pearson &lt;bailey.pearson@mongodb.com&gt;
diff --git a/.evergreen/config.in.yml b/.evergreen/config.in.yml
@@ -1093,6 +1093,58 @@ tasks:
         params:
           file: src/results.json
 
+  - name: "test-gcpkms-task"
+    commands:
+      - func: "install dependencies"
+      # Upload node driver to a GCP instance
+      - command: subprocess.exec
+        type: setup
+        params:
+          binary: bash
+          add_expansions_to_env: true
+          env:
+            DRIVERS_TOOLS: ${DRIVERS_TOOLS}
+            GCPKMS_GCLOUD: ${GCPKMS_GCLOUD}
+            GCPKMS_PROJECT: ${GCPKMS_PROJECT}
+            GCPKMS_ZONE: ${GCPKMS_ZONE}
+            GCPKMS_INSTANCENAME: ${GCPKMS_INSTANCENAME}
+          args:
+            - src/.evergreen/setup-gcp-testing.sh
+      # Run Mocha test over on GCE instance
+      - command: subprocess.exec
+        type: test
+        params:
+          working_dir: src
+          binary: bash
+          env:
+            GCPKMS_GCLOUD: ${GCPKMS_GCLOUD}
+            GCPKMS_PROJECT: ${GCPKMS_PROJECT}
+            GCPKMS_ZONE: ${GCPKMS_ZONE}
+            GCPKMS_INSTANCENAME: ${GCPKMS_INSTANCENAME}
+            GCPKMS_CMD: "env EXPECTED_GCPKMS_OUTCOME=success bash src/.evergreen/run-gcp-kms-tests.sh"
+          args:
+            - ${DRIVERS_TOOLS}/.evergreen/csfle/gcpkms/run-command.sh
+
+
+  - name: "test-gcpkms-fail-task"
+    # test-gcpkms-fail-task runs in a non-GCE environment.
+    # It is expected to fail to obtain GCE credentials.
+    commands:
+      - func: "install dependencies"
+      - func: bootstrap mongo-orchestration
+        vars:
+          VERSION: latest
+          TOPOLOGY: server
+          AUTH: noauth
+      - command: subprocess.exec
+        type: test
+        params:
+          binary: bash
+          env:
+            EXPECTED_GCPKMS_OUTCOME: "failure"
+          args:
+            - src/.evergreen/run-gcp-kms-tests.sh
+
 task_groups:
   - name: serverless_task_group
     setup_group_can_fail_task: true
@@ -1101,7 +1153,7 @@ task_groups:
       - func: "fetch source"
       - command: shell.exec
         params:
-          shell: "bash"
+          shell: bash
           script: |
             ${PREPARE_SHELL}
             set +o xtrace
@@ -1128,6 +1180,43 @@ task_groups:
     tasks:
       - ".serverless"
 
+  - name: test_gcpkms_task_group
+    setup_group_can_fail_task: true
+    setup_group_timeout_secs: 1800 # 30 minutes
+    setup_group:
+      - func: fetch source
+      - command: subprocess.exec
+        params:
+          working_dir: "src"
+          binary: bash
+          add_expansions_to_env: true
+          env:
+            testgcpkms_key_file: ${gcpkms_key_file}
+            GCPKMS_SERVICEACCOUNT: ${gcpkms_service_account}
+            GCPKMS_DRIVERS_TOOLS: ${DRIVERS_TOOLS}
+            GCPKMS_MACHINETYPE: "e2-standard-4"
+          args:
+            - .evergreen/setup-gcp-instance.sh
+      - command: expansions.update
+        # Load the GCPKMS_GCLOUD, GCPKMS_INSTANCE, GCPKMS_REGION, and GCPKMS_ZONE expansions.
+        params:
+          file: src/testgcpkms-expansions.yml
+
+    teardown_group:
+      - command: subprocess.exec
+        params:
+          binary: bash
+          add_expansions_to_env: true
+          env:
+            GCPKMS_GCLOUD: ${GCPKMS_GCLOUD}
+            GCPKMS_PROJECT: ${GCPKMS_PROJECT}
+            GCPKMS_ZONE: ${GCPKMS_ZONE}
+            GCPKMS_INSTANCENAME: ${GCPKMS_INSTANCENAME}
+          args:
+            - ${DRIVERS_TOOLS}/.evergreen/csfle/gcpkms/delete-instance.sh
+    tasks:
+      - test-gcpkms-task
+
 pre:
   - func: "fetch source"
   - func: "windows fix"
diff --git a/.evergreen/config.yml b/.evergreen/config.yml
@@ -1031,6 +1031,51 @@ tasks:
       - command: perf.send
         params:
           file: src/results.json
+  - name: test-gcpkms-task
+    commands:
+      - func: install dependencies
+      - command: subprocess.exec
+        type: setup
+        params:
+          binary: bash
+          add_expansions_to_env: true
+          env:
+            DRIVERS_TOOLS: ${DRIVERS_TOOLS}
+            GCPKMS_GCLOUD: ${GCPKMS_GCLOUD}
+            GCPKMS_PROJECT: ${GCPKMS_PROJECT}
+            GCPKMS_ZONE: ${GCPKMS_ZONE}
+            GCPKMS_INSTANCENAME: ${GCPKMS_INSTANCENAME}
+          args:
+            - src/.evergreen/setup-gcp-testing.sh
+      - command: subprocess.exec
+        type: test
+        params:
+          working_dir: src
+          binary: bash
+          env:
+            GCPKMS_GCLOUD: ${GCPKMS_GCLOUD}
+            GCPKMS_PROJECT: ${GCPKMS_PROJECT}
+            GCPKMS_ZONE: ${GCPKMS_ZONE}
+            GCPKMS_INSTANCENAME: ${GCPKMS_INSTANCENAME}
+            GCPKMS_CMD: env EXPECTED_GCPKMS_OUTCOME=success bash src/.evergreen/run-gcp-kms-tests.sh
+          args:
+            - ${DRIVERS_TOOLS}/.evergreen/csfle/gcpkms/run-command.sh
+  - name: test-gcpkms-fail-task
+    commands:
+      - func: install dependencies
+      - func: bootstrap mongo-orchestration
+        vars:
+          VERSION: latest
+          TOPOLOGY: server
+          AUTH: noauth
+      - command: subprocess.exec
+        type: test
+        params:
+          binary: bash
+          env:
+            EXPECTED_GCPKMS_OUTCOME: failure
+          args:
+            - src/.evergreen/run-gcp-kms-tests.sh
   - name: test-latest-server
     tags:
       - latest
@@ -3002,6 +3047,40 @@ task_groups:
               bash ${DRIVERS_TOOLS}/.evergreen/serverless/delete-instance.sh
     tasks:
       - .serverless
+  - name: test_gcpkms_task_group
+    setup_group_can_fail_task: true
+    setup_group_timeout_secs: 1800
+    setup_group:
+      - func: fetch source
+      - command: subprocess.exec
+        params:
+          working_dir: src
+          binary: bash
+          add_expansions_to_env: true
+          env:
+            testgcpkms_key_file: ${gcpkms_key_file}
+            GCPKMS_SERVICEACCOUNT: ${gcpkms_service_account}
+            GCPKMS_DRIVERS_TOOLS: ${DRIVERS_TOOLS}
+            GCPKMS_MACHINETYPE: e2-standard-4
+          args:
+            - .evergreen/setup-gcp-instance.sh
+      - command: expansions.update
+        params:
+          file: src/testgcpkms-expansions.yml
+    teardown_group:
+      - command: subprocess.exec
+        params:
+          binary: bash
+          add_expansions_to_env: true
+          env:
+            GCPKMS_GCLOUD: ${GCPKMS_GCLOUD}
+            GCPKMS_PROJECT: ${GCPKMS_PROJECT}
+            GCPKMS_ZONE: ${GCPKMS_ZONE}
+            GCPKMS_INSTANCENAME: ${GCPKMS_INSTANCENAME}
+          args:
+            - ${DRIVERS_TOOLS}/.evergreen/csfle/gcpkms/delete-instance.sh
+    tasks:
+      - test-gcpkms-task
 pre:
   - func: fetch source
   - func: windows fix
@@ -3461,6 +3540,12 @@ buildvariants:
       NODE_LTS_NAME: fermium
     tasks:
       - serverless_task_group
+  - name: rhel8-test-gcp-kms
+    display_name: GCP KMS Test
+    run_on: debian11-small
+    tasks:
+      - test_gcpkms_task_group
+      - test-gcpkms-fail-task
   - name: rhel8-no-auth-tests
     display_name: No Auth Tests
     run_on: rhel80-large
diff --git a/.evergreen/generate_evergreen_tasks.js b/.evergreen/generate_evergreen_tasks.js
@@ -663,6 +663,13 @@ BUILD_VARIANTS.push({
   tasks: ['serverless_task_group']
 });
 
+BUILD_VARIANTS.push({
+  name: 'rhel8-test-gcp-kms',
+  display_name: 'GCP KMS Test',
+  run_on: 'debian11-small',
+  tasks: ['test_gcpkms_task_group', 'test-gcpkms-fail-task']
+});
+
 BUILD_VARIANTS.push({
   name: 'rhel8-no-auth-tests',
   display_name: 'No Auth Tests',
diff --git a/.evergreen/run-gcp-kms-tests.sh b/.evergreen/run-gcp-kms-tests.sh
@@ -0,0 +1,21 @@
+#! /usr/bin/env bash
+
+set -o errexit
+
+pushd "src"
+PROJECT_DIRECTORY="$(pwd)"
+export PROJECT_DIRECTORY
+source ".evergreen/init-nvm.sh"
+
+set -o xtrace
+
+npm install 'mongodb-client-encryption@2.6.0-alpha.0'
+npm install 'gcp-metadata'
+
+export MONGODB_URI="mongodb://localhost:27017"
+
+export EXPECTED_GCPKMS_OUTCOME=${EXPECTED_GCPKMS_OUTCOME:-omitted}
+export TEST_CSFLE=true
+export CSFLE_KMS_PROVIDERS='not json'
+
+npx mocha --config test/mocha_mongodb.json test/integration/client-side-encryption/client_side_encryption.prose.17.on_demand_gcp.test.ts
diff --git a/.evergreen/setup-gcp-instance.sh b/.evergreen/setup-gcp-instance.sh
@@ -0,0 +1,9 @@
+#! /usr/bin/env bash
+
+set -o errexit
+if [ -z ${testgcpkms_key_file+omitted} ]; then echo "testgcpkms_key_file is unset" && exit 1; fi
+
+echo "${testgcpkms_key_file}" > ./testgcpkms_key_file.json
+export GCPKMS_KEYFILE=./testgcpkms_key_file.json
+
+"$GCPKMS_DRIVERS_TOOLS/.evergreen/csfle/gcpkms/create-and-setup-instance.sh"
diff --git a/.evergreen/setup-gcp-testing.sh b/.evergreen/setup-gcp-testing.sh
@@ -0,0 +1,28 @@
+#! /usr/bin/env bash
+
+# Assert required environment variables are present without printing them
+if [ -z ${GCPKMS_GCLOUD+omitted} ]; then echo "GCPKMS_GCLOUD is unset" && exit 1; fi
+if [ -z ${GCPKMS_PROJECT+omitted} ]; then echo "GCPKMS_PROJECT is unset" && exit 1; fi
+if [ -z ${GCPKMS_ZONE+omitted} ]; then echo "GCPKMS_ZONE is unset" && exit 1; fi
+if [ -z ${GCPKMS_INSTANCENAME+omitted} ]; then echo "GCPKMS_INSTANCENAME is unset" && exit 1; fi
+
+set -o errexit
+
+source "${PROJECT_DIRECTORY}/.evergreen/init-nvm.sh"
+
+export GCPKMS_SRC=node-driver-source.tgz
+export GCPKMS_DST=$GCPKMS_INSTANCENAME:
+
+# Box up the entire driver and it's node_modules
+echo "compressing node driver source ... begin"
+tar -czf $GCPKMS_SRC src
+echo "compressing node driver source ... end"
+
+echo "copying node driver tar ... begin"
+"${DRIVERS_TOOLS}/.evergreen/csfle/gcpkms/copy-file.sh"
+echo "copying node driver tar ... end"
+
+echo "decompressing node driver tar on gcp ... begin"
+export GCPKMS_CMD="tar -xzf $GCPKMS_SRC"
+"${DRIVERS_TOOLS}/.evergreen/csfle/gcpkms/run-command.sh"
+echo "decompressing node driver tar on gcp ... end"
diff --git a/src/deps.ts b/src/deps.ts
@@ -223,17 +223,19 @@ export interface AutoEncryptionOptions {
   /** Configuration options that are used by specific KMS providers during key generation, encryption, and decryption. */
   kmsProviders?: {
     /** Configuration options for using 'aws' as your KMS provider */
-    aws?: {
-      /** The access key used for the AWS KMS provider */
-      accessKeyId: string;
-      /** The secret access key used for the AWS KMS provider */
-      secretAccessKey: string;
-      /**
-       * An optional AWS session token that will be used as the
-       * X-Amz-Security-Token header for AWS requests.
-       */
-      sessionToken?: string;
-    };
+    aws?:
+      | {
+          /** The access key used for the AWS KMS provider */
+          accessKeyId: string;
+          /** The secret access key used for the AWS KMS provider */
+          secretAccessKey: string;
+          /**
+           * An optional AWS session token that will be used as the
+           * X-Amz-Security-Token header for AWS requests.
+           */
+          sessionToken?: string;
+        }
+      | Record<string, never>;
     /** Configuration options for using 'local' as your KMS provider */
     local?: {
       /**
@@ -243,33 +245,48 @@ export interface AutoEncryptionOptions {
       key: Buffer | string;
     };
     /** Configuration options for using 'azure' as your KMS provider */
-    azure?: {
-      /** The tenant ID identifies the organization for the account */
-      tenantId: string;
-      /** The client ID to authenticate a registered application */
-      clientId: string;
-      /** The client secret to authenticate a registered application */
-      clientSecret: string;
-      /**
-       * If present, a host with optional port. E.g. "example.com" or "example.com:443".
-       * This is optional, and only needed if customer is using a non-commercial Azure instance
-       * (e.g. a government or China account, which use different URLs).
-       * Defaults to "login.microsoftonline.com"
-       */
-      identityPlatformEndpoint?: string | undefined;
-    };
+    azure?:
+      | {
+          /** The tenant ID identifies the organization for the account */
+          tenantId: string;
+          /** The client ID to authenticate a registered application */
+          clientId: string;
+          /** The client secret to authenticate a registered application */
+          clientSecret: string;
+          /**
+           * If present, a host with optional port. E.g. "example.com" or "example.com:443".
+           * This is optional, and only needed if customer is using a non-commercial Azure instance
+           * (e.g. a government or China account, which use different URLs).
+           * Defaults to "login.microsoftonline.com"
+           */
+          identityPlatformEndpoint?: string | undefined;
+        }
+      | {
+          /**
+           * If present, an access token to authenticate with Azure.
+           */
+          accessToken: string;
+        };
     /** Configuration options for using 'gcp' as your KMS provider */
-    gcp?: {
-      /** The service account email to authenticate */
-      email: string;
-      /** A PKCS#8 encrypted key. This can either be a base64 string or a binary representation */
-      privateKey: string | Buffer;
-      /**
-       * If present, a host with optional port. E.g. "example.com" or "example.com:443".
-       * Defaults to "oauth2.googleapis.com"
-       */
-      endpoint?: string | undefined;
-    };
+    gcp?:
+      | {
+          /** The service account email to authenticate */
+          email: string;
+          /** A PKCS#8 encrypted key. This can either be a base64 string or a binary representation */
+          privateKey: string | Buffer;
+          /**
+           * If present, a host with optional port. E.g. "example.com" or "example.com:443".
+           * Defaults to "oauth2.googleapis.com"
+           */
+          endpoint?: string | undefined;
+        }
+      | {
+          /**
+           * If present, an access token to authenticate with GCP.
+           */
+          accessToken: string;
+        }
+      | Record<string, never>;
     /**
      * Configuration options for using 'kmip' as your KMS provider
      */
diff --git a/test/integration/client-side-encryption/client_side_encryption.prose.17.on_demand_gcp.test.ts b/test/integration/client-side-encryption/client_side_encryption.prose.17.on_demand_gcp.test.ts
diff --git a/test/tools/runner/filters/client_encryption_filter.js b/test/tools/runner/filters/client_encryption_filter.js
diff --git a/test/types/encryption.test-d.ts b/test/types/encryption.test-d.ts
