Skip to content

Commit 9c965a0

Browse files
author
Jesse Williamson
authored
Support Azure and GCP keystores in FLE (CXX-2111) (mongodb#848)
* Add gcpKMS test files. Add test .json to test_files.txt Signed-off-by: Jesse Williamson <jesse.williamson@mongodb.com> * Get test gcp credentials from environment. Signed-off-by: Jesse Williamson <jesse.williamson@mongodb.com> * Add Azure client-side encryption tests and data files. Signed-off-by: Jesse Williamson <jesse.williamson@mongodb.com> * Add prose tests. Signed-off-by: Jesse Williamson <jesse.williamson@mongodb.com> * Corpus changes: support Azure and GCP; formatting. Signed-off-by: Jesse Williamson <jesse.williamson@mongodb.com> * Add Azure and GCP corpus keys. Signed-off-by: Jesse Williamson <jesse.williamson@mongodb.com> * Update mci.yml for client-side encryption corpus tests. Signed-off-by: Jesse Williamson <jesse.williamson@mongodb.com> * Integrate client side encryption tests and helpers from azure-gcp.CXX-2111. Meld tests with Kevin's earlier PR's. Track down connection vs. SSL error and test failure. Signed-off-by: Jesse Williamson <jesse.williamson@mongodb.com> * Disable Power8 tests. Signed-off-by: Jesse Williamson <jesse.williamson@mongodb.com> * Add versioned API selection to failing tests. Signed-off-by: Jesse Williamson <jesse.williamson@mongodb.com> * Apply clang-format Signed-off-by: Jesse Williamson <jesse.williamson@mongodb.com> * Restore Power8 tests; move setting up environment into relevant script clause. Signed-off-by: Jesse Williamson <jesse.williamson@mongodb.com> * Remove +o xtrace from .mci.yml Signed-off-by: Jesse Williamson <jesse.williamson@mongodb.com> * Add set-virtualenv.sh script. Signed-off-by: Jesse Williamson <jesse.williamson@mongodb.com> * Remove set-virtualenv.sh Signed-off-by: Jesse Williamson <jesse.williamson@mongodb.com> * Restore Power8 tests; remove extra script references and variable setup. Signed-off-by: Jesse Williamson <jesse.williamson@mongodb.com> * Apply clang-format. Signed-off-by: Jesse Williamson <jesse.williamson@mongodb.com> * Remove GCP "endpoint" variable. Signed-off-by: Jesse Williamson <jesse.williamson@mongodb.com> * Remove "endpoint" variable for Azure. Signed-off-by: Jesse Williamson <jesse.williamson@mongodb.com> * Only set encryption test vars in one place; restore Power8 tests. Signed-off-by: Jesse Williamson <jesse.williamson@mongodb.com> * Fixup client side spec test (build). Signed-off-by: Jesse Williamson <jesse.williamson@mongodb.com> * Fixup test (remove endpoint capture). Signed-off-by: Jesse Williamson <jesse.williamson@mongodb.com> * Added checking for GCP and Azure enviornment variables needed to run encryption tests. Signed-off-by: Jesse Williamson <jesse.williamson@mongodb.com> * Amend comments to include new KMS providers. Signed-off-by: Jesse Williamson <jesse.williamson@mongodb.com> * clang-format Signed-off-by: Jesse Williamson <jesse.williamson@mongodb.com> * Change "example.com" to "doesnoteexist.invalid". See: mongodb#843 Signed-off-by: Jesse Williamson <jesse.williamson@mongodb.com> * Restore versioned server API usage. Signed-off-by: Jesse Williamson <jesse.williamson@mongodb.com> * Move tests from expecting old "parse error" to host resolution error. Signed-off-by: Jesse Williamson <jesse.williamson@mongodb.com> * Revert some clnag-format changes Signed-off-by: Jesse Williamson <jesse.williamson@mongodb.com> * Revert versioned API where one should not be selected. Signed-off-by: Jesse Williamson <jesse.williamson@mongodb.com> * More fiddling with test versioning. Signed-off-by: Jesse Williamson <jesse.williamson@mongodb.com> * clang-format client_side_encryption.cpp Signed-off-by: Jesse Williamson <jesse.williamson@mongodb.com> * Remove redundant add_test_api() Signed-off-by: Jesse Williamson <jesse.williamson@mongodb.com> * Require that all KMS enviornment variables be set to run client side encryption tests. Signed-off-by: Jesse Williamson <jesse.williamson@mongodb.com> * Rework test_util function for skipping or failing CSE tests. Signed-off-by: Jesse Williamson <jesse.williamson@mongodb.com>
1 parent 4b86d67 commit 9c965a0

18 files changed

+12149
-482
lines changed

.mci.yml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -362,6 +362,12 @@ functions:
362362
set +o errexit
363363
export MONGOCXX_TEST_AWS_SECRET_ACCESS_KEY="${cse_aws_secret_access_key}"
364364
export MONGOCXX_TEST_AWS_ACCESS_KEY_ID="${cse_aws_access_key_id}"
365+
export MONGOCXX_TEST_AZURE_TENANT_ID="${cse_azure_tenant_id}"
366+
export MONGOCXX_TEST_AZURE_CLIENT_ID="${cse_azure_client_id}"
367+
export MONGOCXX_TEST_AZURE_CLIENT_SECRET="${cse_azure_client_secret}"
368+
export MONGOCXX_TEST_GCP_EMAIL="${cse_gcp_email}"
369+
export MONGOCXX_TEST_GCP_PRIVATEKEY="${cse_gcp_privatekey}"
370+
365371
set -o errexit
366372
fi
367373

Lines changed: 222 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,222 @@
1+
{
2+
"runOn": [
3+
{
4+
"minServerVersion": "4.1.10"
5+
}
6+
],
7+
"database_name": "default",
8+
"collection_name": "default",
9+
"data": [],
10+
"json_schema": {
11+
"properties": {
12+
"encrypted_string_aws": {
13+
"encrypt": {
14+
"keyId": [
15+
{
16+
"$binary": {
17+
"base64": "AAAAAAAAAAAAAAAAAAAAAA==",
18+
"subType": "04"
19+
}
20+
}
21+
],
22+
"bsonType": "string",
23+
"algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
24+
}
25+
},
26+
"encrypted_string_azure": {
27+
"encrypt": {
28+
"keyId": [
29+
{
30+
"$binary": {
31+
"base64": "AZURE+AAAAAAAAAAAAAAAA==",
32+
"subType": "04"
33+
}
34+
}
35+
],
36+
"bsonType": "string",
37+
"algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
38+
}
39+
},
40+
"encrypted_string_gcp": {
41+
"encrypt": {
42+
"keyId": [
43+
{
44+
"$binary": {
45+
"base64": "GCP+AAAAAAAAAAAAAAAAAA==",
46+
"subType": "04"
47+
}
48+
}
49+
],
50+
"bsonType": "string",
51+
"algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
52+
}
53+
},
54+
"encrypted_string_local": {
55+
"encrypt": {
56+
"keyId": [
57+
{
58+
"$binary": {
59+
"base64": "AAAAAAAAAAAAAAAAAAAAAA==",
60+
"subType": "04"
61+
}
62+
}
63+
],
64+
"bsonType": "string",
65+
"algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
66+
}
67+
}
68+
},
69+
"bsonType": "object"
70+
},
71+
"key_vault_data": [
72+
{
73+
"_id": {
74+
"$binary": {
75+
"base64": "AZURE+AAAAAAAAAAAAAAAA==",
76+
"subType": "04"
77+
}
78+
},
79+
"keyMaterial": {
80+
"$binary": {
81+
"base64": "n+HWZ0ZSVOYA3cvQgP7inN4JSXfOH85IngmeQxRpQHjCCcqT3IFqEWNlrsVHiz3AELimHhX4HKqOLWMUeSIT6emUDDoQX9BAv8DR1+E1w4nGs/NyEneac78EYFkK3JysrFDOgl2ypCCTKAypkn9CkAx1if4cfgQE93LW4kczcyHdGiH36CIxrCDGv1UzAvERN5Qa47DVwsM6a+hWsF2AAAJVnF0wYLLJU07TuRHdMrrphPWXZsFgyV+lRqJ7DDpReKNO8nMPLV/mHqHBHGPGQiRdb9NoJo8CvokGz4+KE8oLwzKf6V24dtwZmRkrsDV4iOhvROAzz+Euo1ypSkL3mw==",
82+
"subType": "00"
83+
}
84+
},
85+
"creationDate": {
86+
"$date": {
87+
"$numberLong": "1601573901680"
88+
}
89+
},
90+
"updateDate": {
91+
"$date": {
92+
"$numberLong": "1601573901680"
93+
}
94+
},
95+
"status": {
96+
"$numberInt": "0"
97+
},
98+
"masterKey": {
99+
"provider": "azure",
100+
"keyVaultEndpoint": "key-vault-csfle.vault.azure.net",
101+
"keyName": "key-name-csfle"
102+
},
103+
"keyAltNames": [
104+
"altname",
105+
"azure_altname"
106+
]
107+
}
108+
],
109+
"tests": [
110+
{
111+
"description": "Insert a document with auto encryption using Azure KMS provider",
112+
"clientOptions": {
113+
"autoEncryptOpts": {
114+
"kmsProviders": {
115+
"azure": {}
116+
}
117+
}
118+
},
119+
"operations": [
120+
{
121+
"name": "insertOne",
122+
"arguments": {
123+
"document": {
124+
"_id": 1,
125+
"encrypted_string_azure": "string0"
126+
}
127+
}
128+
}
129+
],
130+
"expectations": [
131+
{
132+
"command_started_event": {
133+
"command": {
134+
"listCollections": 1,
135+
"filter": {
136+
"name": "default"
137+
}
138+
},
139+
"command_name": "listCollections"
140+
}
141+
},
142+
{
143+
"command_started_event": {
144+
"command": {
145+
"listCollections": 1,
146+
"filter": {
147+
"name": "datakeys"
148+
},
149+
"$db": "keyvault"
150+
},
151+
"command_name": "listCollections"
152+
}
153+
},
154+
{
155+
"command_started_event": {
156+
"command": {
157+
"find": "datakeys",
158+
"filter": {
159+
"$or": [
160+
{
161+
"_id": {
162+
"$in": [
163+
{
164+
"$binary": {
165+
"base64": "AZURE+AAAAAAAAAAAAAAAA==",
166+
"subType": "04"
167+
}
168+
}
169+
]
170+
}
171+
},
172+
{
173+
"keyAltNames": {
174+
"$in": []
175+
}
176+
}
177+
]
178+
},
179+
"$db": "keyvault"
180+
},
181+
"command_name": "find"
182+
}
183+
},
184+
{
185+
"command_started_event": {
186+
"command": {
187+
"insert": "default",
188+
"documents": [
189+
{
190+
"_id": 1,
191+
"encrypted_string_azure": {
192+
"$binary": {
193+
"base64": "AQGVERPgAAAAAAAAAAAAAAAC5DbBSwPwfSlBrDtRuglvNvCXD1KzDuCKY2P+4bRFtHDjpTOE2XuytPAUaAbXf1orsPq59PVZmsbTZbt2CB8qaQ==",
194+
"subType": "06"
195+
}
196+
}
197+
}
198+
],
199+
"ordered": true
200+
},
201+
"command_name": "insert"
202+
}
203+
}
204+
],
205+
"outcome": {
206+
"collection": {
207+
"data": [
208+
{
209+
"_id": 1,
210+
"encrypted_string_azure": {
211+
"$binary": {
212+
"base64": "AQGVERPgAAAAAAAAAAAAAAAC5DbBSwPwfSlBrDtRuglvNvCXD1KzDuCKY2P+4bRFtHDjpTOE2XuytPAUaAbXf1orsPq59PVZmsbTZbt2CB8qaQ==",
213+
"subType": "06"
214+
}
215+
}
216+
}
217+
]
218+
}
219+
}
220+
}
221+
]
222+
}
Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
runOn:
2+
- minServerVersion: "4.1.10"
3+
database_name: &database_name "default"
4+
collection_name: &collection_name "default"
5+
6+
data: []
7+
json_schema: {'properties': {'encrypted_string_aws': {'encrypt': {'keyId': [{'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}, 'encrypted_string_azure': {'encrypt': {'keyId': [{'$binary': {'base64': 'AZURE+AAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}, 'encrypted_string_gcp': {'encrypt': {'keyId': [{'$binary': {'base64': 'GCP+AAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}, 'encrypted_string_local': {'encrypt': {'keyId': [{'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}}, 'bsonType': 'object'}
8+
key_vault_data: [{'_id': {'$binary': {'base64': 'AZURE+AAAAAAAAAAAAAAAA==', 'subType': '04'}}, 'keyMaterial': {'$binary': {'base64': 'n+HWZ0ZSVOYA3cvQgP7inN4JSXfOH85IngmeQxRpQHjCCcqT3IFqEWNlrsVHiz3AELimHhX4HKqOLWMUeSIT6emUDDoQX9BAv8DR1+E1w4nGs/NyEneac78EYFkK3JysrFDOgl2ypCCTKAypkn9CkAx1if4cfgQE93LW4kczcyHdGiH36CIxrCDGv1UzAvERN5Qa47DVwsM6a+hWsF2AAAJVnF0wYLLJU07TuRHdMrrphPWXZsFgyV+lRqJ7DDpReKNO8nMPLV/mHqHBHGPGQiRdb9NoJo8CvokGz4+KE8oLwzKf6V24dtwZmRkrsDV4iOhvROAzz+Euo1ypSkL3mw==', 'subType': '00'}}, 'creationDate': {'$date': {'$numberLong': '1601573901680'}}, 'updateDate': {'$date': {'$numberLong': '1601573901680'}}, 'status': {'$numberInt': '0'}, 'masterKey': {'provider': 'azure', 'keyVaultEndpoint': 'key-vault-csfle.vault.azure.net', 'keyName': 'key-name-csfle'}, 'keyAltNames': ['altname', 'azure_altname']}]
9+
10+
tests:
11+
- description: "Insert a document with auto encryption using Azure KMS provider"
12+
clientOptions:
13+
autoEncryptOpts:
14+
kmsProviders:
15+
azure: {}
16+
operations:
17+
- name: insertOne
18+
arguments:
19+
document: &doc0 { _id: 1, encrypted_string_azure: "string0" }
20+
expectations:
21+
# Auto encryption will request the collection info.
22+
- command_started_event:
23+
command:
24+
listCollections: 1
25+
filter:
26+
name: *collection_name
27+
command_name: listCollections
28+
- command_started_event:
29+
command:
30+
listCollections: 1
31+
filter:
32+
name: "datakeys"
33+
$db: keyvault
34+
command_name: listCollections
35+
# Then key is fetched from the key vault.
36+
- command_started_event:
37+
command:
38+
find: datakeys
39+
filter: { $or: [ { _id: { $in: [ {'$binary': {'base64': 'AZURE+AAAAAAAAAAAAAAAA==', 'subType': '04'}} ] } }, { keyAltNames: { $in: [] } } ] }
40+
$db: keyvault
41+
command_name: find
42+
- command_started_event:
43+
command:
44+
insert: *collection_name
45+
documents:
46+
- &doc0_encrypted { _id: 1, encrypted_string_azure: {'$binary': {'base64': 'AQGVERPgAAAAAAAAAAAAAAAC5DbBSwPwfSlBrDtRuglvNvCXD1KzDuCKY2P+4bRFtHDjpTOE2XuytPAUaAbXf1orsPq59PVZmsbTZbt2CB8qaQ==', 'subType': '06'}} }
47+
ordered: true
48+
command_name: insert
49+
outcome:
50+
collection:
51+
# Outcome is checked using a separate MongoClient without auto encryption.
52+
data:
53+
- *doc0_encrypted

0 commit comments

Comments
 (0)