From 7b9c68f814e5f838caea3551efcffd455f95cf98 Mon Sep 17 00:00:00 2001 From: Justin Beckwith Date: Tue, 12 Feb 2019 10:00:59 -0800 Subject: [PATCH] refactor: modernize the sample tests (#237) --- dlp/package.json | 7 +- dlp/system-test/.eslintrc.yml | 2 - dlp/system-test/deid.test.js | 274 +++++++--------- dlp/system-test/inspect.test.js | 511 ++++++++++++----------------- dlp/system-test/jobs.test.js | 130 ++++---- dlp/system-test/metadata.test.js | 34 +- dlp/system-test/quickstart.test.js | 18 +- dlp/system-test/redact.test.js | 155 ++++----- dlp/system-test/risk.test.js | 363 ++++++++------------ dlp/system-test/templates.test.js | 140 ++++---- dlp/system-test/triggers.test.js | 117 +++---- 11 files changed, 747 insertions(+), 1004 deletions(-) diff --git a/dlp/package.json b/dlp/package.json index 87800b96be..44be97f10a 100644 --- a/dlp/package.json +++ b/dlp/package.json @@ -1,11 +1,13 @@ { "name": "dlp-samples", "description": "Code samples for Google Cloud Platform's Data Loss Prevention API", - "version": "0.0.1", "private": true, "license": "Apache-2.0", "author": "Google Inc.", "repository": "googleapis/nodejs-dlp", + "files": [ + "*.js" + ], "engines": { "node": ">=8" }, @@ -19,7 +21,8 @@ "yargs": "^12.0.1" }, "devDependencies": { - "@google-cloud/nodejs-repo-tools": "^3.0.0", + "chai": "^4.2.0", + "execa": "^1.0.0", "mocha": "^5.2.0", "pixelmatch": "^4.0.2", "pngjs": "^3.3.3", diff --git a/dlp/system-test/.eslintrc.yml b/dlp/system-test/.eslintrc.yml index 73f7bbc946..6db2a46c53 100644 --- a/dlp/system-test/.eslintrc.yml +++ b/dlp/system-test/.eslintrc.yml @@ -1,5 +1,3 @@ --- env: mocha: true -rules: - node/no-unpublished-require: off diff --git a/dlp/system-test/deid.test.js b/dlp/system-test/deid.test.js index 6746340a5a..98fb521a03 100644 --- a/dlp/system-test/deid.test.js +++ b/dlp/system-test/deid.test.js @@ -16,174 +16,138 @@ 'use strict'; const path = require('path'); -const assert = require('assert'); +const {assert} = require('chai'); const fs = require('fs'); -const tools = require('@google-cloud/nodejs-repo-tools'); +const execa = require('execa'); const cmd = 'node deid.js'; -const cwd = path.join(__dirname, '..'); - +const exec = async cmd => { + const res = await execa.shell(cmd); + if (res.stderr) { + throw new Error(res.stderr); + } + return res.stdout; +}; const harmfulString = 'My SSN is 372819127'; const harmlessString = 'My favorite color is blue'; - const surrogateType = 'SSN_TOKEN'; let labeledFPEString; - const wrappedKey = process.env.DLP_DEID_WRAPPED_KEY; const keyName = process.env.DLP_DEID_KEY_NAME; - const csvFile = 'resources/dates.csv'; const tempOutputFile = path.join(__dirname, 'temp.result.csv'); const csvContextField = 'name'; const dateShiftAmount = 30; const dateFields = 'birth_date register_date'; -before(tools.checkCredentials); - -// deidentify_masking -it('should mask sensitive data in a string', async () => { - const output = await tools.runAsync( - `${cmd} deidMask "${harmfulString}" -m x -n 5`, - cwd - ); - assert.strictEqual(output, 'My SSN is xxxxx9127'); -}); - -it('should ignore insensitive data when masking a string', async () => { - const output = await tools.runAsync( - `${cmd} deidMask "${harmlessString}"`, - cwd - ); - assert.strictEqual(output, harmlessString); -}); - -it('should handle masking errors', async () => { - const output = await tools.runAsync( - `${cmd} deidMask "${harmfulString}" -n -1`, - cwd - ); - assert.strictEqual( - new RegExp(/Error in deidentifyWithMask/).test(output), - true - ); -}); - -// deidentify_fpe -it('should FPE encrypt sensitive data in a string', async () => { - const output = await tools.runAsync( - `${cmd} deidFpe "${harmfulString}" ${wrappedKey} ${keyName} -a NUMERIC`, - cwd - ); - assert.strictEqual(new RegExp(/My SSN is \d{9}/).test(output), true); - assert.notStrictEqual(output, harmfulString); -}); - -it('should use surrogate info types in FPE encryption', async () => { - const output = await tools.runAsync( - `${cmd} deidFpe "${harmfulString}" ${wrappedKey} ${keyName} -a NUMERIC -s ${surrogateType}`, - cwd - ); - assert.strictEqual( - new RegExp(/My SSN is SSN_TOKEN\(9\):\d{9}/).test(output), - true - ); - labeledFPEString = output; -}); - -it('should ignore insensitive data when FPE encrypting a string', async () => { - const output = await tools.runAsync( - `${cmd} deidFpe "${harmlessString}" ${wrappedKey} ${keyName}`, - cwd - ); - assert.strictEqual(output, harmlessString); -}); - -it('should handle FPE encryption errors', async () => { - const output = await tools.runAsync( - `${cmd} deidFpe "${harmfulString}" ${wrappedKey} BAD_KEY_NAME`, - cwd - ); - assert.strictEqual( - new RegExp(/Error in deidentifyWithFpe/).test(output), - true - ); -}); - -// reidentify_fpe -it('should FPE decrypt surrogate-typed sensitive data in a string', async () => { - assert.ok(labeledFPEString, 'Verify that FPE encryption succeeded.'); - const output = await tools.runAsync( - `${cmd} reidFpe "${labeledFPEString}" ${surrogateType} ${wrappedKey} ${keyName} -a NUMERIC`, - cwd - ); - assert.strictEqual(output, harmfulString); -}); - -it('should handle FPE decryption errors', async () => { - const output = await tools.runAsync( - `${cmd} reidFpe "${harmfulString}" ${surrogateType} ${wrappedKey} BAD_KEY_NAME -a NUMERIC`, - cwd - ); - assert.strictEqual( - new RegExp(/Error in reidentifyWithFpe/).test(output), - true - ); -}); - -// deidentify_date_shift -it('should date-shift a CSV file', async () => { - const outputCsvFile = 'dates.actual.csv'; - const output = await tools.runAsync( - `${cmd} deidDateShift "${csvFile}" "${outputCsvFile}" ${dateShiftAmount} ${dateShiftAmount} ${dateFields}`, - cwd - ); - assert.strictEqual( - output.includes(`Successfully saved date-shift output to ${outputCsvFile}`), - true - ); - assert.notStrictEqual( - fs.readFileSync(outputCsvFile).toString(), - fs.readFileSync(csvFile).toString() - ); -}); - -it('should date-shift a CSV file using a context field', async () => { - const outputCsvFile = 'dates-context.actual.csv'; - const expectedCsvFile = - 'system-test/resources/date-shift-context.expected.csv'; - const output = await tools.runAsync( - `${cmd} deidDateShift "${csvFile}" "${outputCsvFile}" ${dateShiftAmount} ${dateShiftAmount} ${dateFields} -f ${csvContextField} -n ${keyName} -w ${wrappedKey}`, - cwd - ); - assert.strictEqual( - output.includes(`Successfully saved date-shift output to ${outputCsvFile}`), - true - ); - assert.strictEqual( - fs.readFileSync(outputCsvFile).toString(), - fs.readFileSync(expectedCsvFile).toString() - ); -}); - -it('should require all-or-none of {contextField, wrappedKey, keyName}', async () => { - const output = await tools.runAsync( - `${cmd} deidDateShift "${csvFile}" "${tempOutputFile}" ${dateShiftAmount} ${dateShiftAmount} ${dateFields} -f ${csvContextField} -n ${keyName}`, - cwd - ); - - assert.strictEqual( - output.includes('You must set either ALL or NONE of'), - true - ); -}); - -it('should handle date-shift errors', async () => { - const output = await tools.runAsync( - `${cmd} deidDateShift "${csvFile}" "${tempOutputFile}" ${dateShiftAmount} ${dateShiftAmount}`, - cwd - ); - assert.strictEqual( - new RegExp(/Error in deidentifyWithDateShift/).test(output), - true - ); +describe('deid', () => { + // deidentify_masking + it('should mask sensitive data in a string', async () => { + const output = await exec(`${cmd} deidMask "${harmfulString}" -m x -n 5`); + assert.strictEqual(output, 'My SSN is xxxxx9127'); + }); + + it('should ignore insensitive data when masking a string', async () => { + const output = await exec(`${cmd} deidMask "${harmlessString}"`); + assert.strictEqual(output, harmlessString); + }); + + it('should handle masking errors', async () => { + const output = await exec(`${cmd} deidMask "${harmfulString}" -n -1`); + assert.match(output, /Error in deidentifyWithMask/); + }); + + // deidentify_fpe + it('should FPE encrypt sensitive data in a string', async () => { + const output = await exec( + `${cmd} deidFpe "${harmfulString}" ${wrappedKey} ${keyName} -a NUMERIC` + ); + assert.match(output, /My SSN is \d{9}/); + assert.notStrictEqual(output, harmfulString); + }); + + it('should use surrogate info types in FPE encryption', async () => { + const output = await exec( + `${cmd} deidFpe "${harmfulString}" ${wrappedKey} ${keyName} -a NUMERIC -s ${surrogateType}` + ); + assert.match(output, /My SSN is SSN_TOKEN\(9\):\d{9}/); + labeledFPEString = output; + }); + + it('should ignore insensitive data when FPE encrypting a string', async () => { + const output = await exec( + `${cmd} deidFpe "${harmlessString}" ${wrappedKey} ${keyName}` + ); + assert.strictEqual(output, harmlessString); + }); + + it('should handle FPE encryption errors', async () => { + const output = await exec( + `${cmd} deidFpe "${harmfulString}" ${wrappedKey} BAD_KEY_NAME` + ); + assert.match(output, /Error in deidentifyWithFpe/); + }); + + // reidentify_fpe + it('should FPE decrypt surrogate-typed sensitive data in a string', async () => { + assert.ok(labeledFPEString, 'Verify that FPE encryption succeeded.'); + const output = await exec( + `${cmd} reidFpe "${labeledFPEString}" ${surrogateType} ${wrappedKey} ${keyName} -a NUMERIC` + ); + assert.strictEqual(output, harmfulString); + }); + + it('should handle FPE decryption errors', async () => { + const output = await exec( + `${cmd} reidFpe "${harmfulString}" ${surrogateType} ${wrappedKey} BAD_KEY_NAME -a NUMERIC` + ); + assert.match(output, /Error in reidentifyWithFpe/); + }); + + // deidentify_date_shift + it('should date-shift a CSV file', async () => { + const outputCsvFile = 'dates.actual.csv'; + const output = await exec( + `${cmd} deidDateShift "${csvFile}" "${outputCsvFile}" ${dateShiftAmount} ${dateShiftAmount} ${dateFields}` + ); + assert.match( + output, + new RegExp(`Successfully saved date-shift output to ${outputCsvFile}`) + ); + assert.notStrictEqual( + fs.readFileSync(outputCsvFile).toString(), + fs.readFileSync(csvFile).toString() + ); + }); + + it('should date-shift a CSV file using a context field', async () => { + const outputCsvFile = 'dates-context.actual.csv'; + const expectedCsvFile = + 'system-test/resources/date-shift-context.expected.csv'; + const output = await exec( + `${cmd} deidDateShift "${csvFile}" "${outputCsvFile}" ${dateShiftAmount} ${dateShiftAmount} ${dateFields} -f ${csvContextField} -n ${keyName} -w ${wrappedKey}` + ); + assert.match( + output, + new RegExp(`Successfully saved date-shift output to ${outputCsvFile}`) + ); + assert.strictEqual( + fs.readFileSync(outputCsvFile).toString(), + fs.readFileSync(expectedCsvFile).toString() + ); + }); + + it('should require all-or-none of {contextField, wrappedKey, keyName}', async () => { + const output = await exec( + `${cmd} deidDateShift "${csvFile}" "${tempOutputFile}" ${dateShiftAmount} ${dateShiftAmount} ${dateFields} -f ${csvContextField} -n ${keyName}` + ); + assert.match(output, /You must set either ALL or NONE of/); + }); + + it('should handle date-shift errors', async () => { + const output = await exec( + `${cmd} deidDateShift "${csvFile}" "${tempOutputFile}" ${dateShiftAmount} ${dateShiftAmount}` + ); + assert.match(output, /Error in deidentifyWithDateShift/); + }); }); diff --git a/dlp/system-test/inspect.test.js b/dlp/system-test/inspect.test.js index a7955f6045..30363884f3 100644 --- a/dlp/system-test/inspect.test.js +++ b/dlp/system-test/inspect.test.js @@ -15,305 +15,228 @@ 'use strict'; -const path = require('path'); -const assert = require('assert'); -const tools = require('@google-cloud/nodejs-repo-tools'); +const {assert} = require('chai'); +const execa = require('execa'); const {PubSub} = require('@google-cloud/pubsub'); const pubsub = new PubSub(); const uuid = require('uuid'); const cmd = 'node inspect.js'; -const cwd = path.join(__dirname, '..'); const bucket = 'nodejs-docs-samples-dlp'; const dataProject = 'nodejs-docs-samples'; - -// Create new custom topic/subscription -let topic, subscription; -const topicName = `dlp-inspect-topic-${uuid.v4()}`; -const subscriptionName = `dlp-inspect-subscription-${uuid.v4()}`; -before(async () => { - tools.checkCredentials(); - await pubsub - .createTopic(topicName) - .then(response => { - topic = response[0]; - return topic.createSubscription(subscriptionName); - }) - .then(response => { - subscription = response[0]; - }); -}); - -// Delete custom topic/subscription -after(async () => await subscription.delete().then(() => topic.delete())); - -// inspect_string -it('should inspect a string', async () => { - const output = await tools.runAsync( - `${cmd} string "I'm Gary and my email is gary@example.com"`, - cwd - ); - assert.strictEqual(new RegExp(/Info type: EMAIL_ADDRESS/).test(output), true); -}); - -it('should inspect a string with custom dictionary', async () => { - const output = await tools.runAsync( - `${cmd} string "I'm Gary and my email is gary@example.com" -d "Gary,email"`, - cwd - ); - assert.strictEqual(new RegExp(/Info type: CUSTOM_DICT_0/).test(output), true); -}); - -it('should inspect a string with custom regex', async () => { - const output = await tools.runAsync( - `${cmd} string "I'm Gary and my email is gary@example.com" -r "gary@example\\.com"`, - cwd - ); - assert.strictEqual( - new RegExp(/Info type: CUSTOM_REGEX_0/).test(output), - true - ); -}); - -it('should handle a string with no sensitive data', async () => { - const output = await tools.runAsync(`${cmd} string "foo"`, cwd); - assert.strictEqual(output, 'No findings.'); -}); - -it('should report string inspection handling errors', async () => { - const output = await tools.runAsync( - `${cmd} string "I'm Gary and my email is gary@example.com" -t BAD_TYPE`, - cwd - ); - assert.strictEqual(new RegExp(/Error in inspectString/).test(output), true); -}); - -// inspect_file -it('should inspect a local text file', async () => { - const output = await tools.runAsync(`${cmd} file resources/test.txt`, cwd); - assert.strictEqual(new RegExp(/Info type: PHONE_NUMBER/).test(output), true); - assert.strictEqual(new RegExp(/Info type: EMAIL_ADDRESS/).test(output), true); -}); - -it('should inspect a local text file with custom dictionary', async () => { - const output = await tools.runAsync( - `${cmd} file resources/test.txt -d "gary@somedomain.com"`, - cwd - ); - assert.strictEqual(new RegExp(/Info type: CUSTOM_DICT_0/).test(output), true); -}); - -it('should inspect a local text file with custom regex', async () => { - const output = await tools.runAsync( - `${cmd} file resources/test.txt -r "\\(\\d{3}\\) \\d{3}-\\d{4}"`, - cwd - ); - assert.strictEqual( - new RegExp(/Info type: CUSTOM_REGEX_0/).test(output), - true - ); -}); - -it('should inspect a local image file', async () => { - const output = await tools.runAsync(`${cmd} file resources/test.png`, cwd); - assert.strictEqual(new RegExp(/Info type: EMAIL_ADDRESS/).test(output), true); -}); - -it('should handle a local file with no sensitive data', async () => { - const output = await tools.runAsync( - `${cmd} file resources/harmless.txt`, - cwd - ); - assert.strictEqual(new RegExp(/No findings/).test(output), true); -}); - -it('should report local file handling errors', async () => { - const output = await tools.runAsync( - `${cmd} file resources/harmless.txt -t BAD_TYPE`, - cwd - ); - assert.strictEqual(new RegExp(/Error in inspectFile/).test(output), true); -}); - -// inspect_gcs_file_promise -it.skip('should inspect a GCS text file', async () => { - const output = await tools.runAsync( - `${cmd} gcsFile ${bucket} test.txt ${topicName} ${subscriptionName}`, - cwd - ); - assert.strictEqual( - new RegExp(/Found \d instance\(s\) of infoType PHONE_NUMBER/).test(output), - true - ); - assert.strictEqual( - new RegExp(/Found \d instance\(s\) of infoType EMAIL_ADDRESS/).test(output), - true - ); -}); - -it.skip('should inspect multiple GCS text files', async () => { - const output = await tools.runAsync( - `${cmd} gcsFile ${bucket} "*.txt" ${topicName} ${subscriptionName}`, - cwd - ); - assert.strictEqual( - new RegExp(/Found \d instance\(s\) of infoType PHONE_NUMBER/).test(output), - true - ); - assert.strictEqual( - new RegExp(/Found \d instance\(s\) of infoType EMAIL_ADDRESS/).test(output), - true - ); -}); - -it.skip('should handle a GCS file with no sensitive data', async () => { - const output = await tools.runAsync( - `${cmd} gcsFile ${bucket} harmless.txt ${topicName} ${subscriptionName}`, - cwd - ); - assert.strictEqual(new RegExp(/No findings/).test(output), true); -}); - -it('should report GCS file handling errors', async () => { - const output = await tools.runAsync( - `${cmd} gcsFile ${bucket} harmless.txt ${topicName} ${subscriptionName} -t BAD_TYPE`, - cwd - ); - assert.strictEqual(new RegExp(/Error in inspectGCSFile/).test(output), true); -}); - -// inspect_datastore -it.skip('should inspect Datastore', async () => { - const output = await tools.runAsync( - `${cmd} datastore Person ${topicName} ${subscriptionName} --namespaceId DLP -p ${dataProject}`, - cwd - ); - assert.strictEqual( - new RegExp(/Found \d instance\(s\) of infoType EMAIL_ADDRESS/).test(output), - true - ); -}); - -it.skip('should handle Datastore with no sensitive data', async () => { - const output = await tools.runAsync( - `${cmd} datastore Harmless ${topicName} ${subscriptionName} --namespaceId DLP -p ${dataProject}`, - cwd - ); - assert.strictEqual(new RegExp(/No findings/).test(output), true); -}); - -it('should report Datastore errors', async () => { - const output = await tools.runAsync( - `${cmd} datastore Harmless ${topicName} ${subscriptionName} --namespaceId DLP -t BAD_TYPE -p ${dataProject}`, - cwd - ); - assert.strictEqual( - new RegExp(/Error in inspectDatastore/).test(output), - true - ); -}); - -// inspect_bigquery -it.skip('should inspect a Bigquery table', async () => { - const output = await tools.runAsync( - `${cmd} bigquery integration_tests_dlp harmful ${topicName} ${subscriptionName} -p ${dataProject}`, - cwd - ); - assert.strictEqual( - new RegExp(/Found \d instance\(s\) of infoType PHONE_NUMBER/).test(output), - true - ); -}); - -it.skip('should handle a Bigquery table with no sensitive data', async () => { - const output = await tools.runAsync( - `${cmd} bigquery integration_tests_dlp harmless ${topicName} ${subscriptionName} -p ${dataProject}`, - cwd - ); - assert.strictEqual(new RegExp(/No findings/).test(output), true); -}); - -it('should report Bigquery table handling errors', async () => { - const output = await tools.runAsync( - `${cmd} bigquery integration_tests_dlp harmless ${topicName} ${subscriptionName} -t BAD_TYPE -p ${dataProject}`, - cwd - ); - assert.strictEqual(new RegExp(/Error in inspectBigquery/).test(output), true); -}); - -// CLI options -it('should have a minLikelihood option', async () => { - const promiseA = tools.runAsync( - `${cmd} string "My phone number is (123) 456-7890." -m VERY_LIKELY`, - cwd - ); - const promiseB = tools.runAsync( - `${cmd} string "My phone number is (123) 456-7890." -m UNLIKELY`, - cwd - ); - - const outputA = await promiseA; - assert.ok(outputA); - assert.strictEqual(new RegExp(/PHONE_NUMBER/).test(outputA), false); - - const outputB = await promiseB; - assert.strictEqual(new RegExp(/PHONE_NUMBER/).test(outputB), true); -}); - -it('should have a maxFindings option', async () => { - const promiseA = tools.runAsync( - `${cmd} string "My email is gary@example.com and my phone number is (223) 456-7890." -f 1`, - cwd - ); - const promiseB = tools.runAsync( - `${cmd} string "My email is gary@example.com and my phone number is (223) 456-7890." -f 2`, - cwd - ); - - const outputA = await promiseA; - assert.notStrictEqual( - outputA.includes('PHONE_NUMBER'), - outputA.includes('EMAIL_ADDRESS') - ); // Exactly one of these should be included - - const outputB = await promiseB; - assert.strictEqual(new RegExp(/PHONE_NUMBER/).test(outputB), true); - assert.strictEqual(new RegExp(/EMAIL_ADDRESS/).test(outputB), true); -}); - -it('should have an option to include quotes', async () => { - const promiseA = tools.runAsync( - `${cmd} string "My phone number is (223) 456-7890." -q false`, - cwd - ); - const promiseB = tools.runAsync( - `${cmd} string "My phone number is (223) 456-7890."`, - cwd - ); - - const outputA = await promiseA; - assert.ok(outputA); - assert.strictEqual(new RegExp(/\(223\) 456-7890/).test(outputA), false); - - const outputB = await promiseB; - assert.strictEqual(new RegExp(/\(223\) 456-7890/).test(outputB), true); -}); - -it('should have an option to filter results by infoType', async () => { - const promiseA = tools.runAsync( - `${cmd} string "My email is gary@example.com and my phone number is (223) 456-7890."`, - cwd - ); - const promiseB = tools.runAsync( - `${cmd} string "My email is gary@example.com and my phone number is (223) 456-7890." -t PHONE_NUMBER`, - cwd - ); - - const outputA = await promiseA; - assert.strictEqual(new RegExp(/EMAIL_ADDRESS/).test(outputA), true); - assert.strictEqual(new RegExp(/PHONE_NUMBER/).test(outputA), true); - - const outputB = await promiseB; - assert.strictEqual(new RegExp(/EMAIL_ADDRESS/).test(outputB), false); - assert.strictEqual(new RegExp(/PHONE_NUMBER/).test(outputB), true); +const exec = async cmd => (await execa.shell(cmd)).stdout; + +describe('inspect', () => { + // Create new custom topic/subscription + let topic, subscription; + const topicName = `dlp-inspect-topic-${uuid.v4()}`; + const subscriptionName = `dlp-inspect-subscription-${uuid.v4()}`; + before(async () => { + [topic] = await pubsub.createTopic(topicName); + [subscription] = await topic.createSubscription(subscriptionName); + }); + + // Delete custom topic/subscription + after(async () => { + await subscription.delete(); + await topic.delete(); + }); + + // inspect_string + it('should inspect a string', async () => { + const output = await exec( + `${cmd} string "I'm Gary and my email is gary@example.com"` + ); + assert.match(output, /Info type: EMAIL_ADDRESS/); + }); + + it('should inspect a string with custom dictionary', async () => { + const output = await exec( + `${cmd} string "I'm Gary and my email is gary@example.com" -d "Gary,email"` + ); + assert.match(output, /Info type: CUSTOM_DICT_0/); + }); + + it('should inspect a string with custom regex', async () => { + const output = await exec( + `${cmd} string "I'm Gary and my email is gary@example.com" -r "gary@example\\.com"` + ); + assert.match(output, /Info type: CUSTOM_REGEX_0/); + }); + + it('should handle a string with no sensitive data', async () => { + const output = await exec(`${cmd} string "foo"`); + assert.strictEqual(output, 'No findings.'); + }); + + it('should report string inspection handling errors', async () => { + const output = await exec( + `${cmd} string "I'm Gary and my email is gary@example.com" -t BAD_TYPE` + ); + assert.match(output, /Error in inspectString/); + }); + + // inspect_file + it('should inspect a local text file', async () => { + const output = await exec(`${cmd} file resources/test.txt`); + assert.match(output, /Info type: PHONE_NUMBER/); + assert.match(output, /Info type: EMAIL_ADDRESS/); + }); + + it('should inspect a local text file with custom dictionary', async () => { + const output = await exec( + `${cmd} file resources/test.txt -d "gary@somedomain.com"` + ); + assert.match(output, /Info type: CUSTOM_DICT_0/); + }); + + it('should inspect a local text file with custom regex', async () => { + const output = await exec( + `${cmd} file resources/test.txt -r "\\(\\d{3}\\) \\d{3}-\\d{4}"` + ); + assert.match(output, /Info type: CUSTOM_REGEX_0/); + }); + + it('should inspect a local image file', async () => { + const output = await exec(`${cmd} file resources/test.png`); + assert.match(output, /Info type: EMAIL_ADDRESS/); + }); + + it('should handle a local file with no sensitive data', async () => { + const output = await exec(`${cmd} file resources/harmless.txt`); + assert.match(output, /No findings/); + }); + + it('should report local file handling errors', async () => { + const output = await exec(`${cmd} file resources/harmless.txt -t BAD_TYPE`); + assert.match(output, /Error in inspectFile/); + }); + + // inspect_gcs_file_promise + it.skip('should inspect a GCS text file', async () => { + const output = await exec( + `${cmd} gcsFile ${bucket} test.txt ${topicName} ${subscriptionName}` + ); + assert.match(output, /Found \d instance\(s\) of infoType PHONE_NUMBER/); + assert.match(output, /Found \d instance\(s\) of infoType EMAIL_ADDRESS/); + }); + + it.skip('should inspect multiple GCS text files', async () => { + const output = await exec( + `${cmd} gcsFile ${bucket} "*.txt" ${topicName} ${subscriptionName}` + ); + assert.match(output, /Found \d instance\(s\) of infoType PHONE_NUMBER/); + assert.match(output, /Found \d instance\(s\) of infoType EMAIL_ADDRESS/); + }); + + it.skip('should handle a GCS file with no sensitive data', async () => { + const output = await exec( + `${cmd} gcsFile ${bucket} harmless.txt ${topicName} ${subscriptionName}` + ); + assert.match(output, /No findings/); + }); + + it('should report GCS file handling errors', async () => { + const output = await exec( + `${cmd} gcsFile ${bucket} harmless.txt ${topicName} ${subscriptionName} -t BAD_TYPE` + ); + assert.match(output, /Error in inspectGCSFile/); + }); + + // inspect_datastore + it.skip('should inspect Datastore', async () => { + const output = await exec( + `${cmd} datastore Person ${topicName} ${subscriptionName} --namespaceId DLP -p ${dataProject}` + ); + assert.match(output, /Found \d instance\(s\) of infoType EMAIL_ADDRESS/); + }); + + it.skip('should handle Datastore with no sensitive data', async () => { + const output = await exec( + `${cmd} datastore Harmless ${topicName} ${subscriptionName} --namespaceId DLP -p ${dataProject}` + ); + assert.match(output, /No findings/); + }); + + it('should report Datastore errors', async () => { + const output = await exec( + `${cmd} datastore Harmless ${topicName} ${subscriptionName} --namespaceId DLP -t BAD_TYPE -p ${dataProject}` + ); + assert.match(output, /Error in inspectDatastore/); + }); + + // inspect_bigquery + it.skip('should inspect a Bigquery table', async () => { + const output = await exec( + `${cmd} bigquery integration_tests_dlp harmful ${topicName} ${subscriptionName} -p ${dataProject}` + ); + assert.match(output, /Found \d instance\(s\) of infoType PHONE_NUMBER/); + }); + + it.skip('should handle a Bigquery table with no sensitive data', async () => { + const output = await exec( + `${cmd} bigquery integration_tests_dlp harmless ${topicName} ${subscriptionName} -p ${dataProject}` + ); + assert.match(output, /No findings/); + }); + + it('should report Bigquery table handling errors', async () => { + const output = await exec( + `${cmd} bigquery integration_tests_dlp harmless ${topicName} ${subscriptionName} -t BAD_TYPE -p ${dataProject}` + ); + assert.match(output, /Error in inspectBigquery/); + }); + + // CLI options + it('should have a minLikelihood option', async () => { + const outputA = await exec( + `${cmd} string "My phone number is (123) 456-7890." -m LIKELY` + ); + const outputB = await exec( + `${cmd} string "My phone number is (123) 456-7890." -m UNLIKELY` + ); + assert.ok(outputA); + assert.notMatch(outputA, /PHONE_NUMBER/); + assert.match(outputB, /PHONE_NUMBER/); + }); + + it('should have a maxFindings option', async () => { + const outputA = await exec( + `${cmd} string "My email is gary@example.com and my phone number is (223) 456-7890." -f 1` + ); + const outputB = await exec( + `${cmd} string "My email is gary@example.com and my phone number is (223) 456-7890." -f 2` + ); + assert.notStrictEqual( + outputA.includes('PHONE_NUMBER'), + outputA.includes('EMAIL_ADDRESS') + ); // Exactly one of these should be included + assert.match(outputB, /PHONE_NUMBER/); + assert.match(outputB, /EMAIL_ADDRESS/); + }); + + it('should have an option to include quotes', async () => { + const outputA = await exec( + `${cmd} string "My phone number is (223) 456-7890." -q false` + ); + const outputB = await exec( + `${cmd} string "My phone number is (223) 456-7890."` + ); + assert.ok(outputA); + assert.notMatch(outputA, /\(223\) 456-7890/); + assert.match(outputB, /\(223\) 456-7890/); + }); + + it('should have an option to filter results by infoType', async () => { + const outputA = await exec( + `${cmd} string "My email is gary@example.com and my phone number is (223) 456-7890."` + ); + const outputB = await exec( + `${cmd} string "My email is gary@example.com and my phone number is (223) 456-7890." -t PHONE_NUMBER` + ); + assert.match(outputA, /EMAIL_ADDRESS/); + assert.match(outputA, /PHONE_NUMBER/); + assert.notMatch(outputB, /EMAIL_ADDRESS/); + assert.match(outputB, /PHONE_NUMBER/); + }); }); diff --git a/dlp/system-test/jobs.test.js b/dlp/system-test/jobs.test.js index 165ebb02b6..5feff8d675 100644 --- a/dlp/system-test/jobs.test.js +++ b/dlp/system-test/jobs.test.js @@ -15,8 +15,8 @@ 'use strict'; -const assert = require('assert'); -const tools = require('@google-cloud/nodejs-repo-tools'); +const {assert} = require('chai'); +const execa = require('execa'); const cmd = `node jobs.js`; const badJobName = `projects/not-a-project/dlpJobs/i-123456789`; @@ -26,80 +26,80 @@ const testTableProjectId = `bigquery-public-data`; const testDatasetId = `san_francisco`; const testTableId = `bikeshare_trips`; const testColumnName = `zip_code`; +const exec = async cmd => (await execa.shell(cmd)).stdout; -// Helper function for creating test jobs -const createTestJob = async () => { - // Initialize client library - const DLP = require('@google-cloud/dlp').v2; - const dlp = new DLP.DlpServiceClient(); +describe('jobs', () => { + // Helper function for creating test jobs + const createTestJob = async () => { + // Initialize client library + const DLP = require('@google-cloud/dlp').v2; + const dlp = new DLP.DlpServiceClient(); - // Construct job request - const request = { - parent: dlp.projectPath(testCallingProjectId), - riskJob: { - privacyMetric: { - categoricalStatsConfig: { - field: { - name: testColumnName, + // Construct job request + const request = { + parent: dlp.projectPath(testCallingProjectId), + riskJob: { + privacyMetric: { + categoricalStatsConfig: { + field: { + name: testColumnName, + }, }, }, + sourceTable: { + projectId: testTableProjectId, + datasetId: testDatasetId, + tableId: testTableId, + }, }, - sourceTable: { - projectId: testTableProjectId, - datasetId: testDatasetId, - tableId: testTableId, - }, - }, + }; + + // Create job + return dlp.createDlpJob(request).then(response => { + return response[0].name; + }); }; - // Create job - return dlp.createDlpJob(request).then(response => { - return response[0].name; + // Create a test job + let testJobName; + before(async () => { + testJobName = await createTestJob(); }); -}; -// Create a test job -let testJobName; -before(async () => { - tools.checkCredentials(); - testJobName = await createTestJob(); -}); - -// dlp_list_jobs -it('should list jobs', async () => { - const output = await tools.runAsync(`${cmd} list 'state=DONE'`); - assert.strictEqual( - new RegExp(/Job projects\/(\w|-)+\/dlpJobs\/\w-\d+ status: DONE/).test( - output - ), - true - ); -}); + // dlp_list_jobs + it('should list jobs', async () => { + const output = await exec(`${cmd} list 'state=DONE'`); + assert.strictEqual( + new RegExp(/Job projects\/(\w|-)+\/dlpJobs\/\w-\d+ status: DONE/).test( + output + ), + true + ); + }); -it('should list jobs of a given type', async () => { - const output = await tools.runAsync( - `${cmd} list 'state=DONE' -t RISK_ANALYSIS_JOB` - ); - assert.strictEqual( - new RegExp(/Job projects\/(\w|-)+\/dlpJobs\/r-\d+ status: DONE/).test( - output - ), - true - ); -}); + it('should list jobs of a given type', async () => { + const output = await exec(`${cmd} list 'state=DONE' -t RISK_ANALYSIS_JOB`); + assert.strictEqual( + new RegExp(/Job projects\/(\w|-)+\/dlpJobs\/r-\d+ status: DONE/).test( + output + ), + true + ); + }); -it('should handle job listing errors', async () => { - const output = await tools.runAsync(`${cmd} list 'state=NOPE'`); - assert.strictEqual(new RegExp(/Error in listJobs/).test(output), true); -}); + it('should handle job listing errors', async () => { + const output = await exec(`${cmd} list 'state=NOPE'`); + assert.match(output, /Error in listJobs/); + }); -// dlp_delete_job -it('should delete job', async () => { - const output = await tools.runAsync(`${cmd} delete ${testJobName}`); - assert.strictEqual(output, `Successfully deleted job ${testJobName}.`); -}); + // dlp_delete_job + it('should delete job', async () => { + const output = await exec(`${cmd} delete ${testJobName}`); + assert.strictEqual(output, `Successfully deleted job ${testJobName}.`); + }); -it('should handle job deletion errors', async () => { - const output = await tools.runAsync(`${cmd} delete ${badJobName}`); - assert.strictEqual(new RegExp(/Error in deleteJob/).test(output), true); + it('should handle job deletion errors', async () => { + const output = await exec(`${cmd} delete ${badJobName}`); + assert.match(output, /Error in deleteJob/); + }); }); diff --git a/dlp/system-test/metadata.test.js b/dlp/system-test/metadata.test.js index fc497f1dd1..eaf0ab288c 100644 --- a/dlp/system-test/metadata.test.js +++ b/dlp/system-test/metadata.test.js @@ -15,30 +15,20 @@ 'use strict'; -const path = require('path'); -const assert = require('assert'); -const tools = require('@google-cloud/nodejs-repo-tools'); +const {assert} = require('chai'); +const execa = require('execa'); const cmd = 'node metadata.js'; -const cwd = path.join(__dirname, '..'); +const exec = async cmd => (await execa.shell(cmd)).stdout; -before(tools.checkCredentials); +describe('metadata', () => { + it('should list info types', async () => { + const output = await exec(`${cmd} infoTypes`); + assert.match(output, /US_DRIVERS_LICENSE_NUMBER/); + }); -it('should list info types', async () => { - const output = await tools.runAsync(`${cmd} infoTypes`, cwd); - assert.strictEqual( - new RegExp(/US_DRIVERS_LICENSE_NUMBER/).test(output), - true - ); -}); - -it('should filter listed info types', async () => { - const output = await tools.runAsync( - `${cmd} infoTypes "supported_by=RISK_ANALYSIS"`, - cwd - ); - assert.strictEqual( - new RegExp(/US_DRIVERS_LICENSE_NUMBER/).test(output), - false - ); + it('should filter listed info types', async () => { + const output = await exec(`${cmd} infoTypes "supported_by=RISK_ANALYSIS"`); + assert.notMatch(output, /US_DRIVERS_LICENSE_NUMBER/); + }); }); diff --git a/dlp/system-test/quickstart.test.js b/dlp/system-test/quickstart.test.js index 66ca8d4698..246fcb0754 100644 --- a/dlp/system-test/quickstart.test.js +++ b/dlp/system-test/quickstart.test.js @@ -15,16 +15,14 @@ 'use strict'; -const path = require('path'); -const assert = require('assert'); -const tools = require('@google-cloud/nodejs-repo-tools'); +const {assert} = require('chai'); +const execa = require('execa'); -const cmd = 'node quickstart.js'; -const cwd = path.join(__dirname, '..'); +const exec = async cmd => (await execa.shell(cmd)).stdout; -before(tools.checkCredentials); - -it('should run', async () => { - const output = await tools.runAsync(cmd, cwd); - assert.strictEqual(new RegExp(/Info type: PERSON_NAME/).test(output), true); +describe('quickstart', () => { + it('should run', async () => { + const output = await exec('node quickstart.js'); + assert.match(output, /Info type: PERSON_NAME/); + }); }); diff --git a/dlp/system-test/redact.test.js b/dlp/system-test/redact.test.js index 5443f82eac..54d0c3380b 100644 --- a/dlp/system-test/redact.test.js +++ b/dlp/system-test/redact.test.js @@ -15,22 +15,18 @@ 'use strict'; -const path = require('path'); -const assert = require('assert'); +const {assert} = require('chai'); const fs = require('fs'); -const tools = require('@google-cloud/nodejs-repo-tools'); -const PNG = require('pngjs').PNG; +const execa = require('execa'); +const {PNG} = require('pngjs'); const pixelmatch = require('pixelmatch'); const cmd = 'node redact.js'; -const cwd = path.join(__dirname, `..`); - const testImage = 'resources/test.png'; const testResourcePath = 'system-test/resources'; +const exec = async cmd => (await execa.shell(cmd)).stdout; -before(tools.checkCredentials); - -function readImage(filePath) { +async function readImage(filePath) { return new Promise((resolve, reject) => { fs.createReadStream(filePath) .pipe(new PNG()) @@ -56,94 +52,67 @@ async function getImageDiffPercentage(image1Path, image2Path) { return diffPixels / (diff.width * diff.height); } -// redact_text -it('should redact a single sensitive data type from a string', async () => { - const output = await tools.runAsync( - `${cmd} string "My email is jenny@example.com" -t EMAIL_ADDRESS`, - cwd - ); - assert.strictEqual( - new RegExp(/My email is \[EMAIL_ADDRESS\]/).test(output), - true - ); -}); - -it('should redact multiple sensitive data types from a string', async () => { - const output = await tools.runAsync( - `${cmd} string "I am 29 years old and my email is jenny@example.com" -t EMAIL_ADDRESS AGE`, - cwd - ); - assert.strictEqual( - new RegExp(/I am \[AGE\] and my email is \[EMAIL_ADDRESS\]/).test(output), - true - ); -}); - -it('should handle string with no sensitive data', async () => { - const output = await tools.runAsync( - `${cmd} string "No sensitive data to redact here" -t EMAIL_ADDRESS AGE`, - cwd - ); - assert.strictEqual( - new RegExp(/No sensitive data to redact here/).test(output), - true - ); -}); - -// redact_image -it('should redact a single sensitive data type from an image', async () => { - const testName = `redact-single-type`; - const output = await tools.runAsync( - `${cmd} image ${testImage} ${testName}.actual.png -t PHONE_NUMBER`, - cwd - ); - - assert.strictEqual( - new RegExp(/Saved image redaction results to path/).test(output), - true - ); +describe('redact', () => { + // redact_text + it('should redact a single sensitive data type from a string', async () => { + const output = await exec( + `${cmd} string "My email is jenny@example.com" -t EMAIL_ADDRESS` + ); + assert.match(output, /My email is \[EMAIL_ADDRESS\]/); + }); - const difference = await getImageDiffPercentage( - `${testName}.actual.png`, - `${testResourcePath}/${testName}.expected.png` - ); - assert.strictEqual(difference < 0.03, true); -}); + it('should redact multiple sensitive data types from a string', async () => { + const output = await exec( + `${cmd} string "I am 29 years old and my email is jenny@example.com" -t EMAIL_ADDRESS AGE` + ); + assert.match(output, /I am \[AGE\] and my email is \[EMAIL_ADDRESS\]/); + }); -it('should redact multiple sensitive data types from an image', async () => { - const testName = `redact-multiple-types`; - const output = await tools.runAsync( - `${cmd} image ${testImage} ${testName}.actual.png -t PHONE_NUMBER EMAIL_ADDRESS`, - cwd - ); + it('should handle string with no sensitive data', async () => { + const output = await exec( + `${cmd} string "No sensitive data to redact here" -t EMAIL_ADDRESS AGE` + ); + assert.match(output, /No sensitive data to redact here/); + }); - assert.strictEqual( - new RegExp(/Saved image redaction results to path/).test(output), - true - ); + // redact_image + it('should redact a single sensitive data type from an image', async () => { + const testName = `redact-single-type`; + const output = await exec( + `${cmd} image ${testImage} ${testName}.actual.png -t PHONE_NUMBER` + ); + assert.match(output, /Saved image redaction results to path/); + const difference = await getImageDiffPercentage( + `${testName}.actual.png`, + `${testResourcePath}/${testName}.expected.png` + ); + assert.isBelow(difference, 0.03); + }); - const difference = await getImageDiffPercentage( - `${testName}.actual.png`, - `${testResourcePath}/${testName}.expected.png` - ); - assert.strictEqual(difference < 0.03, true); -}); + it('should redact multiple sensitive data types from an image', async () => { + const testName = `redact-multiple-types`; + const output = await exec( + `${cmd} image ${testImage} ${testName}.actual.png -t PHONE_NUMBER EMAIL_ADDRESS` + ); + assert.match(output, /Saved image redaction results to path/); + const difference = await getImageDiffPercentage( + `${testName}.actual.png`, + `${testResourcePath}/${testName}.expected.png` + ); + assert.isBelow(difference, 0.03); + }); -it('should report info type errors', async () => { - const output = await tools.runAsync( - `${cmd} string "My email is jenny@example.com" -t NONEXISTENT`, - cwd - ); - assert.strictEqual( - new RegExp(/Error in deidentifyContent/).test(output), - true - ); -}); + it('should report info type errors', async () => { + const output = await exec( + `${cmd} string "My email is jenny@example.com" -t NONEXISTENT` + ); + assert.match(output, /Error in deidentifyContent/); + }); -it('should report image redaction handling errors', async () => { - const output = await tools.runAsync( - `${cmd} image ${testImage} output.png -t BAD_TYPE`, - cwd - ); - assert.strictEqual(new RegExp(/Error in redactImage/).test(output), true); + it('should report image redaction handling errors', async () => { + const output = await exec( + `${cmd} image ${testImage} output.png -t BAD_TYPE` + ); + assert.match(output, /Error in redactImage/); + }); }); diff --git a/dlp/system-test/risk.test.js b/dlp/system-test/risk.test.js index 4200655228..1d8e304bd4 100644 --- a/dlp/system-test/risk.test.js +++ b/dlp/system-test/risk.test.js @@ -15,235 +15,158 @@ 'use strict'; -const path = require('path'); -const assert = require('assert'); +const {assert} = require('chai'); const uuid = require('uuid'); const {PubSub} = require(`@google-cloud/pubsub`); -const pubsub = new PubSub(); -const tools = require('@google-cloud/nodejs-repo-tools'); +const execa = require('execa'); const cmd = 'node risk.js'; -const cwd = path.join(__dirname, '..'); - const dataset = 'integration_tests_dlp'; const uniqueField = 'Name'; const repeatedField = 'Mystery'; const numericField = 'Age'; const stringBooleanField = 'Gender'; const testProjectId = process.env.GCLOUD_PROJECT; - -// Create new custom topic/subscription -let topic, subscription; -const topicName = `dlp-risk-topic-${uuid.v4()}`; -const subscriptionName = `dlp-risk-subscription-${uuid.v4()}`; -before(async () => { - tools.checkCredentials(); - await pubsub - .createTopic(topicName) - .then(response => { - topic = response[0]; - return topic.createSubscription(subscriptionName); - }) - .then(response => { - subscription = response[0]; - }); -}); - -// Delete custom topic/subscription -after(async () => await subscription.delete().then(() => topic.delete())); - -// numericalRiskAnalysis -it('should perform numerical risk analysis', async () => { - const output = await tools.runAsync( - `${cmd} numerical ${dataset} harmful ${numericField} ${topicName} ${subscriptionName} -p ${testProjectId}`, - cwd - ); - assert.strictEqual( - new RegExp(/Value at 0% quantile: \d{2}/).test(output), - true - ); - assert.strictEqual( - new RegExp(/Value at \d{2}% quantile: \d{2}/).test(output), - true - ); -}); - -it('should handle numerical risk analysis errors', async () => { - const output = await tools.runAsync( - `${cmd} numerical ${dataset} nonexistent ${numericField} ${topicName} ${subscriptionName} -p ${testProjectId}`, - cwd - ); - assert.strictEqual( - new RegExp(/Error in numericalRiskAnalysis/).test(output), - true - ); -}); - -// categoricalRiskAnalysis -it('should perform categorical risk analysis on a string field', async () => { - const output = await tools.runAsync( - `${cmd} categorical ${dataset} harmful ${uniqueField} ${topicName} ${subscriptionName} -p ${testProjectId}`, - cwd - ); - assert.strictEqual( - new RegExp(/Most common value occurs \d time\(s\)/).test(output), - true - ); -}); - -it('should perform categorical risk analysis on a number field', async () => { - const output = await tools.runAsync( - `${cmd} categorical ${dataset} harmful ${numericField} ${topicName} ${subscriptionName} -p ${testProjectId}`, - cwd - ); - assert.strictEqual( - new RegExp(/Most common value occurs \d time\(s\)/).test(output), - true - ); -}); - -it('should handle categorical risk analysis errors', async () => { - const output = await tools.runAsync( - `${cmd} categorical ${dataset} nonexistent ${uniqueField} ${topicName} ${subscriptionName} -p ${testProjectId}`, - cwd - ); - assert.strictEqual( - new RegExp(/Error in categoricalRiskAnalysis/).test(output), - true - ); -}); - -// kAnonymityAnalysis -it('should perform k-anonymity analysis on a single field', async () => { - const output = await tools.runAsync( - `${cmd} kAnonymity ${dataset} harmful ${topicName} ${subscriptionName} ${numericField} -p ${testProjectId}`, - cwd - ); - assert.strictEqual( - new RegExp(/Quasi-ID values: \{\d{2}\}/).test(output), - true - ); - assert.strictEqual(new RegExp(/Class size: \d/).test(output), true); -}); - -it('should perform k-anonymity analysis on multiple fields', async () => { - const output = await tools.runAsync( - `${cmd} kAnonymity ${dataset} harmful ${topicName} ${subscriptionName} ${numericField} ${repeatedField} -p ${testProjectId}`, - cwd - ); - assert.strictEqual( - new RegExp(/Quasi-ID values: \{\d{2}, \d{4} \d{4} \d{4} \d{4}\}/).test( - output - ), - true - ); - assert.strictEqual(new RegExp(/Class size: \d/).test(output), true); -}); - -it('should handle k-anonymity analysis errors', async () => { - const output = await tools.runAsync( - `${cmd} kAnonymity ${dataset} nonexistent ${topicName} ${subscriptionName} ${numericField} -p ${testProjectId}`, - cwd - ); - assert.strictEqual( - new RegExp(/Error in kAnonymityAnalysis/).test(output), - true - ); -}); - -// kMapAnalysis -it('should perform k-map analysis on a single field', async () => { - const output = await tools.runAsync( - `${cmd} kMap ${dataset} harmful ${topicName} ${subscriptionName} ${numericField} -t AGE -p ${testProjectId}`, - cwd - ); - assert.strictEqual( - new RegExp(/Anonymity range: \[\d+, \d+\]/).test(output), - true - ); - assert.strictEqual(new RegExp(/Size: \d/).test(output), true); - assert.strictEqual(new RegExp(/Values: \d{2}/).test(output), true); -}); - -it('should perform k-map analysis on multiple fields', async () => { - const output = await tools.runAsync( - `${cmd} kMap ${dataset} harmful ${topicName} ${subscriptionName} ${numericField} ${stringBooleanField} -t AGE GENDER -p ${testProjectId}`, - cwd - ); - assert.strictEqual( - new RegExp(/Anonymity range: \[\d+, \d+\]/).test(output), - true - ); - assert.strictEqual(new RegExp(/Size: \d/).test(output), true); - assert.strictEqual(new RegExp(/Values: \d{2} Female/).test(output), true); -}); - -it('should handle k-map analysis errors', async () => { - const output = await tools.runAsync( - `${cmd} kMap ${dataset} nonexistent ${topicName} ${subscriptionName} ${numericField} -t AGE -p ${testProjectId}`, - cwd - ); - assert.strictEqual( - new RegExp(/Error in kMapEstimationAnalysis/).test(output), - true - ); -}); - -it('should check that numbers of quasi-ids and info types are equal', async () => { - const errors = await tools.runAsyncWithIO( - `${cmd} kMap ${dataset} nonexistent ${topicName} ${subscriptionName} ${numericField} -t AGE GENDER -p ${testProjectId}`, - cwd - ); - assert.strictEqual( - new RegExp( +const pubsub = new PubSub(); +const exec = async cmd => (await execa.shell(cmd)).stdout; + +describe('risk', () => { + // Create new custom topic/subscription + let topic, subscription; + const topicName = `dlp-risk-topic-${uuid.v4()}`; + const subscriptionName = `dlp-risk-subscription-${uuid.v4()}`; + before(async () => { + [topic] = await pubsub.createTopic(topicName); + [subscription] = await topic.createSubscription(subscriptionName); + }); + + // Delete custom topic/subscription + after(async () => { + await subscription.delete(); + await topic.delete(); + }); + + // numericalRiskAnalysis + it('should perform numerical risk analysis', async () => { + const output = await exec( + `${cmd} numerical ${dataset} harmful ${numericField} ${topicName} ${subscriptionName} -p ${testProjectId}` + ); + assert.match(output, /Value at 0% quantile: \d{2}/); + assert.match(output, /Value at \d{2}% quantile: \d{2}/); + }); + + it('should handle numerical risk analysis errors', async () => { + const output = await exec( + `${cmd} numerical ${dataset} nonexistent ${numericField} ${topicName} ${subscriptionName} -p ${testProjectId}` + ); + assert.match(output, /Error in numericalRiskAnalysis/); + }); + + // categoricalRiskAnalysis + it('should perform categorical risk analysis on a string field', async () => { + const output = await exec( + `${cmd} categorical ${dataset} harmful ${uniqueField} ${topicName} ${subscriptionName} -p ${testProjectId}` + ); + assert.match(output, /Most common value occurs \d time\(s\)/); + }); + + it('should perform categorical risk analysis on a number field', async () => { + const output = await exec( + `${cmd} categorical ${dataset} harmful ${numericField} ${topicName} ${subscriptionName} -p ${testProjectId}` + ); + assert.match(output, /Most common value occurs \d time\(s\)/); + }); + + it('should handle categorical risk analysis errors', async () => { + const output = await exec( + `${cmd} categorical ${dataset} nonexistent ${uniqueField} ${topicName} ${subscriptionName} -p ${testProjectId}` + ); + assert.match(output, /Error in categoricalRiskAnalysis/); + }); + + // kAnonymityAnalysis + it('should perform k-anonymity analysis on a single field', async () => { + const output = await exec( + `${cmd} kAnonymity ${dataset} harmful ${topicName} ${subscriptionName} ${numericField} -p ${testProjectId}` + ); + assert.match(output, /Quasi-ID values: \{\d{2}\}/); + assert.match(output, /Class size: \d/); + }); + + it('should perform k-anonymity analysis on multiple fields', async () => { + const output = await exec( + `${cmd} kAnonymity ${dataset} harmful ${topicName} ${subscriptionName} ${numericField} ${repeatedField} -p ${testProjectId}` + ); + assert.match(output, /Quasi-ID values: \{\d{2}, \d{4} \d{4} \d{4} \d{4}\}/); + assert.match(output, /Class size: \d/); + }); + + it('should handle k-anonymity analysis errors', async () => { + const output = await exec( + `${cmd} kAnonymity ${dataset} nonexistent ${topicName} ${subscriptionName} ${numericField} -p ${testProjectId}` + ); + assert.match(output, /Error in kAnonymityAnalysis/); + }); + + // kMapAnalysis + it('should perform k-map analysis on a single field', async () => { + const output = await exec( + `${cmd} kMap ${dataset} harmful ${topicName} ${subscriptionName} ${numericField} -t AGE -p ${testProjectId}` + ); + assert.match(output, /Anonymity range: \[\d+, \d+\]/); + assert.match(output, /Size: \d/); + assert.match(output, /Values: \d{2}/); + }); + + it('should perform k-map analysis on multiple fields', async () => { + const output = await exec( + `${cmd} kMap ${dataset} harmful ${topicName} ${subscriptionName} ${numericField} ${stringBooleanField} -t AGE GENDER -p ${testProjectId}` + ); + assert.match(output, /Anonymity range: \[\d+, \d+\]/); + assert.match(output, /Size: \d/); + assert.match(output, /Values: \d{2} Female/); + }); + + it('should handle k-map analysis errors', async () => { + const output = await exec( + `${cmd} kMap ${dataset} nonexistent ${topicName} ${subscriptionName} ${numericField} -t AGE -p ${testProjectId}` + ); + assert.match(output, /Error in kMapEstimationAnalysis/); + }); + + it('should check that numbers of quasi-ids and info types are equal', async () => { + const {stderr} = await execa.shell( + `${cmd} kMap ${dataset} nonexistent ${topicName} ${subscriptionName} ${numericField} -t AGE GENDER -p ${testProjectId}` + ); + assert.match( + stderr, /Number of infoTypes and number of quasi-identifiers must be equal!/ - ).test(errors.stderr), - true - ); -}); - -// lDiversityAnalysis -it('should perform l-diversity analysis on a single field', async () => { - const output = await tools.runAsync( - `${cmd} lDiversity ${dataset} harmful ${uniqueField} ${topicName} ${subscriptionName} ${numericField} -p ${testProjectId}`, - cwd - ); - assert.strictEqual( - new RegExp(/Quasi-ID values: \{\d{2}\}/).test(output), - true - ); - assert.strictEqual(new RegExp(/Class size: \d/).test(output), true); - assert.strictEqual( - new RegExp(/Sensitive value James occurs \d time\(s\)/).test(output), - true - ); -}); - -it('should perform l-diversity analysis on multiple fields', async () => { - const output = await tools.runAsync( - `${cmd} lDiversity ${dataset} harmful ${uniqueField} ${topicName} ${subscriptionName} ${numericField} ${repeatedField} -p ${testProjectId}`, - cwd - ); - assert.strictEqual( - new RegExp(/Quasi-ID values: \{\d{2}, \d{4} \d{4} \d{4} \d{4}\}/).test( - output - ), - true - ); - assert.strictEqual(new RegExp(/Class size: \d/).test(output), true); - assert.strictEqual( - new RegExp(/Sensitive value James occurs \d time\(s\)/).test(output), - true - ); -}); - -it('should handle l-diversity analysis errors', async () => { - const output = await tools.runAsync( - `${cmd} lDiversity ${dataset} nonexistent ${topicName} ${subscriptionName} ${numericField} -p ${testProjectId}`, - cwd - ); - assert.strictEqual( - new RegExp(/Error in lDiversityAnalysis/).test(output), - true - ); + ); + }); + + // lDiversityAnalysis + it('should perform l-diversity analysis on a single field', async () => { + const output = await exec( + `${cmd} lDiversity ${dataset} harmful ${uniqueField} ${topicName} ${subscriptionName} ${numericField} -p ${testProjectId}` + ); + assert.match(output, /Quasi-ID values: \{\d{2}\}/); + assert.match(output, /Class size: \d/); + assert.match(output, /Sensitive value James occurs \d time\(s\)/); + }); + + it('should perform l-diversity analysis on multiple fields', async () => { + const output = await exec( + `${cmd} lDiversity ${dataset} harmful ${uniqueField} ${topicName} ${subscriptionName} ${numericField} ${repeatedField} -p ${testProjectId}` + ); + assert.match(output, /Quasi-ID values: \{\d{2}, \d{4} \d{4} \d{4} \d{4}\}/); + assert.match(output, /Class size: \d/); + assert.match(output, /Sensitive value James occurs \d time\(s\)/); + }); + + it('should handle l-diversity analysis errors', async () => { + const output = await exec( + `${cmd} lDiversity ${dataset} nonexistent ${topicName} ${subscriptionName} ${numericField} -p ${testProjectId}` + ); + assert.match(output, /Error in lDiversityAnalysis/); + }); }); diff --git a/dlp/system-test/templates.test.js b/dlp/system-test/templates.test.js index abd4ddb022..96abcd8c10 100644 --- a/dlp/system-test/templates.test.js +++ b/dlp/system-test/templates.test.js @@ -15,92 +15,80 @@ 'use strict'; -const path = require('path'); -const assert = require('assert'); -const tools = require('@google-cloud/nodejs-repo-tools'); +const {assert} = require('chai'); +const execa = require('execa'); const uuid = require('uuid'); const cmd = 'node templates.js'; -const cwd = path.join(__dirname, '..'); const templateName = ''; +const exec = async cmd => (await execa.shell(cmd)).stdout; -const INFO_TYPE = 'PERSON_NAME'; -const MIN_LIKELIHOOD = 'VERY_LIKELY'; -const MAX_FINDINGS = 5; -const INCLUDE_QUOTE = false; -const DISPLAY_NAME = `My Template ${uuid.v4()}`; -const TEMPLATE_NAME = `my-template-${uuid.v4()}`; +describe('templates', () => { + const INFO_TYPE = 'PERSON_NAME'; + const MIN_LIKELIHOOD = 'VERY_LIKELY'; + const MAX_FINDINGS = 5; + const INCLUDE_QUOTE = false; + const DISPLAY_NAME = `My Template ${uuid.v4()}`; + const TEMPLATE_NAME = `my-template-${uuid.v4()}`; -const fullTemplateName = `projects/${ - process.env.GCLOUD_PROJECT -}/inspectTemplates/${TEMPLATE_NAME}`; + const fullTemplateName = `projects/${ + process.env.GCLOUD_PROJECT + }/inspectTemplates/${TEMPLATE_NAME}`; -// create_inspect_template -it('should create template', async () => { - const output = await tools.runAsync( - `${cmd} create -m ${MIN_LIKELIHOOD} -t ${INFO_TYPE} -f ${MAX_FINDINGS} -q ${INCLUDE_QUOTE} -d "${DISPLAY_NAME}" -i "${TEMPLATE_NAME}"`, - cwd - ); - assert.strictEqual( - output.includes(`Successfully created template ${fullTemplateName}`), - true - ); -}); + // create_inspect_template + it('should create template', async () => { + const output = await exec( + `${cmd} create -m ${MIN_LIKELIHOOD} -t ${INFO_TYPE} -f ${MAX_FINDINGS} -q ${INCLUDE_QUOTE} -d "${DISPLAY_NAME}" -i "${TEMPLATE_NAME}"` + ); + assert.match( + output, + new RegExp(`Successfully created template ${fullTemplateName}`) + ); + }); -it('should handle template creation errors', async () => { - const output = await tools.runAsync( - `${cmd} create -i invalid_template#id`, - cwd - ); - assert.strictEqual( - new RegExp(/Error in createInspectTemplate/).test(output), - true - ); -}); + it('should handle template creation errors', async () => { + const output = await exec(`${cmd} create -i invalid_template#id`); + assert.match(output, /Error in createInspectTemplate/); + }); -// list_inspect_templates -it('should list templates', async () => { - const output = await tools.runAsync(`${cmd} list`, cwd); - assert.strictEqual(output.includes(`Template ${templateName}`), true); - assert.strictEqual( - new RegExp(/Created: \d{1,2}\/\d{1,2}\/\d{4}/).test(output), - true - ); - assert.strictEqual( - new RegExp(/Updated: \d{1,2}\/\d{1,2}\/\d{4}/).test(output), - true - ); -}); + // list_inspect_templates + it('should list templates', async () => { + const output = await exec(`${cmd} list`); + assert.match(output, new RegExp(`Template ${templateName}`)); + assert.match(output, /Created: \d{1,2}\/\d{1,2}\/\d{4}/); + assert.match(output, /Updated: \d{1,2}\/\d{1,2}\/\d{4}/); + }); -it('should pass creation settings to template', async () => { - const output = await tools.runAsync(`${cmd} list`, cwd); - assert.strictEqual(output.includes(`Template ${fullTemplateName}`), true); - assert.strictEqual(output.includes(`Display name: ${DISPLAY_NAME}`), true); - assert.strictEqual(output.includes(`InfoTypes: ${INFO_TYPE}`), true); - assert.strictEqual( - output.includes(`Minimum likelihood: ${MIN_LIKELIHOOD}`), - true - ); - assert.strictEqual(output.includes(`Include quotes: ${INCLUDE_QUOTE}`), true); - assert.strictEqual( - output.includes(`Max findings per request: ${MAX_FINDINGS}`), - true - ); -}); + it('should pass creation settings to template', async () => { + const output = await exec(`${cmd} list`); + assert.strictEqual(output.includes(`Template ${fullTemplateName}`), true); + assert.strictEqual(output.includes(`Display name: ${DISPLAY_NAME}`), true); + assert.strictEqual(output.includes(`InfoTypes: ${INFO_TYPE}`), true); + assert.strictEqual( + output.includes(`Minimum likelihood: ${MIN_LIKELIHOOD}`), + true + ); + assert.strictEqual( + output.includes(`Include quotes: ${INCLUDE_QUOTE}`), + true + ); + assert.strictEqual( + output.includes(`Max findings per request: ${MAX_FINDINGS}`), + true + ); + }); -// delete_inspect_template -it('should delete template', async () => { - const output = await tools.runAsync(`${cmd} delete ${fullTemplateName}`, cwd); - assert.strictEqual( - output.includes(`Successfully deleted template ${fullTemplateName}.`), - true - ); -}); + // delete_inspect_template + it('should delete template', async () => { + const output = await exec(`${cmd} delete ${fullTemplateName}`); + assert.strictEqual( + output.includes(`Successfully deleted template ${fullTemplateName}.`), + true + ); + }); -it('should handle template deletion errors', async () => { - const output = await tools.runAsync(`${cmd} delete BAD_TEMPLATE`, cwd); - assert.strictEqual( - new RegExp(/Error in deleteInspectTemplate/).test(output), - true - ); + it('should handle template deletion errors', async () => { + const output = await exec(`${cmd} delete BAD_TEMPLATE`); + assert.match(output, /Error in deleteInspectTemplate/); + }); }); diff --git a/dlp/system-test/triggers.test.js b/dlp/system-test/triggers.test.js index 85825c5ab4..6cb19fe5d6 100644 --- a/dlp/system-test/triggers.test.js +++ b/dlp/system-test/triggers.test.js @@ -14,76 +14,63 @@ */ 'use strict'; -const path = require('path'); -const assert = require('assert'); -const tools = require('@google-cloud/nodejs-repo-tools'); -const uuid = require('uuid'); -const projectId = process.env.GCLOUD_PROJECT; -const cmd = 'node triggers.js'; -const cwd = path.join(__dirname, '..'); -const triggerName = `my-trigger-${uuid.v4()}`; -const fullTriggerName = `projects/${projectId}/jobTriggers/${triggerName}`; -const triggerDisplayName = `My Trigger Display Name: ${uuid.v4()}`; -const triggerDescription = `My Trigger Description: ${uuid.v4()}`; +const {assert} = require('chai'); +const execa = require('execa'); +const uuid = require('uuid'); -const infoType = 'US_CENSUS_NAME'; -const minLikelihood = 'VERY_LIKELY'; -const maxFindings = 5; -const bucketName = process.env.BUCKET_NAME; +describe('triggers', () => { + const projectId = process.env.GCLOUD_PROJECT; + const cmd = 'node triggers.js'; + const triggerName = `my-trigger-${uuid.v4()}`; + const fullTriggerName = `projects/${projectId}/jobTriggers/${triggerName}`; + const triggerDisplayName = `My Trigger Display Name: ${uuid.v4()}`; + const triggerDescription = `My Trigger Description: ${uuid.v4()}`; + const infoType = 'US_CENSUS_NAME'; + const minLikelihood = 'VERY_LIKELY'; + const maxFindings = 5; + const bucketName = process.env.BUCKET_NAME; + const exec = async cmd => (await execa.shell(cmd)).stdout; -it('should create a trigger', async () => { - const output = await tools.runAsync( - `${cmd} create ${bucketName} 1 -n ${triggerName} --autoPopulateTimespan \ - -m ${minLikelihood} -t ${infoType} -f ${maxFindings} -d "${triggerDisplayName}" -s "${triggerDescription}"`, - cwd - ); - assert.strictEqual( - output.includes(`Successfully created trigger ${fullTriggerName}`), - true - ); -}); + it('should create a trigger', async () => { + const output = await exec( + `${cmd} create ${bucketName} 1 -n ${triggerName} --autoPopulateTimespan \ + -m ${minLikelihood} -t ${infoType} -f ${maxFindings} -d "${triggerDisplayName}" -s "${triggerDescription}"` + ); + assert.match( + output, + new RegExp(`Successfully created trigger ${fullTriggerName}`) + ); + }); -it('should list triggers', async () => { - const output = await tools.runAsync(`${cmd} list`, cwd); - assert.strictEqual(output.includes(`Trigger ${fullTriggerName}`), true); - assert.strictEqual( - output.includes(`Display Name: ${triggerDisplayName}`), - true - ); - assert.strictEqual( - output.includes(`Description: ${triggerDescription}`), - true - ); - assert.strictEqual( - new RegExp(/Created: \d{1,2}\/\d{1,2}\/\d{4}/).test(output), - true - ); - assert.strictEqual( - new RegExp(/Updated: \d{1,2}\/\d{1,2}\/\d{4}/).test(output), - true - ); - assert.strictEqual(new RegExp(/Status: HEALTHY/).test(output), true); - assert.strictEqual(new RegExp(/Error count: 0/).test(output), true); -}); + it('should list triggers', async () => { + const output = await exec(`${cmd} list`); + assert.match(output, new RegExp(`Trigger ${fullTriggerName}`), true); + assert.match(output, new RegExp(`Display Name: ${triggerDisplayName}`)); + assert.match(output, new RegExp(`Description: ${triggerDescription}`)); + assert.match(output, /Created: \d{1,2}\/\d{1,2}\/\d{4}/); + assert.match(output, /Updated: \d{1,2}\/\d{1,2}\/\d{4}/); + assert.match(output, /Status: HEALTHY/); + assert.match(output, /Error count: 0/); + }); -it('should delete a trigger', async () => { - const output = await tools.runAsync(`${cmd} delete ${fullTriggerName}`, cwd); - assert.strictEqual( - output.includes(`Successfully deleted trigger ${fullTriggerName}.`), - true - ); -}); + it('should delete a trigger', async () => { + const output = await exec(`${cmd} delete ${fullTriggerName}`); + assert.match( + output, + new RegExp(`Successfully deleted trigger ${fullTriggerName}.`) + ); + }); -it('should handle trigger creation errors', async () => { - const output = await tools.runAsync( - `${cmd} create ${bucketName} 1 -n "@@@@@" -m ${minLikelihood} -t ${infoType} -f ${maxFindings}`, - cwd - ); - assert.strictEqual(new RegExp(/Error in createTrigger/).test(output), true); -}); + it('should handle trigger creation errors', async () => { + const output = await exec( + `${cmd} create ${bucketName} 1 -n "@@@@@" -m ${minLikelihood} -t ${infoType} -f ${maxFindings}` + ); + assert.match(output, /Error in createTrigger/); + }); -it('should handle trigger deletion errors', async () => { - const output = await tools.runAsync(`${cmd} delete bad-trigger-path`, cwd); - assert.strictEqual(new RegExp(/Error in deleteTrigger/).test(output), true); + it('should handle trigger deletion errors', async () => { + const output = await exec(`${cmd} delete bad-trigger-path`); + assert.match(output, /Error in deleteTrigger/); + }); });