Skip to content

Commit

Permalink
samples: Added custom dictionary and regex code samples (#204)
Browse files Browse the repository at this point in the history
  • Loading branch information
@mwdaub @JustinBeckwith
mwdaub authored and JustinBeckwith committed Dec 20, 2018
1 parent 1f8b9d3 commit 9c95491
Show file tree
Hide file tree
Showing 2 changed files with 105 additions and 6 deletions.
73 changes: 67 additions & 6 deletions dlp/inspect.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ async function inspectString(
minLikelihood,
maxFindings,
infoTypes,
customInfoTypes,
includeQuote
) {
// [START dlp_inspect_string]
Expand All @@ -45,6 +46,10 @@ async function inspectString(
// The infoTypes of information to match
// const infoTypes = [{ name: 'PHONE_NUMBER' }, { name: 'EMAIL_ADDRESS' }, { name: 'CREDIT_CARD_NUMBER' }];

// The customInfoTypes of information to match
// const customInfoTypes = [{ name: 'DICT_TYPE', dictionary: { wordList: { words: ['foo', 'bar', 'baz']}}},
// { name: 'REGEX_TYPE', regex: '\\(\\d{3}\\) \\d{3}-\\d{4}'}];

// Whether to include the matching string
// const includeQuote = true;

Expand All @@ -56,6 +61,7 @@ async function inspectString(
parent: dlp.projectPath(callingProjectId),
inspectConfig: {
infoTypes: infoTypes,
customInfoTypes: customInfoTypes,
minLikelihood: minLikelihood,
includeQuote: includeQuote,
limits: {
Expand Down Expand Up @@ -94,6 +100,7 @@ async function inspectFile(
minLikelihood,
maxFindings,
infoTypes,
customInfoTypes,
includeQuote
) {
// [START dlp_inspect_file]
Expand Down Expand Up @@ -122,6 +129,10 @@ async function inspectFile(
// The infoTypes of information to match
// const infoTypes = [{ name: 'PHONE_NUMBER' }, { name: 'EMAIL_ADDRESS' }, { name: 'CREDIT_CARD_NUMBER' }];

// The customInfoTypes of information to match
// const customInfoTypes = [{ name: 'DICT_TYPE', dictionary: { wordList: { words: ['foo', 'bar', 'baz']}}},
// { name: 'REGEX_TYPE', regex: '\\(\\d{3}\\) \\d{3}-\\d{4}'}];

// Whether to include the matching string
// const includeQuote = true;

Expand All @@ -143,6 +154,7 @@ async function inspectFile(
parent: dlp.projectPath(callingProjectId),
inspectConfig: {
infoTypes: infoTypes,
customInfoTypes: customInfoTypes,
minLikelihood: minLikelihood,
includeQuote: includeQuote,
limits: {
Expand Down Expand Up @@ -182,7 +194,8 @@ async function inspectGCSFile(
subscriptionId,
minLikelihood,
maxFindings,
infoTypes
infoTypes,
customInfoTypes
) {
// [START dlp_inspect_gcs]
// Import the Google Cloud client libraries
Expand Down Expand Up @@ -212,6 +225,10 @@ async function inspectGCSFile(
// The infoTypes of information to match
// const infoTypes = [{ name: 'PHONE_NUMBER' }, { name: 'EMAIL_ADDRESS' }, { name: 'CREDIT_CARD_NUMBER' }];

// The customInfoTypes of information to match
// const customInfoTypes = [{ name: 'DICT_TYPE', dictionary: { wordList: { words: ['foo', 'bar', 'baz']}}},
// { name: 'REGEX_TYPE', regex: '\\(\\d{3}\\) \\d{3}-\\d{4}'}];

// The name of the Pub/Sub topic to notify once the job completes
// TODO(developer): create a Pub/Sub topic to use for this
// const topicId = 'MY-PUBSUB-TOPIC'
Expand All @@ -234,6 +251,7 @@ async function inspectGCSFile(
inspectJob: {
inspectConfig: {
infoTypes: infoTypes,
customInfoTypes: customInfoTypes,
minLikelihood: minLikelihood,
limits: {
maxFindingsPerRequest: maxFindings,
Expand Down Expand Up @@ -316,7 +334,8 @@ async function inspectDatastore(
subscriptionId,
minLikelihood,
maxFindings,
infoTypes
infoTypes,
customInfoTypes
) {
// [START dlp_inspect_datastore]
// Import the Google Cloud client libraries
Expand Down Expand Up @@ -350,6 +369,10 @@ async function inspectDatastore(
// The infoTypes of information to match
// const infoTypes = [{ name: 'PHONE_NUMBER' }, { name: 'EMAIL_ADDRESS' }, { name: 'CREDIT_CARD_NUMBER' }];

// The customInfoTypes of information to match
// const customInfoTypes = [{ name: 'DICT_TYPE', dictionary: { wordList: { words: ['foo', 'bar', 'baz']}}},
// { name: 'REGEX_TYPE', regex: '\\(\\d{3}\\) \\d{3}-\\d{4}'}];

// The name of the Pub/Sub topic to notify once the job completes
// TODO(developer): create a Pub/Sub topic to use for this
// const topicId = 'MY-PUBSUB-TOPIC'
Expand Down Expand Up @@ -378,6 +401,7 @@ async function inspectDatastore(
inspectJob: {
inspectConfig: {
infoTypes: infoTypes,
customInfoTypes: customInfoTypes,
minLikelihood: minLikelihood,
limits: {
maxFindingsPerRequest: maxFindings,
Expand Down Expand Up @@ -458,7 +482,8 @@ async function inspectBigquery(
subscriptionId,
minLikelihood,
maxFindings,
infoTypes
infoTypes,
customInfoTypes
) {
// [START dlp_inspect_bigquery]
// Import the Google Cloud client libraries
Expand Down Expand Up @@ -491,6 +516,10 @@ async function inspectBigquery(
// The infoTypes of information to match
// const infoTypes = [{ name: 'PHONE_NUMBER' }, { name: 'EMAIL_ADDRESS' }, { name: 'CREDIT_CARD_NUMBER' }];

// The customInfoTypes of information to match
// const customInfoTypes = [{ name: 'DICT_TYPE', dictionary: { wordList: { words: ['foo', 'bar', 'baz']}}},
// { name: 'REGEX_TYPE', regex: '\\(\\d{3}\\) \\d{3}-\\d{4}'}];

// The name of the Pub/Sub topic to notify once the job completes
// TODO(developer): create a Pub/Sub topic to use for this
// const topicId = 'MY-PUBSUB-TOPIC'
Expand All @@ -517,6 +546,7 @@ async function inspectBigquery(
inspectJob: {
inspectConfig: {
infoTypes: infoTypes,
customInfoTypes: customInfoTypes,
minLikelihood: minLikelihood,
limits: {
maxFindingsPerRequest: maxFindings,
Expand Down Expand Up @@ -602,6 +632,7 @@ const cli = require(`yargs`) // eslint-disable-line
opts.minLikelihood,
opts.maxFindings,
opts.infoTypes,
opts.customDictionaries.concat(opts.customRegexes),
opts.includeQuote
)
)
Expand All @@ -616,6 +647,7 @@ const cli = require(`yargs`) // eslint-disable-line
opts.minLikelihood,
opts.maxFindings,
opts.infoTypes,
opts.customDictionaries.concat(opts.customRegexes),
opts.includeQuote
)
)
Expand All @@ -632,7 +664,8 @@ const cli = require(`yargs`) // eslint-disable-line
opts.subscriptionId,
opts.minLikelihood,
opts.maxFindings,
opts.infoTypes
opts.infoTypes,
opts.customDictionaries.concat(opts.customRegexes)
)
)
.command(
Expand All @@ -649,7 +682,8 @@ const cli = require(`yargs`) // eslint-disable-line
opts.subscriptionId,
opts.minLikelihood,
opts.maxFindings,
opts.infoTypes
opts.infoTypes,
opts.customDictionaries.concat(opts.customRegexes)
);
}
)
Expand All @@ -673,7 +707,8 @@ const cli = require(`yargs`) // eslint-disable-line
opts.subscriptionId,
opts.minLikelihood,
opts.maxFindings,
opts.infoTypes
opts.infoTypes,
opts.customDictionaries.concat(opts.customRegexes)
)
)
.option('m', {
Expand Down Expand Up @@ -722,6 +757,32 @@ const cli = require(`yargs`) // eslint-disable-line
return {name: type};
}),
})
.option('d', {
alias: 'customDictionaries',
default: [],
type: 'array',
global: true,
coerce: customDictionaries =>
customDictionaries.map((dict, idx) => {
return {
infoType: {name: 'CUSTOM_DICT_'.concat(idx.toString())},
dictionary: {wordList: {words: dict.split(',')}},
};
}),
})
.option('r', {
alias: 'customRegexes',
default: [],
type: 'array',
global: true,
coerce: customRegexes =>
customRegexes.map((rgx, idx) => {
return {
infoType: {name: 'CUSTOM_REGEX_'.concat(idx.toString())},
regex: {pattern: rgx},
};
}),
})
.option('n', {
alias: 'notificationTopic',
type: 'string',
Expand Down
38 changes: 38 additions & 0 deletions dlp/system-test/inspect.test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,25 @@ it('should inspect a string', async () => {
assert.strictEqual(new RegExp(/Info type: EMAIL_ADDRESS/).test(output), true);
});

it('should inspect a string with custom dictionary', async () => {
const output = await tools.runAsync(
`${cmd} string "I'm Gary and my email is gary@example.com" -d "Gary,email"`,
cwd
);
assert.strictEqual(new RegExp(/Info type: CUSTOM_DICT_0/).test(output), true);
});

it('should inspect a string with custom regex', async () => {
const output = await tools.runAsync(
`${cmd} string "I'm Gary and my email is gary@example.com" -r "gary@example\\.com"`,
cwd
);
assert.strictEqual(
new RegExp(/Info type: CUSTOM_REGEX_0/).test(output),
true
);
});

it('should handle a string with no sensitive data', async () => {
const output = await tools.runAsync(`${cmd} string "foo"`, cwd);
assert.strictEqual(output, 'No findings.');
Expand All @@ -76,6 +95,25 @@ it('should inspect a local text file', async () => {
assert.strictEqual(new RegExp(/Info type: EMAIL_ADDRESS/).test(output), true);
});

it('should inspect a local text file with custom dictionary', async () => {
const output = await tools.runAsync(
`${cmd} file resources/test.txt -d "gary@somedomain.com"`,
cwd
);
assert.strictEqual(new RegExp(/Info type: CUSTOM_DICT_0/).test(output), true);
});

it('should inspect a local text file with custom regex', async () => {
const output = await tools.runAsync(
`${cmd} file resources/test.txt -r "\\(\\d{3}\\) \\d{3}-\\d{4}"`,
cwd
);
assert.strictEqual(
new RegExp(/Info type: CUSTOM_REGEX_0/).test(output),
true
);
});

it('should inspect a local image file', async () => {
const output = await tools.runAsync(`${cmd} file resources/test.png`, cwd);
assert.strictEqual(new RegExp(/Info type: EMAIL_ADDRESS/).test(output), true);
Expand Down

0 comments on commit 9c95491

Please sign in to comment.