diff --git a/dlp/inspect.js b/dlp/inspect.js index f30789f86b..a9fde80f30 100644 --- a/dlp/inspect.js +++ b/dlp/inspect.js @@ -15,167 +15,140 @@ 'use strict'; -const API_URL = 'https://dlp.googleapis.com/v2beta1'; const fs = require('fs'); -const requestPromise = require('request-promise'); const mime = require('mime'); const Buffer = require('safe-buffer').Buffer; -// Helper function to poll the rest API using exponential backoff -function pollJob (body, initialTimeout, tries, authToken) { - const jobName = body.name.split('/')[2]; - - // Construct polling function - const doPoll = (timeout, tries, resolve, reject) => { - // Construct REST request for polling an inspect job - const options = { - url: `${API_URL}/inspect/operations/${jobName}`, - headers: { - 'Authorization': `Bearer ${authToken}`, - 'Content-Type': 'application/json' - }, - json: true - }; - - // Poll the inspect job - setTimeout(() => { - requestPromise.get(options) - .then((body) => { - if (tries <= 0) { - reject('polling timed out'); - } - - // Job not finished - try again if possible - if (!(body && body.done)) { - return doPoll(timeout * 2, tries - 1, resolve, reject); - } +function inspectString (string, minLikelihood, maxFindings, infoTypes, includeQuote) { + // [START inspect_string] + // Imports the Google Cloud Data Loss Prevention library + const DLP = require('@google-cloud/dlp'); - // Job finished successfully! - return resolve(jobName); - }) - .catch((err) => { - reject(err); - }); - }, timeout); - }; + // Instantiates a client + const dlp = DLP(); - // Return job-polling REST request as a Promise - return new Promise((resolve, reject) => { - doPoll(initialTimeout, tries, resolve, reject); - }); -} + // The string to inspect + // const string = 'My name is Gary and my email is gary@example.com'; -// Helper function to get results of a long-running (polling-required) job -function getJobResults (authToken, jobName) { - // Construct REST request to get results of finished inspect job - const options = { - url: `${API_URL}/inspect/results/${jobName}/findings`, - headers: { - 'Authorization': `Bearer ${authToken}`, - 'Content-Type': 'application/json' - }, - json: true - }; + // The minimum likelihood required before returning a match + // const minLikelihood = LIKELIHOOD_UNSPECIFIED; - // Run job-results-fetching REST request - return requestPromise.get(options); -} + // The maximum number of findings to report (0 = server maximum) + // const maxFindings = 0; -function inspectString (authToken, string, inspectConfig) { - // [START inspect_string] - // Your gcloud auth token - // const authToken = 'YOUR_AUTH_TOKEN'; + // The infoTypes of information to match + // const infoTypes = ['US_MALE_NAME', 'US_FEMALE_NAME']; - // The string to inspect - // const string = 'My name is Gary and my email is gary@example.com'; + // Whether to include the matching string + // const includeQuote = true; // Construct items to inspect const items = [{ type: 'text/plain', value: string }]; - // Construct REST request body - const requestBody = { + // Construct request + const request = { inspectConfig: { - infoTypes: inspectConfig.infoTypes, - minLikelihood: inspectConfig.minLikelihood, - maxFindings: inspectConfig.maxFindings, - includeQuote: inspectConfig.includeQuote + infoTypes: infoTypes, + minLikelihood: minLikelihood, + maxFindings: maxFindings, + includeQuote: includeQuote }, items: items }; - // Construct REST request - const options = { - url: `${API_URL}/content:inspect`, - headers: { - 'Authorization': `Bearer ${authToken}`, - 'Content-Type': 'application/json' - }, - json: requestBody - }; - - // Run REST request - requestPromise.post(options) - .then((body) => { - const results = body.results[0].findings; - console.log(JSON.stringify(results, null, 2)); + // Run request + dlp.inspectContent(request) + .then((response) => { + const findings = response[0].results[0].findings; + if (findings.length > 0) { + console.log(`Findings:`); + findings.forEach((finding) => { + if (includeQuote) { + console.log(`\tQuote: ${finding.quote}`); + } + console.log(`\tInfo type: ${finding.infoType.name}`); + console.log(`\tLikelihood: ${finding.likelihood}`); + }); + } else { + console.log(`No findings.`); + } }) .catch((err) => { - console.log('Error in inspectString:', err); + console.log(`Error in inspectString: ${err.message || err}`); }); // [END inspect_string] } -function inspectFile (authToken, filepath, inspectConfig) { +function inspectFile (filepath, minLikelihood, maxFindings, infoTypes, includeQuote) { // [START inspect_file] - // Your gcloud auth token. - // const authToken = 'YOUR_AUTH_TOKEN'; + // Imports the Google Cloud Data Loss Prevention library + const DLP = require('@google-cloud/dlp'); + + // Instantiates a client + const dlp = DLP(); // The path to a local file to inspect. Can be a text, JPG, or PNG file. // const fileName = 'path/to/image.png'; + // The minimum likelihood required before returning a match + // const minLikelihood = LIKELIHOOD_UNSPECIFIED; + + // The maximum number of findings to report (0 = server maximum) + // const maxFindings = 0; + + // The infoTypes of information to match + // const infoTypes = ['US_MALE_NAME', 'US_FEMALE_NAME']; + + // Whether to include the matching string + // const includeQuote = true; + // Construct file data to inspect const fileItems = [{ type: mime.lookup(filepath) || 'application/octet-stream', data: Buffer.from(fs.readFileSync(filepath)).toString('base64') }]; - // Construct REST request body - const requestBody = { + // Construct request + const request = { inspectConfig: { - infoTypes: inspectConfig.infoTypes, - minLikelihood: inspectConfig.minLikelihood, - maxFindings: inspectConfig.maxFindings, - includeQuote: inspectConfig.includeQuote + infoTypes: infoTypes, + minLikelihood: minLikelihood, + maxFindings: maxFindings, + includeQuote: includeQuote }, items: fileItems }; - // Construct REST request - const options = { - url: `${API_URL}/content:inspect`, - headers: { - 'Authorization': `Bearer ${authToken}`, - 'Content-Type': 'application/json' - }, - json: requestBody - }; - - // Run REST request - requestPromise.post(options) - .then((body) => { - const results = body.results[0].findings; - console.log(JSON.stringify(results, null, 2)); + // Run request + dlp.inspectContent(request) + .then((response) => { + const findings = response[0].results[0].findings; + if (findings.length > 0) { + console.log(`Findings:`); + findings.forEach((finding) => { + if (includeQuote) { + console.log(`\tQuote: ${finding.quote}`); + } + console.log(`\tInfo type: ${finding.infoType.name}`); + console.log(`\tLikelihood: ${finding.likelihood}`); + }); + } else { + console.log(`No findings.`); + } }) .catch((err) => { - console.log('Error in inspectFile:', err); + console.log(`Error in inspectFile: ${err.message || err}`); }); // [END inspect_file] } -function inspectGCSFile (authToken, bucketName, fileName, inspectConfig) { - // [START inspect_gcs_file] - // Your gcloud auth token. - // const authToken = 'YOUR_AUTH_TOKEN'; +function promiseInspectGCSFile (bucketName, fileName, minLikelihood, maxFindings, infoTypes) { + // [START inspect_gcs_file_promise] + // Imports the Google Cloud Data Loss Prevention library + const DLP = require('@google-cloud/dlp'); + + // Instantiates a client + const dlp = DLP(); // The name of the bucket where the file resides. // const bucketName = 'YOUR-BUCKET'; @@ -184,6 +157,15 @@ function inspectGCSFile (authToken, bucketName, fileName, inspectConfig) { // Can contain wildcards, e.g. "my-image.*" // const fileName = 'my-image.png'; + // The minimum likelihood required before returning a match + // const minLikelihood = LIKELIHOOD_UNSPECIFIED; + + // The maximum number of findings to report (0 = server maximum) + // const maxFindings = 0; + + // The infoTypes of information to match + // const infoTypes = ['US_MALE_NAME', 'US_FEMALE_NAME']; + // Get reference to the file to be inspected const storageItems = { cloudStorageOptions: { @@ -192,56 +174,165 @@ function inspectGCSFile (authToken, bucketName, fileName, inspectConfig) { }; // Construct REST request body for creating an inspect job - const requestBody = { + const request = { inspectConfig: { - infoTypes: inspectConfig.infoTypes, - minLikelihood: inspectConfig.minLikelihood, - maxFindings: inspectConfig.maxFindings + infoTypes: infoTypes, + minLikelihood: minLikelihood, + maxFindings: maxFindings }, storageConfig: storageItems }; - // Construct REST request for creating an inspect job - let options = { - url: `${API_URL}/inspect/operations`, - headers: { - 'Authorization': `Bearer ${authToken}`, - 'Content-Type': 'application/json' + // Create a GCS File inspection job and wait for it to complete (using promises) + dlp.createInspectOperation(request) + .then((createJobResponse) => { + const operation = createJobResponse[0]; + + // Start polling for job completion + return operation.promise(); + }) + .then((completeJobResponse) => { + // When job is complete, get its results + const jobName = completeJobResponse[0].name; + return dlp.listInspectFindings({ + name: jobName + }); + }) + .then((results) => { + const findings = results[0].result.findings; + if (findings.length > 0) { + console.log(`Findings:`); + findings.forEach((finding) => { + console.log(`\tInfo type: ${finding.infoType.name}`); + console.log(`\tLikelihood: ${finding.likelihood}`); + }); + } else { + console.log(`No findings.`); + } + }) + .catch((err) => { + console.log(`Error in promiseInspectGCSFile: ${err.message || err}`); + }); + // [END inspect_gcs_file_promise] +} + +function eventInspectGCSFile (bucketName, fileName, minLikelihood, maxFindings, infoTypes) { + // [START inspect_gcs_file_event] + // Imports the Google Cloud Data Loss Prevention library + const DLP = require('@google-cloud/dlp'); + + // Instantiates a client + const dlp = DLP(); + + // The name of the bucket where the file resides. + // const bucketName = 'YOUR-BUCKET'; + + // The path to the file within the bucket to inspect. + // Can contain wildcards, e.g. "my-image.*" + // const fileName = 'my-image.png'; + + // The minimum likelihood required before returning a match + // const minLikelihood = LIKELIHOOD_UNSPECIFIED; + + // The maximum number of findings to report (0 = server maximum) + // const maxFindings = 0; + + // The infoTypes of information to match + // const infoTypes = ['US_MALE_NAME', 'US_FEMALE_NAME']; + + // Get reference to the file to be inspected + const storageItems = { + cloudStorageOptions: { + fileSet: { url: `gs://${bucketName}/${fileName}` } + } + }; + + // Construct REST request body for creating an inspect job + const request = { + inspectConfig: { + infoTypes: infoTypes, + minLikelihood: minLikelihood, + maxFindings: maxFindings }, - json: requestBody + storageConfig: storageItems }; - // Run inspect-job creation REST request - requestPromise.post(options) - .then((createBody) => pollJob(createBody, inspectConfig.initialTimeout, inspectConfig.tries, authToken)) - .then((jobName) => getJobResults(authToken, jobName)) - .then((findingsBody) => { - const findings = findingsBody.result.findings; - console.log(JSON.stringify(findings, null, 2)); + // Create a GCS File inspection job, and handle its completion (using event handlers) + // Promises are used (only) to avoid nested callbacks + dlp.createInspectOperation(request) + .then((createJobResponse) => { + const operation = createJobResponse[0]; + return new Promise((resolve, reject) => { + operation.on('complete', (completeJobResponse) => { + return resolve(completeJobResponse); + }); + + // Handle changes in job metadata (e.g. progress updates) + operation.on('progress', (metadata) => { + console.log(`Processed ${metadata.processedBytes} of approximately ${metadata.totalEstimatedBytes} bytes.`); + }); + + operation.on('error', (err) => { + return reject(err); + }); + }); + }) + .then((completeJobResponse) => { + const jobName = completeJobResponse.name; + return dlp.listInspectFindings({ + name: jobName + }); + }) + .then((results) => { + const findings = results[0].result.findings; + if (findings.length > 0) { + console.log(`Findings:`); + findings.forEach((finding) => { + console.log(`\tInfo type: ${finding.infoType.name}`); + console.log(`\tLikelihood: ${finding.likelihood}`); + }); + } else { + console.log(`No findings.`); + } }) .catch((err) => { - console.log('Error in inspectGCSFile:', err); + console.log(`Error in eventInspectGCSFile: ${err.message || err}`); }); - // [END inspect_gcs_file] + // [END inspect_gcs_file_event] } -function inspectDatastore (authToken, namespaceId, kind, inspectConfig) { +function inspectDatastore (projectId, namespaceId, kind, minLikelihood, maxFindings, infoTypes, includeQuote) { // [START inspect_datastore] - // Your gcloud auth token - // const authToken = 'YOUR_AUTH_TOKEN'; + // Imports the Google Cloud Data Loss Prevention library + const DLP = require('@google-cloud/dlp'); + + // Instantiates a client + const dlp = DLP(); + + // (Optional) The project ID containing the target Datastore + // const projectId = process.env.GCLOUD_PROJECT; // (Optional) The ID namespace of the Datastore document to inspect. // To ignore Datastore namespaces, set this to an empty string ('') - // const namespace = ''; + // const namespaceId = ''; // The kind of the Datastore entity to inspect. // const kind = 'Person'; + // The minimum likelihood required before returning a match + // const minLikelihood = LIKELIHOOD_UNSPECIFIED; + + // The maximum number of findings to report (0 = server maximum) + // const maxFindings = 0; + + // The infoTypes of information to match + // const infoTypes = ['US_MALE_NAME', 'US_FEMALE_NAME']; + // Get reference to the file to be inspected const storageItems = { datastoreOptions: { partitionId: { - projectId: inspectConfig.projectId, + projectId: projectId, namespaceId: namespaceId }, kind: { @@ -250,158 +341,165 @@ function inspectDatastore (authToken, namespaceId, kind, inspectConfig) { } }; - // Construct REST request body for creating an inspect job - const requestBody = { + // Construct request for creating an inspect job + const request = { inspectConfig: { - infoTypes: inspectConfig.infoTypes, - minLikelihood: inspectConfig.minLikelihood, - maxFindings: inspectConfig.maxFindings + infoTypes: infoTypes, + minLikelihood: minLikelihood, + maxFindings: maxFindings }, storageConfig: storageItems }; - // Construct REST request for creating an inspect job - let options = { - url: `${API_URL}/inspect/operations`, - headers: { - 'Authorization': `Bearer ${authToken}`, - 'Content-Type': 'application/json' - }, - json: requestBody - }; + // Run inspect-job creation request + dlp.createInspectOperation(request) + .then((createJobResponse) => { + const operation = createJobResponse[0]; - // Run inspect-job creation REST request - requestPromise.post(options) - .then((createBody) => pollJob(createBody, inspectConfig.initialTimeout, inspectConfig.tries, authToken)) - .then((jobName) => getJobResults(authToken, jobName)) - .then((findingsBody) => { - const findings = findingsBody.result.findings; - console.log(JSON.stringify(findings, null, 2)); + // Start polling for job completion + return operation.promise(); + }) + .then((completeJobResponse) => { + // When job is complete, get its results + const jobName = completeJobResponse[0].name; + return dlp.listInspectFindings({ + name: jobName + }); + }) + .then((results) => { + const findings = results[0].result.findings; + if (findings.length > 0) { + console.log(`Findings:`); + findings.forEach((finding) => { + console.log(`\tInfo type: ${finding.infoType.name}`); + console.log(`\tLikelihood: ${finding.likelihood}`); + }); + } else { + console.log(`No findings.`); + } }) .catch((err) => { - console.log('Error in inspectDatastore:', err); + console.log(`Error in inspectDatastore: ${err.message || err}`); }); // [END inspect_datastore] } -if (module === require.main) { - const auth = require('google-auto-auth')({ - keyFilename: process.env.GOOGLE_APPLICATION_CREDENTIALS, - scopes: ['https://www.googleapis.com/auth/cloud-platform'] - }); - auth.getToken((err, token) => { - if (err) { - console.err('Error fetching auth token:', err); - process.exit(1); +const cli = require(`yargs`) // eslint-disable-line + .demand(1) + .command( + `string `, + `Inspect a string using the Data Loss Prevention API.`, + {}, + (opts) => inspectString( + opts.string, + opts.minLikelihood, + opts.maxFindings, + opts.infoTypes, + opts.includeQuote + ) + ) + .command( + `file `, + `Inspects a local text, PNG, or JPEG file using the Data Loss Prevention API.`, + {}, + (opts) => inspectFile( + opts.filepath, + opts.minLikelihood, + opts.maxFindings, + opts.infoTypes, + opts.includeQuote + ) + ) + .command( + `gcsFilePromise `, + `Inspects a text file stored on Google Cloud Storage using the Data Loss Prevention API and the promise pattern.`, + {}, + (opts) => promiseInspectGCSFile( + opts.bucketName, + opts.fileName, + opts.minLikelihood, + opts.maxFindings, + opts.infoTypes + ) + ) + .command( + `gcsFileEvent `, + `Inspects a text file stored on Google Cloud Storage using the Data Loss Prevention API and the event-handler pattern.`, + {}, + (opts) => eventInspectGCSFile( + opts.bucketName, + opts.fileName, + opts.minLikelihood, + opts.maxFindings, + opts.infoTypes + ) + ) + .command( + `datastore `, + `Inspect a Datastore instance using the Data Loss Prevention API.`, + { + projectId: { + type: 'string', + alias: 'p', + default: process.env.GCLOUD_PROJECT + }, + namespaceId: { + type: 'string', + alias: 'n', + default: '' } + }, + (opts) => inspectDatastore(opts.projectId, opts.namespaceId, opts.kind, opts.minLikelihood, opts.maxFindings, opts.infoTypes, opts.includeQuote) + ) + .option('m', { + alias: 'minLikelihood', + default: 'LIKELIHOOD_UNSPECIFIED', + type: 'string', + choices: [ + 'LIKELIHOOD_UNSPECIFIED', + 'VERY_UNLIKELY', + 'UNLIKELY', + 'POSSIBLE', + 'LIKELY', + 'VERY_LIKELY' + ], + global: true + }) + .option('f', { + alias: 'maxFindings', + default: 0, + type: 'number', + global: true + }) + .option('q', { + alias: 'includeQuote', + default: true, + type: 'boolean', + global: true + }) + .option('l', { + alias: 'languageCode', + default: 'en-US', + type: 'string', + global: true + }) + .option('t', { + alias: 'infoTypes', + default: [], + type: 'array', + global: true, + coerce: (infoTypes) => infoTypes.map((type) => { + return { name: type }; + }) + }) + .example(`node $0 string "My phone number is (123) 456-7890 and my email address is me@somedomain.com"`) + .example(`node $0 file resources/test.txt`) + .example(`node $0 gcsFilePromise my-bucket my-file.txt`) + .example(`node $0 gcsFileEvent my-bucket my-file.txt`) + .wrap(120) + .recommendCommands() + .epilogue(`For more information, see https://cloud.google.com/dlp/docs. Optional flags are explained at https://cloud.google.com/dlp/docs/reference/rest/v2beta1/content/inspect#InspectConfig`); - require(`yargs`) // eslint-disable-line - .demand(1) - .command( - `string `, - `Inspect a string using the Data Loss Prevention API.`, - {}, - (opts) => inspectString(opts.authToken, opts.string, opts) - ) - .command( - `file `, - `Inspects a local text, PNG, or JPEG file using the Data Loss Prevention API.`, - {}, - (opts) => inspectFile(opts.authToken, opts.filepath, opts) - ) - .command( - `gcsFile `, - `Inspects a text file stored on Google Cloud Storage using the Data Loss Prevention API.`, - { - initialTimeout: { - type: 'number', - alias: 'i', - default: 5000 - }, - tries: { - type: 'number', - alias: 'r', - default: 5 - } - }, - (opts) => inspectGCSFile(opts.authToken, opts.bucketName, opts.fileName, opts) - ) - .command( - `datastore `, - `Inspect a Datastore instance using the Data Loss Prevention API.`, - { - projectId: { - type: 'string', - alias: 'p', - default: process.env.GCLOUD_PROJECT - }, - namespaceId: { - type: 'string', - alias: 'n', - default: '' - }, - initialTimeout: { - type: 'number', - alias: 'i', - default: 5000 - }, - tries: { - type: 'number', - alias: 'r', - default: 5 - } - }, - (opts) => inspectDatastore(opts.authToken, opts.namespaceId, opts.kind, opts) - ) - .option('m', { - alias: 'minLikelihood', - default: 'LIKELIHOOD_UNSPECIFIED', - type: 'string', - choices: [ - 'LIKELIHOOD_UNSPECIFIED', - 'VERY_UNLIKELY', - 'UNLIKELY', - 'POSSIBLE', - 'LIKELY', - 'VERY_LIKELY' - ], - global: true - }) - .option('f', { - alias: 'maxFindings', - default: 0, - type: 'number', - global: true - }) - .option('q', { - alias: 'includeQuote', - default: true, - type: 'boolean', - global: true - }) - .option('a', { - alias: 'authToken', - default: token, - type: 'string', - global: true - }) - .option('t', { - alias: 'infoTypes', - default: [], - type: 'array', - global: true, - coerce: (infoTypes) => infoTypes.map((type) => { - return { name: type }; - }) - }) - .example(`node $0 string "My phone number is (123) 456-7890 and my email address is me@somedomain.com"`) - .example(`node $0 file resources/test.txt`) - .example(`node $0 gcsFile my-bucket my-file.txt`) - .wrap(120) - .recommendCommands() - .epilogue(`For more information, see https://cloud.google.com/dlp/docs. Optional flags are explained at https://cloud.google.com/dlp/docs/reference/rest/v2beta1/content/inspect#InspectConfig`) - .help() - .strict() - .argv; - }); +if (module === require.main) { + cli.help().strict().argv; // eslint-disable-line } diff --git a/dlp/metadata.js b/dlp/metadata.js index 57b372114a..be492f5c03 100644 --- a/dlp/metadata.js +++ b/dlp/metadata.js @@ -15,102 +15,90 @@ 'use strict'; -const API_URL = 'https://dlp.googleapis.com/v2beta1'; -const requestPromise = require('request-promise'); - -function listInfoTypes (authToken, category) { +function listInfoTypes (category, languageCode) { // [START list_info_types] - // Your gcloud auth token. - // const authToken = 'YOUR_AUTH_TOKEN'; + // Imports the Google Cloud Data Loss Prevention library + const DLP = require('@google-cloud/dlp'); + + // Instantiates a client + const dlp = DLP(); // The category of info types to list. // const category = 'CATEGORY_TO_LIST'; - // Construct REST request - const options = { - url: `${API_URL}/rootCategories/${category}/infoTypes`, - headers: { - 'Authorization': `Bearer ${authToken}`, - 'Content-Type': 'application/json' - }, - json: true - }; + // The BCP-47 language code to use, e.g. 'en-US' + // const languageCode = 'en-US'; - // Run REST request - requestPromise.get(options) - .then((body) => { - console.log(body); - }) - .catch((err) => { - console.log('Error in listInfoTypes:', err); + dlp.listInfoTypes({ + category: category, + languageCode: languageCode + }) + .then((body) => { + const infoTypes = body[0].infoTypes; + console.log(`Info types for category ${category}:`); + infoTypes.forEach((infoType) => { + console.log(`\t${infoType.name} (${infoType.displayName})`); }); + }) + .catch((err) => { + console.log(`Error in listInfoTypes: ${err.message || err}`); + }); // [END list_info_types] } -function listCategories (authToken) { +function listRootCategories (languageCode) { // [START list_categories] - // Your gcloud auth token. - // const authToken = 'YOUR_AUTH_TOKEN'; + // Imports the Google Cloud Data Loss Prevention library + const DLP = require('@google-cloud/dlp'); - // Construct REST request - const options = { - url: `${API_URL}/rootCategories`, - headers: { - 'Authorization': `Bearer ${authToken}`, - 'Content-Type': 'application/json' - }, - json: true - }; + // Instantiates a client + const dlp = DLP(); - // Run REST request - requestPromise.get(options) - .then((body) => { - const categories = body.categories; - console.log(categories); - }) - .catch((err) => { - console.log('Error in listCategories:', err); + // The BCP-47 language code to use, e.g. 'en-US' + // const languageCode = 'en-US'; + + dlp.listRootCategories({ + languageCode: languageCode + }) + .then((body) => { + const categories = body[0].categories; + console.log(`Categories:`); + categories.forEach((category) => { + console.log(`\t${category.name}: ${category.displayName}`); }); + }) + .catch((err) => { + console.log(`Error in listRootCategories: ${err.message || err}`); + }); // [END list_categories] } -if (module === require.main) { - const auth = require('google-auto-auth')({ - keyFilename: process.env.GOOGLE_APPLICATION_CREDENTIALS, - scopes: ['https://www.googleapis.com/auth/cloud-platform'] - }); - auth.getToken((err, token) => { - if (err) { - console.err('Error fetching auth token:', err); - process.exit(1); - } - - const cli = require(`yargs`) - .demand(1) - .command( - `infoTypes `, - `List types of sensitive information within a category.`, - {}, - (opts) => listInfoTypes(opts.authToken, opts.category) - ) - .command( - `categories`, - `List root categories of sensitive information.`, - {}, - (opts) => listCategories(opts.authToken) - ) - .option('a', { - alias: 'authToken', - default: token, - type: 'string', - global: true - }) - .example(`node $0 infoTypes GOVERNMENT`) - .example(`node $0 categories`) - .wrap(120) - .recommendCommands() - .epilogue(`For more information, see https://cloud.google.com/dlp/docs`); +const cli = require(`yargs`) + .demand(1) + .command( + `infoTypes `, + `List types of sensitive information within a category.`, + {}, + (opts) => listInfoTypes(opts.category, opts.languageCode) + ) + .command( + `categories`, + `List root categories of sensitive information.`, + {}, + (opts) => listRootCategories(opts.languageCode) + ) + .option('l', { + alias: 'languageCode', + default: 'en-US', + type: 'string', + global: true + }) + .example(`node $0 infoTypes GOVERNMENT`) + .example(`node $0 categories`) + .wrap(120) + .recommendCommands() + .epilogue(`For more information, see https://cloud.google.com/dlp/docs`); - cli.help().strict().argv; // eslint-disable-line - }); +if (module === require.main) { + cli.help().strict().argv; // eslint-disable-line } diff --git a/dlp/package.json b/dlp/package.json index c7e43dec65..519047d9af 100644 --- a/dlp/package.json +++ b/dlp/package.json @@ -19,8 +19,10 @@ "test": "npm run system-test" }, "dependencies": { + "@google-cloud/dlp": "^0.1.0", "google-auth-library": "0.10.0", "google-auto-auth": "0.7.0", + "google-proto-files": "0.12.0", "mime": "1.3.6", "request": "2.81.0", "request-promise": "4.2.1", diff --git a/dlp/redact.js b/dlp/redact.js index 56e91810b4..ec1c03ddf2 100644 --- a/dlp/redact.js +++ b/dlp/redact.js @@ -15,13 +15,13 @@ 'use strict'; -const API_URL = 'https://dlp.googleapis.com/v2beta1'; -const requestPromise = require('request-promise'); - -function redactString (authToken, string, replaceString, inspectConfig) { +function redactString (string, replaceString, minLikelihood, infoTypes) { // [START redact_string] - // Your gcloud auth token - // const authToken = 'YOUR_AUTH_TOKEN'; + // Imports the Google Cloud Data Loss Prevention library + const DLP = require('@google-cloud/dlp'); + + // Instantiates a client + const dlp = DLP(); // The string to inspect // const string = 'My name is Gary and my email is gary@example.com'; @@ -29,102 +29,77 @@ function redactString (authToken, string, replaceString, inspectConfig) { // The string to replace sensitive data with // const replaceString = 'REDACTED'; - // Construct items to inspect + // The minimum likelihood required before redacting a match + // const minLikelihood = LIKELIHOOD_UNSPECIFIED; + + // The infoTypes of information to redact + // const infoTypes = ['US_MALE_NAME', 'US_FEMALE_NAME']; + const items = [{ type: 'text/plain', value: string }]; - // Construct info types + replacement configs - const replaceConfigs = inspectConfig.infoTypes.map((infoType) => { + const replaceConfigs = infoTypes.map((infoType) => { return { infoType: infoType, replaceWith: replaceString }; }); - // Construct REST request body - const requestBody = { + const request = { inspectConfig: { - infoTypes: inspectConfig.infoTypes, - minLikelihood: inspectConfig.minLikelihood + infoTypes: infoTypes, + minLikelihood: minLikelihood }, items: items, replaceConfigs: replaceConfigs }; - // Construct REST request - const options = { - url: `${API_URL}/content:redact`, - headers: { - 'Authorization': `Bearer ${authToken}`, - 'Content-Type': 'application/json' - }, - json: requestBody - }; - - // Run REST request - requestPromise.post(options) + dlp.redactContent(request) .then((body) => { - const results = body.items[0].value; + const results = body[0].items[0].value; console.log(results); }) .catch((err) => { - console.log('Error in redactString:', err); + console.log(`Error in redactString: ${err.message || err}`); }); // [END redact_string] } -if (module === require.main) { - const auth = require('google-auto-auth')({ - keyFilename: process.env.GOOGLE_APPLICATION_CREDENTIALS, - scopes: ['https://www.googleapis.com/auth/cloud-platform'] - }); - auth.getToken((err, token) => { - if (err) { - console.err('Error fetching auth token:', err); - process.exit(1); - } - - const cli = require(`yargs`) - .demand(1) - .command( - `string `, - `Redact sensitive data from a string using the Data Loss Prevention API.`, - {}, - (opts) => redactString(opts.authToken, opts.string, opts.replaceString, opts) - ) - .option('m', { - alias: 'minLikelihood', - default: 'LIKELIHOOD_UNSPECIFIED', - type: 'string', - choices: [ - 'LIKELIHOOD_UNSPECIFIED', - 'VERY_UNLIKELY', - 'UNLIKELY', - 'POSSIBLE', - 'LIKELY', - 'VERY_LIKELY' - ], - global: true - }) - .option('a', { - alias: 'authToken', - default: token, - type: 'string', - global: true - }) - .option('t', { - alias: 'infoTypes', - required: true, - type: 'array', - global: true, - coerce: (infoTypes) => infoTypes.map((type) => { - return { name: type }; - }) - }) - .example(`node $0 string "My name is Gary" "REDACTED" -t US_MALE_NAME`) - .wrap(120) - .recommendCommands() - .epilogue(`For more information, see https://cloud.google.com/dlp/docs. Optional flags are explained at https://cloud.google.com/dlp/docs/reference/rest/v2beta1/content/inspect#InspectConfig`); +const cli = require(`yargs`) + .demand(1) + .command( + `string `, + `Redact sensitive data from a string using the Data Loss Prevention API.`, + {}, + (opts) => redactString(opts.string, opts.replaceString, opts.minLikelihood, opts.infoTypes) + ) + .option('m', { + alias: 'minLikelihood', + default: 'LIKELIHOOD_UNSPECIFIED', + type: 'string', + choices: [ + 'LIKELIHOOD_UNSPECIFIED', + 'VERY_UNLIKELY', + 'UNLIKELY', + 'POSSIBLE', + 'LIKELY', + 'VERY_LIKELY' + ], + global: true + }) + .option('t', { + alias: 'infoTypes', + required: true, + type: 'array', + global: true, + coerce: (infoTypes) => infoTypes.map((type) => { + return { name: type }; + }) + }) + .example(`node $0 string "My name is Gary" "REDACTED" -t US_MALE_NAME`) + .wrap(120) + .recommendCommands() + .epilogue(`For more information, see https://cloud.google.com/dlp/docs. Optional flags are explained at https://cloud.google.com/dlp/docs/reference/rest/v2beta1/content/inspect#InspectConfig`); - cli.help().strict().argv; // eslint-disable-line - }); +if (module === require.main) { + cli.help().strict().argv; // eslint-disable-line } diff --git a/dlp/system-test/inspect.test.js b/dlp/system-test/inspect.test.js index d6c00d4b6d..246f52fd40 100644 --- a/dlp/system-test/inspect.test.js +++ b/dlp/system-test/inspect.test.js @@ -27,89 +27,106 @@ test.before(tools.checkCredentials); // inspect_string test(`should inspect a string`, async (t) => { const output = await tools.runAsync(`${cmd} string "I'm Gary and my email is gary@example.com"`, cwd); - t.regex(output, /"name": "EMAIL_ADDRESS"/); + t.regex(output, /Info type: EMAIL_ADDRESS/); }); test(`should handle a string with no sensitive data`, async (t) => { const output = await tools.runAsync(`${cmd} string "foo"`, cwd); - t.is(output, 'undefined'); + t.is(output, 'No findings.'); }); test(`should report string inspection handling errors`, async (t) => { - const output = await tools.runAsync(`${cmd} string "I'm Gary and my email is gary@example.com" -a foo`, cwd); + const output = await tools.runAsync(`${cmd} string "I'm Gary and my email is gary@example.com" -t BAD_TYPE`, cwd); t.regex(output, /Error in inspectString/); }); // inspect_file test(`should inspect a local text file`, async (t) => { const output = await tools.runAsync(`${cmd} file resources/test.txt`, cwd); - t.regex(output, /"name": "PHONE_NUMBER"/); - t.regex(output, /"name": "EMAIL_ADDRESS"/); + t.regex(output, /Info type: PHONE_NUMBER/); + t.regex(output, /Info type: EMAIL_ADDRESS/); }); test(`should inspect a local image file`, async (t) => { const output = await tools.runAsync(`${cmd} file resources/test.png`, cwd); - t.regex(output, /"name": "PHONE_NUMBER"/); + t.regex(output, /Info type: PHONE_NUMBER/); }); test(`should handle a local file with no sensitive data`, async (t) => { const output = await tools.runAsync(`${cmd} file resources/harmless.txt`, cwd); - t.is(output, 'undefined'); + t.is(output, 'No findings.'); }); test(`should report local file handling errors`, async (t) => { - const output = await tools.runAsync(`${cmd} file resources/harmless.txt -a foo`, cwd); + const output = await tools.runAsync(`${cmd} file resources/harmless.txt -t BAD_TYPE`, cwd); t.regex(output, /Error in inspectFile/); }); -// inspect_gcs_file -test.serial(`should inspect a GCS text file`, async (t) => { - const output = await tools.runAsync(`${cmd} gcsFile nodejs-docs-samples-dlp test.txt`, cwd); - t.regex(output, /"name": "PHONE_NUMBER"/); - t.regex(output, /"name": "EMAIL_ADDRESS"/); +// inspect_gcs_file_event +test.serial(`should inspect a GCS text file with event handlers`, async (t) => { + const output = await tools.runAsync(`${cmd} gcsFileEvent nodejs-docs-samples-dlp test.txt`, cwd); + t.regex(output, /Processed \d+ of approximately \d+ bytes./); + t.regex(output, /Info type: PHONE_NUMBER/); + t.regex(output, /Info type: EMAIL_ADDRESS/); }); -test.serial(`should inspect multiple GCS text files`, async (t) => { - const output = await tools.runAsync(`${cmd} gcsFile nodejs-docs-samples-dlp *.txt`, cwd); - t.regex(output, /"name": "PHONE_NUMBER"/); - t.regex(output, /"name": "EMAIL_ADDRESS"/); - t.regex(output, /"name": "CREDIT_CARD_NUMBER"/); +test.serial(`should inspect multiple GCS text files with event handlers`, async (t) => { + const output = await tools.runAsync(`${cmd} gcsFileEvent nodejs-docs-samples-dlp *.txt`, cwd); + t.regex(output, /Processed \d+ of approximately \d+ bytes./); + t.regex(output, /Info type: PHONE_NUMBER/); + t.regex(output, /Info type: EMAIL_ADDRESS/); + t.regex(output, /Info type: CREDIT_CARD_NUMBER/); }); -test.serial(`should accept try limits for inspecting GCS files`, async (t) => { - const output = await tools.runAsync(`${cmd} gcsFile nodejs-docs-samples-dlp test.txt --tries 0`, cwd); - t.regex(output, /polling timed out/); +test.serial(`should handle a GCS file with no sensitive data with event handlers`, async (t) => { + const output = await tools.runAsync(`${cmd} gcsFileEvent nodejs-docs-samples-dlp harmless.txt`, cwd); + t.regex(output, /Processed \d+ of approximately \d+ bytes./); + t.regex(output, /No findings./); }); -test.serial(`should handle a GCS file with no sensitive data`, async (t) => { - const output = await tools.runAsync(`${cmd} gcsFile nodejs-docs-samples-dlp harmless.txt`, cwd); - t.is(output, 'undefined'); +test.serial(`should report GCS file handling errors with event handlers`, async (t) => { + const output = await tools.runAsync(`${cmd} gcsFileEvent nodejs-docs-samples-dlp harmless.txt -t BAD_TYPE`, cwd); + t.regex(output, /Error in eventInspectGCSFile/); }); -test.serial(`should report GCS file handling errors`, async (t) => { - const output = await tools.runAsync(`${cmd} gcsFile nodejs-docs-samples-dlp harmless.txt -a foo`, cwd); - t.regex(output, /Error in inspectGCSFile/); +// inspect_gcs_file_promise +test.serial(`should inspect a GCS text file with promises`, async (t) => { + const output = await tools.runAsync(`${cmd} gcsFilePromise nodejs-docs-samples-dlp test.txt`, cwd); + t.regex(output, /Info type: PHONE_NUMBER/); + t.regex(output, /Info type: EMAIL_ADDRESS/); +}); + +test.serial(`should inspect multiple GCS text files with promises`, async (t) => { + const output = await tools.runAsync(`${cmd} gcsFilePromise nodejs-docs-samples-dlp *.txt`, cwd); + t.regex(output, /Info type: PHONE_NUMBER/); + t.regex(output, /Info type: EMAIL_ADDRESS/); + t.regex(output, /Info type: CREDIT_CARD_NUMBER/); +}); + +test.serial(`should handle a GCS file with no sensitive data with promises`, async (t) => { + const output = await tools.runAsync(`${cmd} gcsFilePromise nodejs-docs-samples-dlp harmless.txt`, cwd); + t.is(output, 'No findings.'); +}); + +test.serial(`should report GCS file handling errors with promises`, async (t) => { + const output = await tools.runAsync(`${cmd} gcsFilePromise nodejs-docs-samples-dlp harmless.txt -t BAD_TYPE`, cwd); + t.regex(output, /Error in promiseInspectGCSFile/); }); // inspect_datastore test.serial(`should inspect Datastore`, async (t) => { const output = await tools.runAsync(`${cmd} datastore Person --namespaceId DLP`, cwd); - t.regex(output, /"name": "PHONE_NUMBER"/); - t.regex(output, /"name": "EMAIL_ADDRESS"/); -}); - -test.serial(`should accept try limits for inspecting Datastore`, async (t) => { - const output = await tools.runAsync(`${cmd} datastore Person --namespaceId DLP --tries 0`, cwd); - t.regex(output, /polling timed out/); + t.regex(output, /Info type: PHONE_NUMBER/); + t.regex(output, /Info type: EMAIL_ADDRESS/); }); test.serial(`should handle Datastore with no sensitive data`, async (t) => { const output = await tools.runAsync(`${cmd} datastore Harmless --namespaceId DLP`, cwd); - t.is(output, 'undefined'); + t.is(output, 'No findings.'); }); test.serial(`should report Datastore file handling errors`, async (t) => { - const output = await tools.runAsync(`${cmd} datastore Harmless --namespaceId DLP -a foo`, cwd); + const output = await tools.runAsync(`${cmd} datastore Harmless --namespaceId DLP -t BAD_TYPE`, cwd); t.regex(output, /Error in inspectDatastore/); }); @@ -162,9 +179,3 @@ test(`should have an option to filter results by infoType`, async (t) => { t.notRegex(outputB, /EMAIL_ADDRESS/); t.regex(outputB, /PHONE_NUMBER/); }); - -test(`should have an option for custom auth tokens`, async (t) => { - const output = await tools.runAsync(`${cmd} string "My name is Gary and my phone number is (223) 456-7890." -a foo`, cwd); - t.regex(output, /Error in inspectString/); - t.regex(output, /invalid authentication/); -}); diff --git a/dlp/system-test/metadata.test.js b/dlp/system-test/metadata.test.js index 7cef1f3d01..086ab9cf22 100644 --- a/dlp/system-test/metadata.test.js +++ b/dlp/system-test/metadata.test.js @@ -26,27 +26,10 @@ test.before(tools.checkCredentials); test(`should list info types for a given category`, async (t) => { const output = await tools.runAsync(`${cmd} infoTypes GOVERNMENT`, cwd); - t.regex(output, /name: 'US_DRIVERS_LICENSE_NUMBER'/); + t.regex(output, /US_DRIVERS_LICENSE_NUMBER/); }); test(`should inspect categories`, async (t) => { const output = await tools.runAsync(`${cmd} categories`, cwd); - t.regex(output, /name: 'FINANCE'/); -}); - -test(`should have an option for custom auth tokens`, async (t) => { - const output = await tools.runAsync(`${cmd} categories -a foo`, cwd); - t.regex(output, /Error in listCategories/); - t.regex(output, /invalid authentication/); -}); - -// Error handling -test(`should report info type listing handling errors`, async (t) => { - const output = await tools.runAsync(`${cmd} infoTypes GOVERNMENT -a foo`, cwd); - t.regex(output, /Error in listInfoTypes/); -}); - -test(`should report category listing handling errors`, async (t) => { - const output = await tools.runAsync(`${cmd} categories -a foo`, cwd); - t.regex(output, /Error in listCategories/); + t.regex(output, /FINANCE/); }); diff --git a/dlp/system-test/redact.test.js b/dlp/system-test/redact.test.js index 7f844a01e1..1b053c6307 100644 --- a/dlp/system-test/redact.test.js +++ b/dlp/system-test/redact.test.js @@ -36,7 +36,7 @@ test(`should ignore unspecified type names when redacting from a string`, async }); test(`should report string redaction handling errors`, async (t) => { - const output = await tools.runAsync(`${cmd} string "My name is Gary and my phone number is (123) 456-7890." REDACTED -t PHONE_NUMBER -a foo`, cwd); + const output = await tools.runAsync(`${cmd} string "My name is Gary and my phone number is (123) 456-7890." REDACTED -t BAD_TYPE`, cwd); t.regex(output, /Error in redactString/); }); @@ -51,9 +51,3 @@ test(`should have a minLikelihood option`, async (t) => { const outputB = await promiseB; t.is(outputB, 'My phone number is REDACTED.'); }); - -test(`should have an option for custom auth tokens`, async (t) => { - const output = await tools.runAsync(`${cmd} string "My name is Gary and my phone number is (123) 456-7890." REDACTED -t PHONE_NUMBER -a foo`, cwd); - t.regex(output, /Error in redactString/); - t.regex(output, /invalid authentication/); -});