From bcebc7256bbbfc4ff7ee69ba0eb7494eb97e7666 Mon Sep 17 00:00:00 2001 From: Mike DaCosta Date: Thu, 16 Aug 2018 15:24:40 -0700 Subject: [PATCH] DLP sample: Add autoPopulateTimespan option for creating job triggers. (#64) --- dlp/.gitignore | 3 ++ dlp/package.json | 2 + dlp/system-test/deid.test.js | 10 ++-- dlp/system-test/redact.test.js | 51 ++++++++++++++---- ...ct.csv => date-shift-context.expected.csv} | 0 ...png => redact-multiple-types.expected.png} | Bin ...ct.png => redact-single-type.expected.png} | Bin .../resources/redact-single-type.output.png | Bin 14841 -> 0 bytes dlp/system-test/risk.test.js | 2 +- dlp/system-test/templates.test.js | 2 +- dlp/system-test/triggers.test.js | 3 +- dlp/triggers.js | 12 +++++ 12 files changed, 66 insertions(+), 19 deletions(-) create mode 100644 dlp/.gitignore rename dlp/system-test/resources/{date-shift-context.correct.csv => date-shift-context.expected.csv} (100%) rename dlp/system-test/resources/{redact-multiple-types.correct.png => redact-multiple-types.expected.png} (100%) rename dlp/system-test/resources/{redact-single-type.correct.png => redact-single-type.expected.png} (100%) delete mode 100644 dlp/system-test/resources/redact-single-type.output.png diff --git a/dlp/.gitignore b/dlp/.gitignore new file mode 100644 index 0000000000..df1525596c --- /dev/null +++ b/dlp/.gitignore @@ -0,0 +1,3 @@ +# Test outputs +*.actual.png +*.actual.csv diff --git a/dlp/package.json b/dlp/package.json index 13f93aa4f3..0be390708b 100644 --- a/dlp/package.json +++ b/dlp/package.json @@ -21,6 +21,8 @@ "devDependencies": { "@google-cloud/nodejs-repo-tools": "^2.3.0", "ava": "^0.25.0", + "pixelmatch": "^4.0.2", + "pngjs": "^3.3.3", "uuid": "^3.3.2" } } diff --git a/dlp/system-test/deid.test.js b/dlp/system-test/deid.test.js index f8adebf758..ee6d50e710 100644 --- a/dlp/system-test/deid.test.js +++ b/dlp/system-test/deid.test.js @@ -123,7 +123,7 @@ test(`should handle FPE decryption errors`, async t => { // deidentify_date_shift test(`should date-shift a CSV file`, async t => { - const outputCsvFile = path.join(__dirname, 'dates.result.csv'); + const outputCsvFile = 'dates.actual.csv'; const output = await tools.runAsync( `${cmd} deidDateShift "${csvFile}" "${outputCsvFile}" ${dateShiftAmount} ${dateShiftAmount} ${dateFields}`, cwd @@ -138,9 +138,9 @@ test(`should date-shift a CSV file`, async t => { }); test(`should date-shift a CSV file using a context field`, async t => { - const outputCsvFile = path.join(__dirname, 'dates-context.result.csv'); - const correctResultFile = - 'system-test/resources/date-shift-context.correct.csv'; + const outputCsvFile = 'dates-context.actual.csv'; + const expectedCsvFile = + 'system-test/resources/date-shift-context.expected.csv'; const output = await tools.runAsync( `${cmd} deidDateShift "${csvFile}" "${outputCsvFile}" ${dateShiftAmount} ${dateShiftAmount} ${dateFields} -f ${csvContextField} -n ${keyName} -w ${wrappedKey}`, cwd @@ -150,7 +150,7 @@ test(`should date-shift a CSV file using a context field`, async t => { ); t.is( fs.readFileSync(outputCsvFile).toString(), - fs.readFileSync(correctResultFile).toString() + fs.readFileSync(expectedCsvFile).toString() ); }); diff --git a/dlp/system-test/redact.test.js b/dlp/system-test/redact.test.js index 4071322bb9..5b88d6e062 100644 --- a/dlp/system-test/redact.test.js +++ b/dlp/system-test/redact.test.js @@ -19,6 +19,8 @@ const path = require('path'); const test = require('ava'); const fs = require('fs'); const tools = require('@google-cloud/nodejs-repo-tools'); +const PNG = require('pngjs').PNG; +const pixelmatch = require('pixelmatch'); const cmd = 'node redact.js'; const cwd = path.join(__dirname, `..`); @@ -28,6 +30,33 @@ const testResourcePath = 'system-test/resources'; test.before(tools.checkCredentials); +function readImage(filePath) { + return new Promise((resolve, reject) => { + fs.createReadStream(filePath) + .pipe(new PNG()) + .on('error', reject) + .on('parsed', function() { + resolve(this); + }); + }); +} + +async function getImageDiffPercentage(image1Path, image2Path) { + let image1 = await readImage(image1Path); + let image2 = await readImage(image2Path); + let diff = new PNG({width: image1.width, height: image1.height}); + + const diffPixels = pixelmatch( + image1.data, + image2.data, + diff.data, + image1.width, + image1.height + ); + return diffPixels / (diff.width * diff.height); +} + +// redact_text test(`should redact a single sensitive data type from a string`, async t => { const output = await tools.runAsync( `${cmd} string "My email is jenny@example.com" -t EMAIL_ADDRESS`, @@ -56,33 +85,33 @@ test(`should handle string with no sensitive data`, async t => { test(`should redact a single sensitive data type from an image`, async t => { const testName = `redact-single-type`; const output = await tools.runAsync( - `${cmd} image ${testImage} ${testName}.result.png -t PHONE_NUMBER`, + `${cmd} image ${testImage} ${testName}.actual.png -t PHONE_NUMBER`, cwd ); t.regex(output, /Saved image redaction results to path/); - const correct = fs.readFileSync( - `${testResourcePath}/${testName}.correct.png` + const difference = await getImageDiffPercentage( + `${testName}.actual.png`, + `${testResourcePath}/${testName}.expected.png` ); - const result = fs.readFileSync(`${testName}.result.png`); - t.deepEqual(correct, result); + t.true(difference < 0.03); }); test(`should redact multiple sensitive data types from an image`, async t => { const testName = `redact-multiple-types`; const output = await tools.runAsync( - `${cmd} image ${testImage} ${testName}.result.png -t PHONE_NUMBER EMAIL_ADDRESS`, + `${cmd} image ${testImage} ${testName}.actual.png -t PHONE_NUMBER EMAIL_ADDRESS`, cwd ); t.regex(output, /Saved image redaction results to path/); - const correct = fs.readFileSync( - `${testResourcePath}/${testName}.correct.png` + const difference = await getImageDiffPercentage( + `${testName}.actual.png`, + `${testResourcePath}/${testName}.expected.png` ); - const result = fs.readFileSync(`${testName}.result.png`); - t.deepEqual(correct, result); + t.true(difference < 0.03); }); test(`should report info type errors`, async t => { @@ -95,7 +124,7 @@ test(`should report info type errors`, async t => { test(`should report image redaction handling errors`, async t => { const output = await tools.runAsync( - `${cmd} image ${testImage} nonexistent.result.png -t BAD_TYPE`, + `${cmd} image ${testImage} output.png -t BAD_TYPE`, cwd ); t.regex(output, /Error in redactImage/); diff --git a/dlp/system-test/resources/date-shift-context.correct.csv b/dlp/system-test/resources/date-shift-context.expected.csv similarity index 100% rename from dlp/system-test/resources/date-shift-context.correct.csv rename to dlp/system-test/resources/date-shift-context.expected.csv diff --git a/dlp/system-test/resources/redact-multiple-types.correct.png b/dlp/system-test/resources/redact-multiple-types.expected.png similarity index 100% rename from dlp/system-test/resources/redact-multiple-types.correct.png rename to dlp/system-test/resources/redact-multiple-types.expected.png diff --git a/dlp/system-test/resources/redact-single-type.correct.png b/dlp/system-test/resources/redact-single-type.expected.png similarity index 100% rename from dlp/system-test/resources/redact-single-type.correct.png rename to dlp/system-test/resources/redact-single-type.expected.png diff --git a/dlp/system-test/resources/redact-single-type.output.png b/dlp/system-test/resources/redact-single-type.output.png deleted file mode 100644 index 838773deeccc368dbbf911814361d134dc447da2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 14841 zcma)@byQnH)9@4AwJi?8rIcdDU6bNYY4PF&Ep9XJ%*a{ALrWsjf_jM~w#n00<#q1#Q&tE!0ms4i@UO|0x|4 z03g~9QIOU3p4(f%M3yT~V!J%iGU{L<(AIo{{!~_x1p`C3QWN(nJ|@1bmYgDnq70`U zkcK9TNW6*$9sNnizPD6R&D2BrP?z5%@xJ6{w!8I|@08#0&9K0ahbC+55es<@$ulIl z;O8~T>4Xlp9jY}UJEx;Thd^D7f5R9Hy4x}BT6(y@SgAl=9@e+rK`#eP@5k zd%G#^40{E8xa^hwB>iwT()BX%C~75?;dc3OJm(%6zz+13*@N8UNt~^V=LB7k=TOg} z`!5;$?bS~EDbEI$tgkn*Kvi4pLJ?})M|Y32_=LsRO}PaWv?4XemTt>pd0#FSl)LsD zP6XP?vkoFBlla#K==)NuiuO_ru6%-2Pwkxh2bRo#NuBW>a8(RPN?3m4tT-=_(11=e zpqAwmG!iFz4ZUsR*9rge`}_yNOeE-T@uF3Dw|*(Gi*DZ~d}XS>^~|N@?Sg&79@JU9 zw6YU#FKEQRq)bR+ol}E~=f~f~^I_+}6Znepf$wx>548iqOP@ZKxrda9V_~_BZc#O* z2yQY_jCcsX4)4{~d;#Rt7?yl&pN&^Y(tIG7mh$V^Rs$vxc~}csFehLo1_lb3X` zJ9&%rm_CC3Uieu(ImVbt5s?ck5)U)Yt2maEsU^H}DwT|$tOt)Fb3>Tdk{?ncw*u}@ zav1h~M*N60ygSJ=tVzFBSYh6-@!^1r&y&2+$PTEPAA)Y?m*}%k5$uE`{u=_}kH)pv z(qGAxL(zAgMs>=3I4r2}2b^?fGx7D}CE<|C39wEMo(O+*FRzT1c2 z8SZQ^SBQgVCT4PO|FZ8L#0BhHc6rW2g-dIO#kzhgGb}XR+7)C8ao0|0cR85e%;jI} z_fh&y$r#`o=xO<=ax<0K+cX5S#y9Wi1ZsaLdxyjLb=U?FW=858Sn^J6J)brwO+A7` zK}7$AtJfefMd+1AJC8<6IS_oEb&M9T^0MWFq35hsxUttlEnh>lT03c9&ck?5bk)*H zR%35K@z^%L>1Ai-tTp`ASBp-)yz<0WG=b<>C-hK`llAem=|KH_?Mb(#Jm)?G*#+5a z=Xj}$A0u@8p7l+y`~*kG(Q zI{ZV#5zXS^r~EprQFLzN_`s_HQ~F_vlfjkAia?K~)m~cnL6`Gcn~@|j;Ps?|sU%9~ zIaG_c8^Xll^uGUGx=ED@4uT$T!Vdy3nx?hf<-hK33Ab#ibvc-vWK0pA3*bJ}D?@wb zXabU!MeFl7p(|YKG`FXj>Ekg^(#i{Gp8=Q4m2l!m#})o)|6N|KCs{ri zr4?o4JDK`S3A>JAof7B7gx={ya0uwR|rEIkC$*ckO2v|UdME!H>ud_nE4F1Cf} zk+6iD`mCpP`D|uQ&spEbEQl{%4X=EDUY<7H3o|u6T}#$cry>&l8$YFVGDw`e*DrbX zH!0k2VaKFPp8hz#=Y#ZBpT=;KOVvur#O-N;^a4wZRJQxDsM^UGg&=9}eLxZxvi-dd4K;DVLEN{uXQT zu=>+672-#Wg|npeE~E8W(x0T654@Vh=U7 zh-A%>mT4ocxdt2hoE>Pz`A!^0C`(v)|C48!>z_phgijUz>(66?|1tM`0#Y(V`Fncg zO$+PhFe>M2w~I@NynYNA8{Nc-Z5Imln%nC6UI#whT{z!avQg2TEp19&*qwaJ&HXNU zK4a;9*~K8`i2SXz*YPJtue~6r9Xco>zTaT!kb2EWH|UqBVW}RHEGf5;*Pk&uBj`i$ zL>v=XT03PF;|}M@Ojp|u((YWY(Rs91VKn){f}B8K^wpZ|v(lAVuVzbT9&G>#NoAj6 zVLwQ1eY2Zs(!j_rj}18C>{dTx5v4SU&EW4cfQ=O2Lyo&N%`cP(go9FcgkgLub6?d& zPEnEV^vlJ^m;Ngr&AnQr05Qgjlh>W5(R1TXc zIIrb=&Y=O-ZBa&HbqjsVed%VgK3IL8M2we1#g7l7tvWAt6saLdVji;n-Pq+{;fdl) zU#;gfE|bdk0|UGdaf6{qq&2m zXVD#`C$v%9&s8FrpZ6|tv^m{RT{*fd(;-UAoXFPZpyk3z)o2#MlW%4Md3&N%0?;SG zp82hZz15b~*Lw#0_vrpl=2}s)kY@32-%B`F{hP#Dkp?u>vC`FT`-O#q0wAPvwxZz% zi^g$uG1A`JVz!nz@Gz7HTqub{x>6NPlF=7I2gzcYeGHnyJkHu`f=o(ZBUlE2U!q-X zp`amGo9@Q@>z#VHwX?i$TkONb4rWClX{DY;R7Vkq+iP_nxNh!!V4t#?8<_!RV7VPedcE@WsbE=VhH<` zUl2r`Vs@EzSpbeX4VsWZdBEXJQyK#Z2>)T8lxUr)tGfVhYD^~go46f~=Ve=S^5=IR z3A#TChcBMyml5{srV3Fj6UbckbY7<6TgvWT823D^F+6xbXE>?jot(U86jjN3f+;F>{vU;Z=9sHs#oG`j)Xs>Dhb~#Rr_?7VpJ+WBd z{bv1z+$=hD(_w3q<@q|BJ#pgdk0uZv{~KN)TiHpFq5Y z%-ox4CCl)hSmBL0zdvgkAM*45`UQNW`m<{->@fH|5HCRWVf51lU*G)F{dpOI5k5Ia z%RhV%I?2VWVJ~`<`{PtoXX78f0XAz!+d)VO1Opv%wg9bPm(O+9lHtruR?v<347+FemgcZA=oG3v)7k;V>vw#L!mbSw z4Fi?}f{XN&brC!k=we)w8?s||7n1v~arsudAzdE|hgiUluft+otJxt7-j!#1^qvg3 zrExuIlzesIv8b1=4U>eF2+KK1yRE30vcNU@7aFbJb`G5!7T@7b(JDx0E@7p2)5B+& zNz>2VhbLB^bZHKeQ;!i-M{D>Dy_zc-a67uz@11dctC3aVHEkBJE8?&~shQKLsrD^i zS4au~!?iG=Hq?$dm2nZNX(JRHh5uQL6)$gz^Q&nBBnBv0I5!{l>}A0sGs0wQeZ|UnHT|vz!gr?C5#Xya+gk7cqzS2x)W|fJCV9fClN$O>%u*jQ z-f|FdU_GGd%NDl;F;HicnQ|HnMaunI%cG43f+ya{zD}&pi!o^4g_%V=yTj)N?Yf$I zr*JZTONkP3EqSvM*^N7idI&o~4A3i{9y$SB%IXe8IWTE)TcBW$Qii4kmK?MDFdD-* zDNT@u6!c81^br&ev}7+arC&M**Sz^OC$@p#SB>@)W{P zW@w^d{!P@f*y!U-VV%tD?F`4H*o3q{fT0T*STZ#=mCS|v!bLmpB_tx@wK|$hzk*!c zq_F}9FHSZmNgov~@xDPaj?H3y{jaS`5W36~@QK_h{g7-ExV3Rl>qp&NBj4pyCOf7g zQhr={-voPIud$?Q!R~F@(FxrvJ|g$emum`^^V3;UJ;RwgJ4HWtHL1hz%_ytnYk=SE z$!NpU;?AiLV5T%aHW08iHZ`uh0h@K6WUB!K^y9N+t2!>mqJd+J+sS8i&T5%DD!d5B z$xz{XBBg~DZ7B6N#b^}-Qo|htNMywtCQ9U)2NwOFkfXOCBeo4rAOAU~gdp+57tZh3 zhKn%yW53f?siT1Cv^234)S-vC6^66rP!F5@lp&!ZsR?#;Mq(LI$)B0Hjm2y4+9VP( z|Ked8VBP+P3BXj!E1P&tCL+ZBZ9*Elrf|gL%+97nR}qdbp{v@CO-9C>==5X#i^G6A zt!!(}`%Xc?yR~@9ien4zo@hkRyJ#ACN`Shg)x}07+DJ=P!HdGUEFAA+>W9K<4+E_Y zX(3Lmfg9>&1CP`S(4x~_SHxMDH#NSDji2_&*~cUm>F5^G-e1v6SDQIYtp$p?0)a$} z7mlAI&R^}w+E+t(Fy4pC8OxN5aF5~t0vw` zl(k~hcH32ZTao%BnBR)dlcRK@pQ!y~LPX)C?W|1yg!*8|2AbI@yAU;?P&ojWf9K)w zZh^%4I5?ym;bb2qL@+7@Rw5z~`wqP(j%N9Cp2WL-Ocx$eos5s@2N{2NQx;%MAj#uv zHw1JQ@Xy`}p&M#{GZ|Bw`NNW*M>$}~BDj3`Jbo7mJc;OVNi>h&)(gKtj`(ZJ^BKVS zTO~#&tki|{U7zTR#Ld~1(7lZPus|oEQB}3P`T2|=feX&jU6A<}!}wg#*Vx zq|k4?;;pYviKl0!I1L#Ys*T1AOH`{58vu*vtL?0(P zGMAb>Bhyu&C+)LqXZ{qKmFU)K`7=y%=^8cgF&mtKu&?vAh5fA*8F&fA99w)MHAd#;4gz?<`!bT z+fm-T;ueD4i*4AhA-%nELQ+`uD_j3A;yc%!78@tRR8J)m(aC)#tUTPwP4aw$s%P2D z$~N^nT6sf7yfAjll-5KLdv9E*o%h-76Vo^h`+A>WhLcyZrgDMH=eh?KVti(MW$Z2% zHQP=1F&gz>CAsQFUkxwD6~BXOjv-@rW_H9&WR0@qul)SW-EO@kiAAIW1=+5BBfJ9( z4tJV7qD(xb)Winq*mP3Ed)Rf^$_S^L*XqK&+0US=W=WZjN#D3s#VDm9iE(kctV@Wi zj1L=bi2|(Q*!J0w*wn@ee!D+CHnuwYub%bM^b# z4k4sF1>$O-9i?avu+w8*vzBf(&gi&LO1#t;Fm)=7D{}7_UMAiM9Z#qPyB7=~aE;T7 zOg_fnShVyHLmngHoE7<`jmhE<&mcIYC2cVAZG&qt9#r~rnXnF!Jin>DHioF zeZ^h@>a5{(OMC*YC3yvy%xb;D2n{P5x1)*Mv_6)MtlDj-1}m`~V6nmUnu&&uK2uYO1Bu%Uk24-Rdn!k^Lr~;aI~&nLyyoK8p8Nn+~m9NN0;`O5X81n zU02C!J2BUP^dX)6hF{_icHt(58vt=Ndm#1yE&7a9XlX_lAw{QTk+?T2y3N#>u%8O>cp z;z)S^0>2bZ`msWMa| zH+G6b3$_Mz6|L9Xd-gZm3jboiW8xEmB=0ZR=KWE?w8Hs8!$8i)^lsaLrmXe6wgOt8 zFw!&M${sTD^q+?RDtY+&m$yz0(=Ge2O6%tAp}ZD|>F2W}Qa8J7P~e2+YN5iL*!0h* zH92>VBLBXI8i~iDlHr}%Uvvn3qCy#;Kd!xmQ7`HY$4$yn*`!^K^3coP$zG}Rs+H;i z9-2TDEHmkS$zLLS{bMA+$@|Y5z3`u-STEY_`_ls9%54ARzUhSgMA?pxDHMh>uNe}e zaq^p=7lK$XgDs1Hp@2_7OAO}XiMTcI*0r}!`Hzi^l_=9&i-F#$29$8ZFHrB0UD#5G z!TLNFxB$r5-NjNHWK6hug;1^0lHMlEW6EX!$EFb0_hGRt<&T{f+d&W0ke{ zMp9c(^Vp~)uFGlf9tLfs1a9}e+TaE`jb|fCS4HYIk3jnJIJNDMAPt1(@7?dXz=`W? z)F<+a{t??+k64yV-)(8YHRJ+%umIY1KX`kAoNqO$j;Z64`8-8((V-70s4})eQkkZg zo{BVp{}m9PGuwS6-gOrx=_#*V_pbl4OV2Lq=x4%v)RrdCQ1knVY-YM|;AW)vG@#V_ z6|nV~d^c=xm6Y;JpeA<;GniKPppJlyp%aOG4NhiJM7;Fw!2eWMzt~7>z-rM3WEWq( zLZz7=Q^?{|fh&!GZJqEv6pZ#Bhc5~`R{eZ7M0Bj#rXjkXyuvPh1sRB2hJBdSGoZ`9 z+qEHn>H9nC6CsKnH4GT<^Ov^WUw82|?T*QOqg!c3QOjxSP;=#h?-evqAlOL~|Xx5oLxYwdj>Q)l{9W zsH(vD2F_vh4yiINpTwS>hC@d0Bh_4yCH5VyL;+e75eFX}uEpF?~i3 zWv?X8tKy)lkbpFHhMz-Do;CXWPEC7L*Vf$hm+jbfo&dFW7-vGA|I>|_9C^6EJvPaw zmHvy*2F(^T)^O-2)MoO44i0S%iy!<+v-2m=)3(Ssrg=^ro(&oEWCDA1vuuHQZcs)V*np}7jEcBTF-x*MJ(SvcGh!c#hp6xwa+vE* zdy+V_oIt=vNV?{U0+(;8596e+|Ls71!Ltq3kjC{F&a)AUbj~PF4Hqjh)_pr>?9&|0 zw*CyoDF0JZm~k6;K?8>2YoRjbdgtX=J~(^urQ40lUV1-*M^63LgaGT`WbQjvc7@3K zgT|fmE^U<;lXK?Wr+uL!2Jc*dD*d_UkS;ojwvqT=8_K-XO|IdXxQW8_ZkE)8II5Me zL#x<;7#FPHixEN!#l|d}9=k{WD4HP*1z=cPmJv|*W*zu% zPF-{nOZ$7H3Jwb}_eZdl@_jhqVxb;LEufdvYGw#?c1%UF6mt~PgB~2oQ?{{7Zb4$y z>*k%BrlbOZ+rn*ELwkP1ik2})Gy-R7IpgZ$WqdsDk&xm8b+1Mp2Vv?No16e@WRw?PxoU7*Rl0V;#XdTsJ=Ttm7gCbw#wA!Dl4m3HAI`@+#4O%Dq|z~;nG z#f$C~PZ$;~%qz1kECb4=Kd%gbRok_EO|M1OTsv*j6zmX~ZN`-hLE}x*)wANFWSUY6 zgLEc|L|8vE)b1yCiv6pFb8=|FL=D=x3l?i#jy7OU9k4173&i?@gbj=mVprP`>6RRCU?vrQfVX(VLbHEgxdZ(@`bw>o^En>Sl3CxxEqEayovt z-Wk*T3}Q~oE$+5fbDlXIMB;^kq4`HFu>SRggzu zJ~FO(8vMC&De&gi7Cy(d#cjF@f~s+>ytLw^`^MzPP8gu(ZiK3~P}h9zj2#Mg%>U)t zjj9_UhbWXj;|lPB9gQ-72n_ds@w!;>a*k#ibW{p`60b_o*FoQS-2a@0;#)|-rudQ_ zdbCa=9(|~Mrz>qjp_U2(-ZE1mNn9jWU??>wnO=iB&Ns&21omwBGD)e30?%W?3<3pN zHy!hJh{=Inz1!mTRK3t!OvA$1h!;svrj!y>7V^oM7xI3i%GxnlBzj~(oeC215P;#q z@rNYsIR;oy!Y5c^If6?`TW_IJXX6)hoMTulr>!&D(+gC&%m96nF`Ra6E ziAAtSzTav-Ta3@GSZqA(K*m_t_Rexucckigtzl4Pi#XoG_S(686pc~lV}Y2?$&Rn@ zIn{_aJlP(Iv5N)hUIBA_ji?F73l%Apiul--@9RZ~_U)WV9J4_Ujbtsm!Pev31T>=E zVhlDBDzZ+fHBi5q?I%R^I;Fd#D3nb}oW!IqCjH$Y+CN_WbF#Oo#a1C}Bz<3$;+9f| z_$*c-+S=$^ljM}w&`e|kG|#acqR5Wxi3-%_1zDcqW0fXqLcWVcZ?6_ z5Q$=PnUuK7ILmf$pWjmLaV$fZ05mjAk~8WctI8;dJ7BF>-ujl;@|OY0m$5muqVo==T-5;1WPe2xU;65DY<(6rsUZNW(?f z8z1>zd8J#b#)D*9FBVt+UBo4d^6Tnkxy(ZFM<&_wVzmoOk|W_+e-Zktqgr0Bx?_Bv z?(NEn8641QVT>|Dmbg}kl#Wn56RnT{`>0O#aWWS5n1)Y+)UE(+CbYLm;guj;L|1`t zF!_pmBA~RbHl53O?|1RB#rp^XbE=(FRxTml1na;q1<}cg@AteUYjK=toST{w18Hzq z3uXO2mWo=9b$}gZrj*6A_U?TQf6uz+K~OZarN!cqD<#DiG#jQ%laASgFEb>YM&Xyx z^U2O?OQWr`a_5~sd_I)a4DeV;gfoweBg9|wQ|G-NuN>v!g=3p#Z{PLc4>hTr<+kmR zaE54#Di#V-z2h>K@qOg~kl>j&Qfj=LAxjX`cHysV!k#^dnj9ItKUh>%VLLac(t7a< z`PMalU@UqAdu><&w>D+!B*iv$0g#=H=BYUcd9V5p90pRYGt#`sf{Yz;xpMJzktmo` zNm-qJqBMxL$QI9$w|nD==0zBC{1aElc*W@gC2A)mhYcvX1u#|xpCXdf?X3W4G^*m> zGUHN~pM8K^|EXY^O~_W;G34t{8lU~bXF2Y<1&pOO|7BH6)UJ&ILO;gIF|4J>EDQ=b z4ND+|3zE;ZC4K7n0tWfkfV_hk%hL>&#E2frL9EM?7t|xAZ9}@PyDKj-S0@`=e12y}yCNq1^l`BjBcQ=P6)iYW6176s^gWZck;j)C--cDQN6qnpNi z6HZcO)URvL#KGcVn{N5)OO^zZ4+wvm{(703!yqAnQK8bU#OB3<rUK-P)jKJD*h4iU*-X*{~nSlK#u>@NqfI0PD-H6b?B z=F>y|S73Fs_s<3F3ipemFs#RQB6~qG*rH^U1s>Xo$qd0m6B=c@YH;7G}Zw82Hc zgM~Ca8+Nl11P&1$;?9oP2A`e5&VWu?R|#Y}fQAuYeFT24{p4RYfg+P{P7c#>S7E(G zSEhcZqoaJCcpI9XG9i{i;*<7jCF;I6S;}kUYP#>Fh&&}W)0X^?gKfVHxk_1p(r}f@ z=L41eyb9oks!G^Hr@YGDgd9LwgrO(;x-$S#T`hX2ag6p9fX?!g-{xH{&JuQ)X7;-+i~d|>aI*8BLWH%V)=e209xBU9Eh(>*hx3Ft+^yI<3< z3wWFrTB4r}Qs+JFT)TTLV}LzoS++SrHY%ozC0cYXa5{^Gs% z#&a|kY#-tmWh}N(Bd}u-(*jp10Z(<0wzb1|j=>97da0sr7VMy+v0A6h9ON@<+>$}w zn9OYm9yyhAB|$^WfRuV`I>rRi*gY!iRzd@Kz6EHpn%8dL((z)`YDGG@G(s5l*&u7& zYws^8G<{b4)M%c6WykErzg*vprao|lABo2{uG#JvH+jrk5uWOWHer^w3lJ5M`&AYr zX8jpRreMIcJG{m#$XNFzoiYc3099`YeU7Bs8;>YEY&-0&Xf!((lbAQ}h^sd-G;0rv zpWt6^tLORxB<)WdV)2%fbyDB8S3okW-Tf;8t}aGjW3pduV=gh*pBtd{%5 zti8hnF#Rr;G?bO%46#i4W;v?&N^Gyt-a(?B^Z7h>iGe7zFE?)6zqt3hCwTdJL`jW< zB*EYhARMuu?x8^a#9f0I>m< zK54AUOR>a$9Ip0~?qliRI1lw3f4+#)aeY8uS(1>HuAVT6{vrFicN1QsTftt%Rg;kf zsvTXB6*$w-?Zuk_a08;dtP;bb3AsiLL-cN{UypBReik#%EA0(@m+PAOwr-MlfOk!$ z0qw9c#ijWRb+3-z8hbF32XoSmi#(%WrTCd>{M^MXYP`6+Qaw4n!}TLS*3-f;j~D!8 zY-6>r3#$6XvFx^sqT7#zZlH8o7&G>5Q!(Uv>%&SBfVvP7voa4o2$f#lx#W6m-kmO{ zz}vqv_ROBxLle@&SkqQ1km0PVO~Gz;%rf;{dm60n$!D~wbiEUob%N`FBDLGjE0J-D z#fW&XvFz=J;`1Yf8UgTwacj|VelvN|8|b;?c3KkXsa8d(^)|yM_iWH48$sH)Z6+f> zZwRuR-0kbay&?R#=DV^94ILRtyU9C%JIB2Eef@M@aY3ZY&+`weYElRIaTed3nLm7c zGhwW-=AnCFMO@X=i1v)5cPX;Zp{-Kln5$x+!Sk1lsei1OG3%EKuRRrdKltVoaMsJm zjn^xxqV3SEQ72~MSm>4UGm@+-14Hm6f?0tqih#6e+p*}oITL>b*A}SZut086gpWCy zkXJ!$w(`Jyn< z{iT@0ETjcdMgcse4b(``Q%zK?RR4q!zqhf!0V;cpFsNUWYL5j_#1o1z&kh3XnqO`6|8LW9;9M1+@4gy)JOubSAJ;?5J9v%5*9Joy*q^wj>RxJ-X5`4?pQ^J@gZ8wnccD0;Id zeCii^d8vEi*fwJS+OL_DDUuK}2236OZq5(noSeT?NRxxb0L}P!rxhM-7?K($tOH1H zAQA*%4pV0pwzvjk(=c?6WEA* znyUC)V*Iwyy+zqQs!2k{#Ksdc<7 z5#mhI&~&N?#Op>jkkqCrtO{C)Gr1fzx!@lzoM~rqA4d&26m`)FI4b|%C`xnecXedZ zI{uAeirK`6-Qb`Q&Ig8!6(WE9jAQY@M~~m2!l5h<%APoqtir@(%?R}kP9nF{xBCY+7? zEbo*_6~CxtB6Jr;OpQ8L#zj*jr891gt-AIj>s6Hgo$epy|87T1qcd9L3xL3`#}jGY z*GkR;5C2l^&;?UNV~O?R&a2v)$+B$2hV*~J9P^G{-#m4N)6P%mk)W9*+4cjq2Ut9U zT=jA-ll0@SW8UZQqZRQ(#!gAWYa;y%wO4AF+R@=%k-j-9r9FOe1sDiDE5j$lZrb9g z9k)5^RZHif`BE_POjDiK3~_z^M0GyT_zO3f|2)52z4n%geqY3la*}EpQkUs%j$dsl z>gIh}RMu`e9=xCmfQqlM;C@-qL@&P(9O~_zEati(4&@4ZvqCrodqkSW^j3qq^YpCLK{wDCZXj5

bp_<1H+KmuFpIioJI%Vid=)y2Nje`Q=d(PL)_0t zy;2cT7tEr)%75QCdaqFwqd&Kh0WS6*&Y$J!GTIK+SkRGSj#nGI< z^GI0|uP7lIV9FAzMTcGvE^&qJW?S=uZ(fm)`wb~bt|$Bo3as%u8n$Y5<8GdoR38)`u8V(WTvuGCemXoo-F~bLF!O|meQb1zH+&D{)6zW~ zs$zB3tDStKd_^btEUFi13MJ74{T_~!IR54NNjs+&(o74iXnuM6UFL14G*=E+;a$Hg z*6-gOjIu~Q0>(}95!4CNFBc*BJhmQ@=3-Cuz5bO zM~nC(BnEqtn0=7h_2fgw-al~qvdPBq5qF%o0-BOIaq$S#(}*SJy0Yuq=cLCy^lw2P zrTA(h;dVTkz}f(H5e=xI6zZg-sY6sv?r^4h@9IM)ps12;7)z`eJyVEY92aIA3~A@Y zcKi7eND>ChCM5c7wq9@!qE`5Lv(ephpjK|C58JNfiQQ4n5%?UU9T`+C)fQR2e=5NP z9e8%g_}+&UzV{XdT@|A~6tu_)E}H0%0PL|h82KA7z(iqNuhFq#KkG={!?!m<$k6U@ zq_#EV4|5|;kv@PlH-4fykP9+muQ|8>3j;s-_ zqCZQ1U%82-HlB-%M>k1DF8O)HJ4y%{(An&>-bv+7lecl2`@F@5vun5lAv;5Q{D!O z;1;N75RY0g_4k61(nlYw)eV^iD24eVVY)v8DOSgtN24)z`Mu^S4^Pl++-0!0(6EZm zcu9xUI8w+`v1H=fwE?P%Zc{`+v?6cb-6UUSWgJp$CC!z77{rZ9C@9c+p|ql;%5E!Xwf zTKH$V{$^jv@D!esRMC<05Xz=svxX`2^s=UW>-ii{;gL!O;OHkvDqLZY0czGz)#839 zZBt$<_Ju&-!!67v9M@LN;r1mypsLzw5iWd6KC(?vIxXzIe_`gX`rf3cj3v8;)0{;! z%=ic5d1N;HH(3=3{;V1Q7T~j$mpV*?5A2^gnG1tlWo6kr-Rd9k?tzTk6DRIyaAX^l z&I$K@j($?q&%Cmj1Mw@Y=Gvou05GM?oHKgx9NF<}!61pBUDT;HU;M0Md=OzWtun4$jF@!FAj_aW z)>LFTHRLT|P$aA{)zQqdh%U{VN8IAmKX|L9^)ZcrRl|O;KqQ7WjbCa%ylz5mT@X89 zl*ip7`fOZ;Drl)nv8bE!Y2{eRX+h`*M%aWIv-)<_S%oidDdg`h$^1n3N@WBk1!li$ zX(Zbf7#*aBsGkvL%Y3a|?V1+fAj11plwy^GeT5#F;j1XdcqpPz@**Xj3k$Iwy(7Kj zJG-c~T9xy6jkefCdHK6g;!L}WBnHP|uXu#G58u;0g+H{mbIxxO zovs-U`>n8=hJ%5v6yZo3rX(rQcdRX6GxcS_lcc(@_b2l#ASvLdD?)Xc0dNKoTkWDH z!Gf^6xXIvg&&ecKGXr17GGip~oK{0$c%|8DbZYxjfFTrS6WC$v0kX-nUE9i+6Ho$$ zrozx4v=pJs@0lRMx>PLWXO!r24eg~%y5>ZzZLa-P*clP5l=j@KQ?cONKS_{}f;}?( zUWtiZJcBIry5eBJyWG#z72Z&YDFFs^cY`fHC7T*%C|^PIVSnWy3qy+VUc?DUVsdZR zHL-Wk?*@LVncX?~fS2ST?Ho*Mb9ISFk%AW-%$@t7h89lxG)WA+VBWj@6*oS*kQ5Kb z63gWjeF%(W$#YY5(~5Jk3hQ_qc{r<~>!?7S)*#i36&@z$Lt4}h`{+{i`8bh;5d{5O zttR&Sz`5O=22Ijq`WRE(=w>z;kuT&L#`wX$Y$+g4q$q)fQ2-Eugiu1Zop$ZpSN}5U z#pDK3qkGub9J7E}-#})ngy+)G8qeNv@q?(z+Xc_JJ)2U+>^LZKvA(Nk+j%v1^A(ty zE4?AtW7nf@;8ih7Vy$||M63K(ckxf=Cn6*jdi>ws(ppHs)D&NFVbey7((2BZQpkt= z-RKTVurJ>mXeBNb-o~fmkzb@v!MmoEJDhdgLm|f0Q2lWr0Edjw}1cXN-~<# z(FG*ljLSCSfX5!5w0_rc3lalU{5F(uRX>I7YhZ6N(NC8D_m8sio*wn6{}jMLNRF93 zb~BIf8UOESQ6Mlm#z0+*RNvhe$QVP`{w8}bSiRzZhnNETsX+tld#68LpgN>`fQIrf z(#L``ZTpURj)EO|>Wk(XVUKsVG(s>3-|A=Dfyz?*#XMHxYyaWIYH0c0uw2*({b7rr zSk)Xso+&2a!G4QshiA~?cmb#{Ts>=03$*TPk$7ct6kP+MRdo6o5Lni2O!n_J?>`0NC*fd@xW|>NfuQOO?*;{u zBuvq4BDLNJB#VL6E4}3>mLBgCg37A$pTTW1hrhB?f}W;{b);f0ks(!htN(KsPGzro zM71e0X1nvVz; zvOnSov}4(h{?qilvL-$o%{a=#A?u0R0>#|pL)qb4D>+qaa5KZHFtX|hEEja2_;r+{ zWQp>1q?%r=7kv$MgIHKz`G8`My~l@yPb}1ho971$VxzfGr{Xpl1qW;sX&4Bj)HvcO zRxIS4^&uZJ_9yv^pZOn~Rl#VLkn2aq6gHjhXHubF-kowiC&S43zF5BNnfT8F1iGR< zXcY}AE{xHf;#?ruIA#M-Ts-b)*woL4BnY08vy|D3>z}`5!Y8#m4{u diff --git a/dlp/system-test/risk.test.js b/dlp/system-test/risk.test.js index af4fa6a796..3a8f5ebf26 100644 --- a/dlp/system-test/risk.test.js +++ b/dlp/system-test/risk.test.js @@ -29,7 +29,7 @@ const uniqueField = 'Name'; const repeatedField = 'Mystery'; const numericField = 'Age'; const stringBooleanField = 'Gender'; -const testProjectId = 'nodejs-docs-samples'; +const testProjectId = process.env.GCLOUD_PROJECT; test.before(tools.checkCredentials); diff --git a/dlp/system-test/templates.test.js b/dlp/system-test/templates.test.js index 8144c2c83c..19902e2e37 100644 --- a/dlp/system-test/templates.test.js +++ b/dlp/system-test/templates.test.js @@ -42,7 +42,7 @@ test.serial(`should create template`, async t => { }); test(`should handle template creation errors`, async t => { - const output = await tools.runAsync(`${cmd} create -t BAD_INFOTYPE`); + const output = await tools.runAsync(`${cmd} create -i invalid_template#id`); t.regex(output, /Error in createInspectTemplate/); }); diff --git a/dlp/system-test/triggers.test.js b/dlp/system-test/triggers.test.js index 0cc12a6852..dd2aa1549b 100644 --- a/dlp/system-test/triggers.test.js +++ b/dlp/system-test/triggers.test.js @@ -33,7 +33,8 @@ const bucketName = process.env.BUCKET_NAME; test.serial(`should create a trigger`, async t => { const output = await tools.runAsync( - `${cmd} create ${bucketName} 1 -n ${triggerName} -m ${minLikelihood} -t ${infoType} -f ${maxFindings} -d "${triggerDisplayName}" -s "${triggerDescription}"` + `${cmd} create ${bucketName} 1 -n ${triggerName} --autoPopulateTimespan \ + -m ${minLikelihood} -t ${infoType} -f ${maxFindings} -d "${triggerDisplayName}" -s "${triggerDescription}"` ); t.true(output.includes(`Successfully created trigger ${fullTriggerName}`)); }); diff --git a/dlp/triggers.js b/dlp/triggers.js index bdb890c8ba..a820f7bfa0 100644 --- a/dlp/triggers.js +++ b/dlp/triggers.js @@ -21,6 +21,7 @@ function createTrigger( displayName, description, bucketName, + autoPopulateTimespan, scanPeriod, infoTypes, minLikelihood, @@ -48,6 +49,9 @@ function createTrigger( // The name of the bucket to scan. // const bucketName = 'YOUR-BUCKET'; + // Limit scan to new content only. + // const autoPopulateTimespan = true; + // How often to wait between scans, in days (minimum = 1 day) // const scanPeriod = 1; @@ -65,6 +69,9 @@ function createTrigger( cloudStorageOptions: { fileSet: {url: `gs://${bucketName}/*`}, }, + timeSpanConfig: { + enableAutoPopulationOfTimespanConfig: autoPopulateTimespan, + }, }; // Construct job to be triggered @@ -221,6 +228,10 @@ const cli = require(`yargs`) // eslint-disable-line default: '', type: 'string', }, + autoPopulateTimespan: { + default: false, + type: 'boolean', + }, minLikelihood: { alias: 'm', default: 'LIKELIHOOD_UNSPECIFIED', @@ -249,6 +260,7 @@ const cli = require(`yargs`) // eslint-disable-line opts.displayName, opts.description, opts.bucketName, + opts.autoPopulateTimespan, opts.scanPeriod, opts.infoTypes, opts.minLikelihood,