New CLAAT export functionality

.gitignore

@@ -32,5 +32,6 @@ src/content/*/showcase/*/index.md
 src/content/*/showcase/tags/*
 src/content/*/rss.xml
 src/content/*/atom.xml
-
+src/data/codelabs/*/img/*
+src/data/codelabs/*/*.md
 

gulp-tasks/build.js

@@ -127,13 +127,6 @@ gulp.task('build:updates', function() {
   wfTemplateHelper.generateLatestWidget(files, options);
 });
 
-gulp.task('build:codelabs', function() {
-  var startPath = path.join(GLOBAL.WF.src.content, 'fundamentals/getting-started/codelabs');
-  wfCodeLabHelper.migrate(startPath);
-});
-
-gulp.task('build:sitelevel', function() {});
-
 gulp.task('build', function(cb) {
   runSequence(
     [

gulp-tasks/claat-export.js

@@ -0,0 +1,65 @@
+'use strict';
+
+const gulp = require('gulp');
+const path = require('path');
+const glob = require('globule');
+const wfHelper = require('./wfHelper');
+const runSequence = require('run-sequence');
+const wfCodeLabHelper = require('./wfCodeLabHelper');
+
+/**
+ * Update the files using the claat tool then return a list of files
+ *
+ * @param {string} srcPath Where to run the claat tool.
+ * @return {Promise} The promise with the list of all files updated.
+ */
+function getCLAATFiles(srcPath) {
+  let cmd = '../../../tools/claat update';
+  return new Promise(function(resolve, reject) {
+    wfHelper.promisedExec(cmd, srcPath)
+    .then(function() {
+      let opts = {
+        srcBase: srcPath,
+        prefixBase: true
+      };
+      let files = glob.find('**/index.md', opts);
+      resolve(files);
+    });
+  });
+}
+
+/**
+ * Updates files, and copies them to the appropriate directory
+ *
+ * @param {string} srcPath The source directory to find the files.
+ * @param {string} destBase The destination to copy to.
+ * @param {Boolean} flatten Whether to flatten the files to one directory.
+ * @param {string} bookPath The location of the book.yaml file
+ * @return {Promise} The promise that will be resolved on completion.
+ */
+function exportAndUpdate(srcPath, destBase, flatten, bookPath) {
+  return getCLAATFiles(srcPath)
+  .then(function(files) {
+    return Promise.all(files.map(function(file) {
+      let srcFile = file;
+      let srcImgPath = file.replace('index.md', 'img/');
+      let destDir = file.replace(srcPath, '').replace('/index.md', '');
+      destDir = path.join(destBase, destDir);
+      let destFile = path.join(destDir, 'index.md');
+      if (flatten === true) {
+        destDir = path.resolve(destDir, '..');
+        destFile = destFile.replace('/index.md', '.md');
+      }
+      let destImgPath = path.join(destDir, 'img');
+      wfCodeLabHelper.updateCodeLab(srcFile, destFile, bookPath)
+      return wfHelper.promisedRSync(srcImgPath, destImgPath);
+    }));
+  });
+}
+
+gulp.task('claat:codelabs', function() {
+  let srcPath = 'src/data/codelabs';
+  let destPath = path.join(GLOBAL.WF.src.content, 'fundamentals/getting-started/codelabs');
+  let bookPath = '/web/fundamentals/_book.yaml';
+  return exportAndUpdate(srcPath, destPath, false, bookPath);
+});

gulp-tasks/test.js

@@ -28,9 +28,6 @@ const runSequence = require('run-sequence');
 
 const MAX_DESCRIPTION_LENGTH = 485;
 const MD_FILES = ['.md', '.mdown', '.markdown'];
-const STD_EXCLUDES = [
-  '!**/fundamentals/getting-started/codelabs/*/*.md'
-];
 const EXTENSIONS_TO_SKIP = [
   '.css',
   '.gif', '.ico', '.jpg', '.png', '.psd', '.svg',
@@ -302,7 +299,7 @@ function getFiles() {
           globs.push(`${lang}/**/*`);
         }); 
       }
-      resolve(glob.find(globs, STD_EXCLUDES, opts));
+      resolve(glob.find(globs, opts));
     });
   } else {
     gutil.log(' ', 'Searching for changed files');

gulp-tasks/wfCodeLabHelper.js

@@ -6,32 +6,34 @@
  */
 
 var fs = require('fs');
+var chalk = require('chalk');
 var glob = require('globule');
 var moment = require('moment');
 var gutil = require('gulp-util');
+const path = require('path');
+const mkdirp = require('mkdirp');
 
-function updateCodeLab(fileName) {
-  gutil.log(' ', 'Processing', fileName);
+function updateCodeLab(sourceFile, destFile, bookPath) {
+  gutil.log(' ', 'Processing', sourceFile);
   var authorId;
-  var metadataFile = fileName.replace('index.md', 'codelab.json');
+  var metadataFile = sourceFile.replace('index.md', 'codelab.json');
   var metadata = fs.readFileSync(metadataFile);
   metadata = JSON.parse(metadata);
   if (metadata.wfProcessed === true) {
-    gutil.log(' ', 'Skipping', fileName);
+    gutil.log(' ', 'Skipping', sourceFile);
     return;
   }
   try {
-    var authorJSON = fs.readFileSync(fileName.replace('index.md', 'author.json'));
+    var authorJSON = fs.readFileSync(sourceFile.replace('index.md', 'author.json'));
     authorJSON = JSON.parse(authorJSON);
     authorId = authorJSON.author;
   } catch (ex) {
-    gutil.log('  ', 'No author.json file found.');
   }
   metadata.wfProcessed = true;
   var result = [];
-  var markdown = fs.readFileSync(fileName, 'utf8');
+  var markdown = fs.readFileSync(sourceFile, 'utf8');
   result.push('project_path: /web/_project.yaml');
-  result.push('book_path: /web/fundamentals/_book.yaml');
+  result.push('book_path: ' + bookPath);
   if (metadata.summary) {
     result.push('description: ' + metadata.summary);
   }
@@ -40,6 +42,7 @@ function updateCodeLab(fileName) {
   if (!dateUpdated) {
     dateUpdated = moment().format('YYYY-MM-DD');
   }
+  result.push('{# wf_auto_generated #}');
   result.push('{# wf_updated_on: ' + dateUpdated + ' #}');
   result.push('{# wf_published_on: 2016-01-01 #}');
   result.push('');
@@ -58,7 +61,14 @@ function updateCodeLab(fileName) {
   markdown = markdown.replace(/^\*Duration is \d+ min\*\n/gm, '');
   markdown = markdown.replace(/\(https:\/\/developers.google.com\//g, '(\/');
   markdown = markdown.replace(/^\[\]\(/gm, '[Link](');
-  markdown = markdown.replace(/^(#+)? __(.*?)__/gm, '$1 $2');
+  markdown = markdown.replace(/__\s?Note:\s?__\s?/g, 'Note: ');
+  markdown = markdown.replace(/<div class="note">((.|\n)*?)<\/div>/g, '$1');
+  markdown = markdown.replace(/<aside markdown="1" class="special">/g, '<aside class="key-point">');
+  markdown = markdown.replace(/<aside markdown="1" class="warning">/g, '<aside class="warning">');
+  markdown = markdown.replace(/^<a id="(.*?)"\s*\/*?>/gm, '<div id="$1"></div>');
+  markdown = markdown.replace(/!\[.+?\]\((.+?)\)\[IMAGEINFO\]:.+,\s*(.+?)\n/gm, '![$2]($1)\n');
+  markdown = markdown.replace(/^## Contents?(\n|\s)*(__.*__(\s|\n)*)*/gm, '');
+  markdown = markdown.replace(/^(#+) __(.*)__/gm, '$1 $2')
   result.push(markdown);
   if (metadata.feedback) {
     result.push('');
@@ -68,17 +78,10 @@ function updateCodeLab(fileName) {
     result.push('[issue](' + metadata.feedback + ') today. And thanks!');
   }
   result = result.join('\n');
-  fs.writeFileSync(fileName, result);
-  fs.writeFileSync(metadataFile, JSON.stringify(metadata, null, 2));
+  gutil.log('  ', chalk.cyan('->'), destFile);
+  let destDir = path.parse(destFile).dir;
+  mkdirp.sync(destDir);
+  fs.writeFileSync(destFile, result);
 }
 
-function migrate(startPath) {
-  var opts = {
-    srcBase: startPath,
-    prefixBase: true
-  };
-  var files = glob.find('**/index.md', opts);
-  files.forEach(updateCodeLab);
-}
-
-exports.migrate = migrate;
+exports.updateCodeLab = updateCodeLab;

gulp-tasks/wfHelper.js

@@ -2,15 +2,16 @@
 
 /*
     wfHelper.js
-    TODO
  */
 
-var fs = require('fs');
-var glob = require('globule');
-var moment = require('moment');
-var chalk = require('chalk');
+const fs = require('fs');
+const chalk = require('chalk');
+const RSync = require('rsync');
+const glob = require('globule');
+const moment = require('moment');
+const mkdirp = require('mkdirp');
 const gutil = require('gulp-util');
-var wfRegEx = require('./wfRegEx');
+const wfRegEx = require('./wfRegEx');
 const exec = require('child_process').exec;
 
 var STD_EXCLUDES = ['!**/_generated.md', '!**/_template.md'];
@@ -59,6 +60,39 @@ function promisedExec(cmd, cwd) {
   });
 }
 
+/**
+ * Uses RSync to copy files from one directory to another.
+ *
+ * @param {string} src The source to copy.
+ * @param {string} dest The destination to copy to.
+ * @return {Promise} The promise that will be resolved on completion.
+ */
+function promisedRSync(src, dest) {
+  gutil.log(' ', chalk.blue('rsync'), src, '->', dest);
+  return new Promise(function(resolve, reject) {
+    if (fs.existsSync(src) === false) {
+      console.log(src, 'doesnt exist');
+      resolve();
+    }
+    const rsync = new RSync()
+      .source(src)
+      .destination(dest)
+      .archive()
+      .exclude('.git*')
+      .exclude('.DS_Store');
+    mkdirp.sync(dest);
+    rsync.execute(function(error, code, cmd) {
+      if (code !== 0) {
+        gutil.log(' ', 'Copying', chalk.blue(src), chalk.red('Failed!'));
+        console.log(error);
+        reject(error);
+        return;
+      }
+      resolve();
+    });
+  });
+}
+
 
 function genericComparator(a, b) {
   if (a < b) {
@@ -90,7 +124,7 @@ function updatedComparator(a, b) {
 }
 
 function getRegEx(regEx, content, defaultResponse) {
-  console.log('WARN: wfHelper.getRegEx is deprecated');
+  console.log(chalk.red('WARN:'), chalk.cyan('wfHelper.getRegEx'), 'is deprecated');
   var result = content.match(regEx);
   if (result && result[1]) {
     return result[1];
@@ -185,10 +219,10 @@ function splitByYear(files) {
   return result;
 }
 
+exports.promisedRSync = promisedRSync;
 exports.promisedExec = promisedExec;
 exports.getRegEx = getRegEx;
 exports.getFileList = getFileList;
 exports.publishedComparator = publishedComparator;
 exports.updatedComparator = updatedComparator;
 exports.splitByYear = splitByYear;
-

gulpfile.js

@@ -95,6 +95,8 @@ gulp.task('clean', function() {
     'src/content/*/updates/_index.yaml',
     'src/content/*/updates/*/index.md',
     'src/content/*/updates/tags/*',
+    'src/data/codelabs/*/*.md',
+    'src/data/codelabs/*/img/**',
     '!src/content/*/**/_generated.md'
   ];
   var opts = {dryRun: false, dot: true};

src/content/en/fundamentals/getting-started/codelabs/debugging-service-workers/index.md

@@ -16,6 +16,8 @@ description: In this codelab, you'll learn how to debug a service worker using t
 ## Introduction
 
 
+
+
 Service Workers give developers the amazing ability to handle spotty networks and create truly offline-first web apps. But being a new technology means they can sometimes be difficult to debug, especially as we wait for our tools to catch up.
 
 This codelab will walk you through creating a basic Service Worker and demonstrate how to use the new Application panel in Chrome DevTools to debug and inspect your worker.
@@ -72,7 +74,7 @@ After installing the Web Server for Chrome app, click on the Apps shortcut on th
 
 ![9efdf0d1258b78e4.png](img/9efdf0d1258b78e4.png)
 
-<aside markdown="1" class="special">
+<aside class="key-point">
 
 More help:  [Add and open Chrome apps](https://support.google.com/chrome_webstore/answer/3060053?hl=en)
 </aside>
@@ -101,7 +103,7 @@ Now visit your work site in your web browser (by clicking on the highlighted Web
 
 Obviously, this app is not yet doing anything interesting. We'll add functionality so we can verify it works offline in subsequent steps. 
 
-<aside markdown="1" class="special">
+<aside class="key-point">
 
 From this point forward, all testing/verification should be performed using this web server setup. You'll usually be able to get away with simply refreshing your test browser tab.
 </aside>
@@ -187,7 +189,7 @@ Note the ID of the first Service Worker. It should match the original Service Wo
 
 The second status indicator shows the new Service Worker we just edited. Right now it's in a waiting state.
 
-<aside markdown="1" class="special">
+<aside class="key-point">
 
 __Try it!__
 
@@ -204,7 +206,7 @@ Note that the console now logs the message from the `activate` event handler
 
 `Finally active. Ready to start serving content!`
 
-<aside markdown="1" class="special">
+<aside class="key-point">
 
 __Skip waiting__
 
@@ -286,7 +288,7 @@ If you go back and click on `my-site-cache-v1` you'll now see that all the store
 
 ![317d24238f05e69c.png](img/317d24238f05e69c.png)
 
-<aside markdown="1" class="special">
+<aside class="key-point">
 
 __TIP:__ You can also use a new Incognito window for testing and debugging Service Workers. When the Incognito window is closed, Chrome will remove any cached data or installed Service Worker, ensuring that you always start from a clean state.
 </aside>
@@ -304,7 +306,7 @@ In the Network panel, you should see an initial set of request for files like `m
 
 The gear icon signifies that these requests came from the Service Worker itself. Specifically, these are the requests being made by the Service Worker's `install` handler to populate the offline cache.
 
-<aside markdown="1" class="special">
+<aside class="key-point">
 
 __Learn More__: For a deeper understanding of the Network panel identifies Service Worker traffic take a look at  [this StackOverflow discussion](http://stackoverflow.com/a/33655173/385997).
 </aside>
@@ -393,7 +395,7 @@ Now our response times jump down to a blazing fast few milliseconds per resource
 
 ![f0f6d3b0a1b1f18d.png](img/f0f6d3b0a1b1f18d.png)
 
-<aside markdown="1" class="warning">
+<aside class="warning">
 
 Before proceeding make sure you set the __Network Throttle__ back to `No throttling`
 </aside>
@@ -437,7 +439,7 @@ The application will pause execution and switch panels over to __Sources__ where
 
 ![d960b322c020d6cc.png](img/d960b322c020d6cc.png)
 
-<aside markdown="1" class="special">
+<aside class="key-point">
 
 __Learn More__: A full explanation of the __Sources__ panel is outside the scope of this codelab but you can  [learn more about the debugging capabilities of the DevTools](/web/tools/chrome-devtools/debug/?hl=en) on the Google Developers site.
 </aside>
@@ -494,7 +496,7 @@ You may have noticed a button in the center of the application asking for the us
 
 ![3e7f08f9d8c1fc5c.png](img/3e7f08f9d8c1fc5c.png)
 
-<aside markdown="1" class="warning">
+<aside class="warning">
 
 The code used to set up this Push subscription is just for demo purposes and should not be used in production. For a thorough guide on setting up Push notifications  [see this post](/web/updates/2015/03/push-notifications-on-the-open-web?hl=en) on the Google Developers site.
 </aside>
@@ -546,4 +548,4 @@ Now that you have some debugging tools in your toolbox, you should be well equip
 
 ## Found an issue, or have feedback? {: .hide-from-toc }
 Help us make our code labs better by submitting an 
-[issue](https://github.com/googlecodelabs/debugging-service-workers/issues) today. And thanks!
+[issue](https://github.com/googlecodelabs/debugging-service-workers/issues) today. And thanks!