[react-packager] Injectible file crawlers (2x crawl speedup)

packager/react-packager/src/DependencyResolver/DependencyGraph/__tests__/DependencyGraph-test.js

@@ -13,6 +13,8 @@ jest
   .dontMock('crypto')
   .dontMock('absolute-path')
   .dontMock('../docblock')
+  .dontMock('../../crawlers')
+  .dontMock('../../crawlers/node')
   .dontMock('../../replacePatterns')
   .dontMock('../../../lib/getAssetDataFromName')
   .dontMock('../../fastfs')
@@ -22,6 +24,8 @@ jest
   .dontMock('../../Package')
   .dontMock('../../ModuleCache');
 
+const Promise = require('promise');
+
 jest.mock('fs');
 
 describe('DependencyGraph', function() {
@@ -36,7 +40,8 @@ describe('DependencyGraph', function() {
     fileWatcher = {
       on: function() {
         return this;
-      }
+      },
+      isWatchman: () => Promise.resolve(false)
     };
   });
 

packager/react-packager/src/DependencyResolver/DependencyGraph/index.js

@@ -8,17 +8,19 @@
  */
 'use strict';
 
-const path = require('path');
+const Activity = require('../../Activity');
+const AssetModule_DEPRECATED = require('../AssetModule_DEPRECATED');
 const Fastfs = require('../fastfs');
 const ModuleCache = require('../ModuleCache');
-const AssetModule_DEPRECATED = require('../AssetModule_DEPRECATED');
-const declareOpts = require('../../lib/declareOpts');
-const isAbsolutePath = require('absolute-path');
+const Promise = require('promise');
+const _ = require('underscore');
+const crawl = require('../crawlers');
 const debug = require('debug')('DependencyGraph');
+const declareOpts = require('../../lib/declareOpts');
 const getAssetDataFromName = require('../../lib/getAssetDataFromName');
+const isAbsolutePath = require('absolute-path');
+const path = require('path');
 const util = require('util');
-const Promise = require('promise');
-const _ = require('underscore');
 
 const validateOpts = declareOpts({
   roots: {
@@ -68,13 +70,18 @@ class DependencyGraph {
       return this._loading;
     }
 
-    const modulePattern = new RegExp(
-      '\.(' + ['js', 'json'].concat(this._assetExts).join('|') + ')$'
-    );
+    const crawlActivity = Activity.startEvent('fs crawl');
+    const allRoots = this._opts.roots.concat(this._opts.assetRoots_DEPRECATED);
+    this._crawling = crawl(allRoots, {
+      ignore: this._opts.ignoreFilePath,
+      exts: ['js', 'json'].concat(this._opts.assetExts),
+      fileWatcher: this._opts.fileWatcher,
+    });
+    this._crawling.then((files) => Activity.endEvent(crawlActivity));
 
     this._fastfs = new Fastfs(this._opts.roots,this._opts.fileWatcher, {
-      pattern: modulePattern,
       ignore: this._opts.ignoreFilePath,
+      crawling: this._crawling,
     });
 
     this._fastfs.on('change', this._processFileChange.bind(this));
@@ -454,14 +461,10 @@ class DependencyGraph {
 
     this._assetMap_DEPRECATED = Object.create(null);
 
-    const pattern = new RegExp(
-      '\.(' + this._opts.assetExts.join('|') + ')$'
-    );
-
     const fastfs = new Fastfs(
       this._opts.assetRoots_DEPRECATED,
       this._opts.fileWatcher,
-      { pattern, ignore: this._opts.ignoreFilePath }
+      { ignore: this._opts.ignoreFilePath, crawling: this._crawling }
     );
 
     fastfs.on('change', this._processAssetChange_DEPRECATED.bind(this));

packager/react-packager/src/DependencyResolver/crawlers/index.js

@@ -0,0 +1,36 @@
+'use strict';
+
+const nodeCrawl = require('./node');
+//const watchmanCrawl = require('./watchman');
+
+function crawl(roots, options) {
+  return nodeCrawl(roots, options);
+
+  // Although, in theory, watchman should be much faster;
+  // there is currently a bottleneck somewhere in the
+  // encoding/decoding that is causing it to be slower
+  // than node crawling. However, this should be fixed soon.
+  // https://github.com/facebook/watchman/issues/113
+  /*
+  const {fileWatcher} = options;
+  return fileWatcher.isWatchman().then(isWatchman => {
+
+    console.log(isWatchman);
+    if (!isWatchman) {
+      return false;
+    }
+
+    // Make sure we're dealing with a version of watchman
+    // that's using `watch-project`
+    // TODO(amasad): properly expose (and document) used sane internals.
+    return fileWatcher.getWatchers().then(([watcher]) => !!watcher.watchProjectInfo.root);
+  }).then(isWatchman => {
+    if (isWatchman) {
+      return watchmanCrawl(roots, options);
+    }
+
+    return nodeCrawl(roots, options);
+  });*/
+}
+
+module.exports = crawl;

packager/react-packager/src/DependencyResolver/crawlers/node.js

@@ -0,0 +1,61 @@
+'use strict';
+
+const Promise = require('promise');
+const debug = require('debug')('DependencyGraph');
+const fs = require('fs');
+const path = require('path');
+
+const readDir = Promise.denodeify(fs.readdir);
+const stat = Promise.denodeify(fs.stat);
+
+function nodeRecReadDir(roots, {ignore, exts}) {
+  const queue = roots.slice();
+  const retFiles = [];
+  const extPattern = new RegExp(
+    '\.(' + exts.join('|') + ')$'
+  );
+
+  function search() {
+    const currDir = queue.shift();
+    if (!currDir) {
+      return Promise.resolve();
+    }
+
+    return readDir(currDir)
+      .then(files => files.map(f => path.join(currDir, f)))
+      .then(files => Promise.all(
+        files.map(f => stat(f).catch(handleBrokenLink))
+      ).then(stats => [
+        // Remove broken links.
+        files.filter((file, i) => !!stats[i]),
+        stats.filter(Boolean),
+      ]))
+      .then(([files, stats]) => {
+        files.forEach((filePath, i) => {
+          if (ignore(filePath)) {
+            return;
+          }
+
+          if (stats[i].isDirectory()) {
+            queue.push(filePath);
+            return;
+          }
+
+          if (filePath.match(extPattern)) {
+            retFiles.push(filePath);
+          }
+        });
+
+        return search();
+      });
+  }
+
+  return search().then(() => retFiles);
+}
+
+function handleBrokenLink(e) {
+  debug('WARNING: error stating, possibly broken symlink', e.message);
+  return Promise.resolve();
+}
+
+module.exports = nodeRecReadDir;

packager/react-packager/src/DependencyResolver/crawlers/watchman.js

@@ -0,0 +1,70 @@
+'use strict';
+
+const Promise = require('promise');
+const path = require('path');
+
+function watchmanRecReadDir(roots, {ignore, fileWatcher, exts}) {
+  const files = [];
+  return Promise.all(
+    roots.map(
+      root => fileWatcher.getWatcherForRoot(root)
+    )
+  ).then(
+    watchers => {
+      // All watchman roots for all watches we have.
+      const watchmanRoots = watchers.map(
+        watcher => watcher.watchProjectInfo.root
+      );
+
+      // Actual unique watchers (because we use watch-project we may end up with
+      // duplicate "real" watches, and that's by design).
+      // TODO(amasad): push this functionality into the `FileWatcher`.
+      const uniqueWatchers = watchers.filter(
+        (watcher, i) => watchmanRoots.indexOf(watcher.watchProjectInfo.root) === i
+      );
+
+      return Promise.all(
+        uniqueWatchers.map(watcher => {
+          const watchedRoot = watcher.watchProjectInfo.root;
+
+          // Build up an expression to filter the output by the relevant roots.
+          const dirExpr = ['anyof'];
+          for (let i = 0; i < roots.length; i++) {
+            const root = roots[i];
+            if (isDescendant(watchedRoot, root)) {
+              dirExpr.push(['dirname', path.relative(watchedRoot, root)]);
+            }
+          }
+
+          const cmd = Promise.promisify(watcher.client.command.bind(watcher.client));
+          return cmd(['query', watchedRoot, {
+            'suffix': exts,
+            'expression': ['allof', ['type', 'f'], 'exists', dirExpr],
+            'fields': ['name'],
+          }]).then(resp => {
+            if ('warning' in resp) {
+              console.warn('watchman warning: ', resp.warning);
+            }
+
+            resp.files.forEach(filePath => {
+              filePath = path.join(
+                watchedRoot,
+                filePath
+              );
+
+              if (!ignore(filePath)) {
+                files.push(filePath);
+              }
+              return false;
+            });
+          });
+        })
+      );
+    }).then(() => files);
+}
+
+function isDescendant(root, child) {
+  return path.relative(root, child).indexOf('..') !== 0;
+}
+
+module.exports = watchmanRecReadDir;