fs: add recursive option to readdir and opendir

Adds a naive, linear recursive algorithm for the following methods:
readdir, readdirSync, opendir, opendirSync, and the promise based
equivalents.

Fixes: #34992
PR-URL: #41439
Refs: nodejs/tooling#130
Reviewed-By: Yagiz Nizipli <yagiz@nizipli.com>
Reviewed-By: Matteo Collina <matteo.collina@gmail.com>
Reviewed-By: James M Snell <jasnell@gmail.com>
Reviewed-By: Moshe Atlow <moshe@atlow.co.il>

doc/api/fs.md

@@ -1214,6 +1214,9 @@ a colon, Node.js will open a file system stream, as described by
 <!-- YAML
 added: v12.12.0
 changes:
+  - version: REPLACEME
+    pr-url: https://github.com/nodejs/node/pull/41439
+    description: Added `recursive` option.
   - version:
      - v13.1.0
      - v12.16.0
@@ -1227,6 +1230,8 @@ changes:
   * `bufferSize` {number} Number of directory entries that are buffered
     internally when reading from the directory. Higher values lead to better
     performance but higher memory usage. **Default:** `32`
+  * `recursive` {boolean} Resolved `Dir` will be an {AsyncIterable}
+    containing all sub files and directories. **Default:** `false`
 * Returns: {Promise}  Fulfills with an {fs.Dir}.
 
 Asynchronously open a directory for iterative scanning. See the POSIX
@@ -1260,6 +1265,9 @@ closed after the iterator exits.
 <!-- YAML
 added: v10.0.0
 changes:
+  - version: REPLACEME
+    pr-url: https://github.com/nodejs/node/pull/41439
+    description: Added `recursive` option.
   - version: v10.11.0
     pr-url: https://github.com/nodejs/node/pull/22020
     description: New option `withFileTypes` was added.
@@ -1269,6 +1277,7 @@ changes:
 * `options` {string|Object}
   * `encoding` {string} **Default:** `'utf8'`
   * `withFileTypes` {boolean} **Default:** `false`
+  * `recursive` {boolean} **Default:** `false`
 * Returns: {Promise}  Fulfills with an array of the names of the files in
   the directory excluding `'.'` and `'..'`.
 
@@ -3344,6 +3353,9 @@ Functions based on `fs.open()` exhibit this behavior as well:
 <!-- YAML
 added: v12.12.0
 changes:
+  - version: REPLACEME
+    pr-url: https://github.com/nodejs/node/pull/41439
+    description: Added `recursive` option.
   - version: v18.0.0
     pr-url: https://github.com/nodejs/node/pull/41678
     description: Passing an invalid callback to the `callback` argument
@@ -3362,6 +3374,7 @@ changes:
   * `bufferSize` {number} Number of directory entries that are buffered
     internally when reading from the directory. Higher values lead to better
     performance but higher memory usage. **Default:** `32`
+  * `recursive` {boolean} **Default:** `false`
 * `callback` {Function}
   * `err` {Error}
   * `dir` {fs.Dir}
@@ -3478,6 +3491,9 @@ above values.
 <!-- YAML
 added: v0.1.8
 changes:
+  - version: REPLACEME
+    pr-url: https://github.com/nodejs/node/pull/41439
+    description: Added `recursive` option.
   - version: v18.0.0
     pr-url: https://github.com/nodejs/node/pull/41678
     description: Passing an invalid callback to the `callback` argument
@@ -3507,6 +3523,7 @@ changes:
 * `options` {string|Object}
   * `encoding` {string} **Default:** `'utf8'`
   * `withFileTypes` {boolean} **Default:** `false`
+  * `recursive` {boolean} **Default:** `false`
 * `callback` {Function}
   * `err` {Error}
   * `files` {string\[]|Buffer\[]|fs.Dirent\[]}
@@ -5470,6 +5487,9 @@ object with an `encoding` property specifying the character encoding to use.
 <!-- YAML
 added: v12.12.0
 changes:
+  - version: REPLACEME
+    pr-url: https://github.com/nodejs/node/pull/41439
+    description: Added `recursive` option.
   - version:
      - v13.1.0
      - v12.16.0
@@ -5483,6 +5503,7 @@ changes:
   * `bufferSize` {number} Number of directory entries that are buffered
     internally when reading from the directory. Higher values lead to better
     performance but higher memory usage. **Default:** `32`
+  * `recursive` {boolean} **Default:** `false`
 * Returns: {fs.Dir}
 
 Synchronously open a directory. See opendir(3).
@@ -5526,6 +5547,9 @@ this API: [`fs.open()`][].
 <!-- YAML
 added: v0.1.21
 changes:
+  - version: REPLACEME
+    pr-url: https://github.com/nodejs/node/pull/41439
+    description: Added `recursive` option.
   - version: v10.10.0
     pr-url: https://github.com/nodejs/node/pull/22020
     description: New option `withFileTypes` was added.
@@ -5539,6 +5563,7 @@ changes:
 * `options` {string|Object}
   * `encoding` {string} **Default:** `'utf8'`
   * `withFileTypes` {boolean} **Default:** `false`
+  * `recursive` {boolean} **Default:** `false`
 * Returns: {string\[]|Buffer\[]|fs.Dirent\[]}
 
 Reads the contents of the directory.
@@ -6384,6 +6409,16 @@ The file name that this {fs.Dirent} object refers to. The type of this
 value is determined by the `options.encoding` passed to [`fs.readdir()`][] or
 [`fs.readdirSync()`][].
 
+#### `dirent.path`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+* {string}
+
+The base path that this {fs.Dirent} object refers to.
+
 ### Class: `fs.FSWatcher`
 
 <!-- YAML

lib/fs.js

@@ -1399,6 +1399,36 @@ function mkdirSync(path, options) {
   }
 }
 
+// TODO(Ethan-Arrowood): Make this iterative too
+function readdirSyncRecursive(path, origPath, options) {
+  nullCheck(path, 'path', true);
+  const ctx = { path };
+  const result = binding.readdir(pathModule.toNamespacedPath(path),
+                                 options.encoding, !!options.withFileTypes, undefined, ctx);
+  handleErrorFromBinding(ctx);
+  return options.withFileTypes ?
+    getDirents(path, result).flatMap((dirent) => {
+      return [
+        dirent,
+        ...(dirent.isDirectory() ?
+          readdirSyncRecursive(
+            pathModule.join(path, dirent.name),
+            origPath,
+            options,
+          ) : []),
+      ];
+    }) :
+    result.flatMap((ent) => {
+      const innerPath = pathModule.join(path, ent);
+      const relativePath = pathModule.relative(origPath, innerPath);
+      const stat = binding.internalModuleStat(innerPath);
+      return [
+        relativePath,
+        ...(stat === 1 ? readdirSyncRecursive(innerPath, origPath, options) : []),
+      ];
+    });
+}
+
 /**
  * Reads the contents of a directory.
  * @param {string | Buffer | URL} path
@@ -1416,6 +1446,14 @@ function readdir(path, options, callback) {
   callback = makeCallback(typeof options === 'function' ? options : callback);
   options = getOptions(options);
   path = getValidatedPath(path);
+  if (options.recursive != null) {
+    validateBoolean(options.recursive, 'options.recursive');
+  }
+
+  if (options.recursive) {
+    callback(null, readdirSyncRecursive(path, path, options));
+    return;
+  }
 
   const req = new FSReqCallback();
   if (!options.withFileTypes) {
@@ -1439,12 +1477,21 @@ function readdir(path, options, callback) {
  * @param {string | {
  *   encoding?: string;
  *   withFileTypes?: boolean;
+ *   recursive?: boolean;
  *   }} [options]
  * @returns {string | Buffer[] | Dirent[]}
  */
 function readdirSync(path, options) {
   options = getOptions(options);
   path = getValidatedPath(path);
+  if (options.recursive != null) {
+    validateBoolean(options.recursive, 'options.recursive');
+  }
+
+  if (options.recursive) {
+    return readdirSyncRecursive(path, path, options);
+  }
+
   const ctx = { path };
   const result = binding.readdir(pathModule.toNamespacedPath(path),
                                  options.encoding, !!options.withFileTypes,

lib/internal/fs/dir.js

@@ -2,8 +2,7 @@
 
 const {
   ArrayPrototypePush,
-  ArrayPrototypeSlice,
-  ArrayPrototypeSplice,
+  ArrayPrototypeShift,
   FunctionPrototypeBind,
   ObjectDefineProperty,
   PromiseReject,
@@ -99,13 +98,21 @@ class Dir {
     }
 
     if (this[kDirBufferedEntries].length > 0) {
-      const { 0: name, 1: type } =
-        ArrayPrototypeSplice(this[kDirBufferedEntries], 0, 2);
-      if (maybeSync)
-        process.nextTick(getDirent, this[kDirPath], name, type, callback);
-      else
-        getDirent(this[kDirPath], name, type, callback);
-      return;
+      try {
+        const dirent = ArrayPrototypeShift(this[kDirBufferedEntries]);
+
+        if (this[kDirOptions].recursive && dirent.isDirectory()) {
+          this.readSyncRecursive(dirent);
+        }
+
+        if (maybeSync)
+          process.nextTick(callback, null, dirent);
+        else
+          callback(null, dirent);
+        return;
+      } catch (error) {
+        return callback(error);
+      }
     }
 
     const req = new FSReqCallback();
@@ -120,8 +127,16 @@ class Dir {
         return callback(err, result);
       }
 
-      this[kDirBufferedEntries] = ArrayPrototypeSlice(result, 2);
-      getDirent(this[kDirPath], result[0], result[1], callback);
+      try {
+        this.processReadResult(this[kDirPath], result);
+        const dirent = ArrayPrototypeShift(this[kDirBufferedEntries]);
+        if (this[kDirOptions].recursive && dirent.isDirectory()) {
+          this.readSyncRecursive(dirent);
+        }
+        callback(null, dirent);
+      } catch (error) {
+        callback(error);
+      }
     };
 
     this[kDirOperationQueue] = [];
@@ -132,6 +147,45 @@ class Dir {
     );
   }
 
+  processReadResult(path, result) {
+    for (let i = 0; i < result.length; i += 2) {
+      ArrayPrototypePush(
+        this[kDirBufferedEntries],
+        getDirent(
+          pathModule.join(path, result[i]),
+          result[i],
+          result[i + 1],
+        ),
+      );
+    }
+  }
+
+  // TODO(Ethan-Arrowood): Review this implementation. Make it iterative.
+  // Can we better leverage the `kDirOperationQueue`?
+  readSyncRecursive(dirent) {
+    const ctx = { path: dirent.path };
+    const handle = dirBinding.opendir(
+      pathModule.toNamespacedPath(dirent.path),
+      this[kDirOptions].encoding,
+      undefined,
+      ctx,
+    );
+    handleErrorFromBinding(ctx);
+    const result = handle.read(
+      this[kDirOptions].encoding,
+      this[kDirOptions].bufferSize,
+      undefined,
+      ctx,
+    );
+
+    if (result) {
+      this.processReadResult(dirent.path, result);
+    }
+
+    handle.close(undefined, ctx);
+    handleErrorFromBinding(ctx);
+  }
+
   readSync() {
     if (this[kDirClosed] === true) {
       throw new ERR_DIR_CLOSED();
@@ -142,9 +196,11 @@ class Dir {
     }
 
     if (this[kDirBufferedEntries].length > 0) {
-      const { 0: name, 1: type } =
-          ArrayPrototypeSplice(this[kDirBufferedEntries], 0, 2);
-      return getDirent(this[kDirPath], name, type);
+      const dirent = ArrayPrototypeShift(this[kDirBufferedEntries]);
+      if (this[kDirOptions].recursive && dirent.isDirectory()) {
+        this.readSyncRecursive(dirent);
+      }
+      return dirent;
     }
 
     const ctx = { path: this[kDirPath] };
@@ -160,8 +216,13 @@ class Dir {
       return result;
     }
 
-    this[kDirBufferedEntries] = ArrayPrototypeSlice(result, 2);
-    return getDirent(this[kDirPath], result[0], result[1]);
+    this.processReadResult(this[kDirPath], result);
+
+    const dirent = ArrayPrototypeShift(this[kDirBufferedEntries]);
+    if (this[kDirOptions].recursive && dirent.isDirectory()) {
+      this.readSyncRecursive(dirent);
+    }
+    return dirent;
   }
 
   close(callback) {