feat: add withIterator

__tests__/symlinks.test.ts

@@ -1,7 +1,7 @@
-import { afterAll, beforeAll, beforeEach, describe, test } from "vitest";
-import { apiTypes, normalize, root } from "./utils";
+import { afterAll, beforeAll, describe, test } from "vitest";
+import { apiTypes, normalize, root, execute } from "./utils";
 import mock from "mock-fs";
-import { fdir, Options } from "../src";
+import { fdir } from "../src";
 import path from "path";
 
 const fsWithRelativeSymlinks = {
@@ -154,7 +154,7 @@ for (const type of apiTypes) {
 
     test(`resolve symlinks`, async (t) => {
       const api = new fdir().withSymlinks().crawl("/some/dir");
-      const files = await api[type]();
+      const files = await execute(api, type);
       t.expect(files.sort()).toStrictEqual(
         normalize([
           "/other/dir/file-2",
@@ -166,7 +166,7 @@ for (const type of apiTypes) {
 
     test(`resolve recursive symlinks`, async (t) => {
       const api = new fdir().withSymlinks().crawl("/recursive");
-      const files = await api[type]();
+      const files = await execute(api, type);
       t.expect(files.sort()).toStrictEqual(
         normalize([
           "/double/recursive/another-file",
@@ -184,7 +184,7 @@ for (const type of apiTypes) {
       const api = new fdir()
         .withSymlinks({ resolvePaths: false })
         .crawl("/recursive");
-      const files = await api[type]();
+      const files = await execute(api, type);
       t.expect(files.sort()).toStrictEqual(
         normalize([
           "/recursive/dir/not-recursive/another-file",
@@ -234,7 +234,7 @@ for (const type of apiTypes) {
         .withRelativePaths()
         .withErrors()
         .crawl("./recursive");
-      const files = await api[type]();
+      const files = await execute(api, type);
       t.expect(files.sort()).toStrictEqual(
         normalize([
           "dir/not-recursive/another-file",
@@ -284,7 +284,7 @@ for (const type of apiTypes) {
         .withRelativePaths()
         .withErrors()
         .crawl("./recursive");
-      const files = await api[type]();
+      const files = await execute(api, type);
       t.expect(files.sort()).toStrictEqual(
         normalize([
           "..//double/recursive/another-file",
@@ -302,7 +302,7 @@ for (const type of apiTypes) {
       const api = new fdir()
         .withSymlinks({ resolvePaths: false })
         .crawl("/some/dir");
-      const files = await api[type]();
+      const files = await execute(api, type);
       t.expect(files.sort()).toStrictEqual(
         normalize([
           "/some/dir/dirSymlink/file-1",
@@ -317,7 +317,7 @@ for (const type of apiTypes) {
         .withSymlinks({ resolvePaths: false })
         .withRelativePaths()
         .crawl("/some/dir");
-      const files = await api[type]();
+      const files = await execute(api, type);
       t.expect(files.sort()).toStrictEqual(
         normalize([
           "dirSymlink/file-1",
@@ -332,7 +332,7 @@ for (const type of apiTypes) {
         .withSymlinks()
         .withRelativePaths()
         .crawl("./relative/dir");
-      const files = await api[type]();
+      const files = await execute(api, type);
       t.expect(files.sort()).toStrictEqual(
         normalize([
           "../../../../other-relative/dir/file-2",
@@ -347,7 +347,7 @@ for (const type of apiTypes) {
         .withSymlinks()
         .exclude((_name, path) => path === resolveSymlinkRoot("/sym/linked/"))
         .crawl("/some/dir");
-      const files = await api[type]();
+      const files = await execute(api, type);
       t.expect(files.sort()).toStrictEqual(normalize(["/other/dir/file-2"]));
     });
 
@@ -358,31 +358,31 @@ for (const type of apiTypes) {
           (_name, path) => path === resolveSymlinkRoot("/some/dir/dirSymlink/")
         )
         .crawl("/some/dir");
-      const files = await api[type]();
+      const files = await execute(api, type);
       t.expect(files.sort()).toStrictEqual(
         normalize(["/some/dir/fileSymlink"])
       );
     });
 
     test(`do not resolve symlinks`, async (t) => {
       const api = new fdir().crawl("/some/dir");
-      const files = await api[type]();
+      const files = await execute(api, type);
       t.expect(files.sort()).toStrictEqual(
         normalize(["dirSymlink", "fileSymlink", "fileSymlink2"])
       );
     });
 
     test(`exclude symlinks`, async (t) => {
       const api = new fdir({ excludeSymlinks: true }).crawl("/some/dir");
-      const files = await api[type]();
+      const files = await execute(api, type);
       t.expect(files).toHaveLength(0);
     });
 
     test(
       "doesn't hang when resolving symlinks in the root directory",
       async (t) => {
         const api = new fdir().withSymlinks({ resolvePaths: false }).crawl("/");
-        const files = await api[type]();
+        const files = await execute(api, type);
         const expectedFiles = normalize(["/lib/file-1", "/usr/lib/file-1"]);
         for (const expectedFile of expectedFiles) {
           t.expect(files).toContain(expectedFile);

__tests__/utils.ts

@@ -1,7 +1,9 @@
 import path from "path";
+import type { APIBuilder } from "../src/builder/api-builder";
+import type { IterableOutput } from "../src/types";
 
 export type APITypes = (typeof apiTypes)[number];
-export const apiTypes = ["withPromise", "sync"] as const;
+export const apiTypes = ["withPromise", "sync", "withIterator"] as const;
 
 export function root() {
   return process.platform === "win32" ? process.cwd().split(path.sep)[0] : "/";
@@ -22,3 +24,19 @@ export function normalize(paths: string[]) {
     path.isAbsolute(p) ? path.resolve(p) : path.normalize(p)
   );
 }
+
+export async function execute<T extends IterableOutput>(
+  api: APIBuilder<T>,
+  type: APITypes
+): Promise<T> {
+  let files: T[number][] = [];
+
+  if (type === "withIterator") {
+    for await (const file of api[type]()) {
+      files.push(file);
+    }
+  } else {
+    files = await api[type]();
+  }
+  return files as T;
+}

src/api/functions/group-files.ts

@@ -3,15 +3,17 @@ import { Group, Options } from "../../types";
 export type GroupFilesFunction = (
   groups: Group[],
   directory: string,
-  files: string[]
+  files: string[],
+  pushGroup: (group: Group, arr: Group[]) => void
 ) => void;
 
 const groupFiles: GroupFilesFunction = (
   groups: Group[],
   directory: string,
-  files: string[]
+  files: string[],
+  pushGroup
 ) => {
-  groups.push({ directory, files, dir: directory });
+  pushGroup({ directory, files, dir: directory }, groups);
 };
 
 const empty: GroupFilesFunction = () => {};

src/api/functions/push-directory.ts

@@ -1,40 +1,53 @@
-import { FilterPredicate, Options } from "../../types";
+import { FilterPredicate, Options, Counts } from "../../types";
 
 export type PushDirectoryFunction = (
   directoryPath: string,
   paths: string[],
+  pushPath: (path: string, arr: string[]) => void,
+  counts: Counts,
   filters?: FilterPredicate[]
 ) => void;
 
 function pushDirectoryWithRelativePath(root: string): PushDirectoryFunction {
-  return function (directoryPath, paths) {
-    paths.push(directoryPath.substring(root.length) || ".");
+  return function (directoryPath, paths, pushPath, counts) {
+    pushPath(directoryPath.substring(root.length) || ".", paths);
+    counts.directories++;
   };
 }
 
 function pushDirectoryFilterWithRelativePath(
   root: string
 ): PushDirectoryFunction {
-  return function (directoryPath, paths, filters) {
+  return function (directoryPath, paths, pushPath, counts, filters) {
     const relativePath = directoryPath.substring(root.length) || ".";
     if (filters!.every((filter) => filter(relativePath, true))) {
-      paths.push(relativePath);
+      pushPath(relativePath, paths);
+      counts.directories++;
     }
   };
 }
 
-const pushDirectory: PushDirectoryFunction = (directoryPath, paths) => {
-  paths.push(directoryPath || ".");
+const pushDirectory: PushDirectoryFunction = (
+  directoryPath,
+  paths,
+  pushPath,
+  counts
+) => {
+  pushPath(directoryPath || ".", paths);
+  counts.directories++;
 };
 
 const pushDirectoryFilter: PushDirectoryFunction = (
   directoryPath,
   paths,
+  pushPath,
+  counts,
   filters
 ) => {
   const path = directoryPath || ".";
   if (filters!.every((filter) => filter(path, true))) {
-    paths.push(path);
+    pushPath(path, paths);
+    counts.directories++;
   }
 };
 

src/api/functions/push-file.ts

@@ -3,13 +3,15 @@ import { FilterPredicate, Options, Counts } from "../../types";
 export type PushFileFunction = (
   directoryPath: string,
   paths: string[],
+  pushPath: (path: string, arr: string[]) => void,
   counts: Counts,
   filters?: FilterPredicate[]
 ) => void;
 
 const pushFileFilterAndCount: PushFileFunction = (
   filename,
   _paths,
+  _pushPath,
   counts,
   filters
 ) => {
@@ -19,23 +21,29 @@ const pushFileFilterAndCount: PushFileFunction = (
 const pushFileFilter: PushFileFunction = (
   filename,
   paths,
-  _counts,
+  pushPath,
+  counts,
   filters
 ) => {
-  if (filters!.every((filter) => filter(filename, false))) paths.push(filename);
+  if (filters!.every((filter) => filter(filename, false))) {
+    pushPath(filename, paths);
+    counts.files++;
+  }
 };
 
 const pushFileCount: PushFileFunction = (
   _filename,
   _paths,
+  _pushPath,
   counts,
   _filters
 ) => {
   counts.files++;
 };
 
-const pushFile: PushFileFunction = (filename, paths) => {
-  paths.push(filename);
+const pushFile: PushFileFunction = (filename, paths, pushPath, counts) => {
+  pushPath(filename, paths);
+  counts.files++;
 };
 
 const empty: PushFileFunction = () => {};

src/api/functions/walk-directory.ts

@@ -25,7 +25,6 @@ const walkAsync: WalkDirectoryFunction = (
   const { fs } = state;
 
   state.visited.push(crawlPath);
-  state.counts.directories++;
 
   // Perf: Node >= 10 introduced withFileTypes that helps us
   // skip an extra fs.stat call.
@@ -46,7 +45,6 @@ const walkSync: WalkDirectoryFunction = (
   const { fs } = state;
   if (currentDepth < 0) return;
   state.visited.push(crawlPath);
-  state.counts.directories++;
 
   let entries: Dirent[] = [];
   try {

src/api/iterator.ts

@@ -0,0 +1,90 @@
+import { Options, IterableOutput, OutputIterator } from "../types";
+import { Walker } from "./walker";
+
+class WalkerIterator<TOutput extends IterableOutput> {
+  #resolver?: () => void;
+  #walker: Walker<TOutput>;
+  #currentGroup?: string[];
+  #queue: TOutput[number][] = [];
+  #error?: unknown;
+
+  public constructor(root: string, options: Options) {
+    const pushPath = options.group ? this.#pushPath : this.#pushResult;
+    this.#walker = new Walker<TOutput>(
+      root,
+      options,
+      this.#onComplete,
+      pushPath,
+      this.#pushResult
+    );
+  }
+
+  #pushPath = (path: string, arr: string[]) => {
+    if (arr !== this.#currentGroup) {
+      this.#currentGroup = arr;
+    }
+    arr.push(path);
+  };
+
+  #pushResult = async (result: TOutput[number]) => {
+    this.#queue.push(result);
+    if (this.#resolver) {
+      const resolver = this.#resolver;
+      this.#resolver = undefined;
+      resolver();
+    }
+  };
+
+  #onComplete = (err: unknown) => {
+    this.#currentGroup = undefined;
+    this.#complete = true;
+    if (err) {
+      this.#error = err;
+    }
+    if (this.#resolver) {
+      const resolver = this.#resolver;
+      this.#resolver = undefined;
+      resolver();
+    }
+  };
+
+  async *start(): OutputIterator<TOutput> {
+    this.#walker.start();
+
+    try {
+      while (true) {
+        for (const item of this.#queue) {
+          if (this.#walker.aborted) {
+            break;
+          }
+          yield item;
+        }
+        this.#queue = [];
+
+        if (this.#error) {
+          throw this.#error;
+        }
+
+        if (this.#complete || this.#walker.aborted) {
+          return;
+        }
+
+        await new Promise<void>((resolve) => {
+          this.#resolver = resolve;
+        });
+      }
+    } finally {
+      this.#walker.stop();
+    }
+  }
+
+  #complete: boolean = false;
+}
+
+export function iterator<TOutput extends IterableOutput>(
+  root: string,
+  options: Options
+): AsyncIterable<TOutput[number]> {
+  const walker = new WalkerIterator<TOutput>(root, options);
+  return walker.start();
+}