Optimize dirList (#435)

* optimize dirlist

* remove istanbul ignore

* decrease concurrency to CPULength - 1

Co-authored-by: KaKa <23028015+climba03003@users.noreply.github.com>
Signed-off-by: Gürgün Dayıoğlu <hey@gurgun.day>

---------

Signed-off-by: Gürgün Dayıoğlu <hey@gurgun.day>
Co-authored-by: KaKa <23028015+climba03003@users.noreply.github.com>

Author: gurgunday

lib/dirList.js

@@ -1,10 +1,60 @@
 'use strict'
 
+const os = require('node:os')
 const path = require('node:path')
 const fs = require('node:fs/promises')
-const pLimit = require('p-limit')
+const fastq = require('fastq')
+const fastqConcurrency = Math.max(1, os.cpus().length - 1)
 
 const dirList = {
+  _getExtendedInfo: async function (dir, info) {
+    const depth = dir.split(path.sep).length
+    const files = await fs.readdir(dir)
+
+    const worker = async (filename) => {
+      const filePath = path.join(dir, filename)
+      let stats
+      try {
+        stats = await fs.stat(filePath)
+      } catch {
+        return
+      }
+
+      if (stats.isDirectory()) {
+        info.totalFolderCount++
+        filePath.split(path.sep).length === depth + 1 && info.folderCount++
+        await dirList._getExtendedInfo(filePath, info)
+      } else {
+        info.totalSize += stats.size
+        info.totalFileCount++
+        filePath.split(path.sep).length === depth + 1 && info.fileCount++
+        info.lastModified = Math.max(info.lastModified, stats.mtimeMs)
+      }
+    }
+    const queue = fastq.promise(worker, fastqConcurrency)
+    await Promise.all(files.map(filename => queue.push(filename)))
+  },
+
+  /**
+   * get extended info about a folder
+   * @param {string} folderPath full path fs dir
+   * @return {Promise<ExtendedInfo>}
+   */
+  getExtendedInfo: async function (folderPath) {
+    const info = {
+      totalSize: 0,
+      fileCount: 0,
+      totalFileCount: 0,
+      folderCount: 0,
+      totalFolderCount: 0,
+      lastModified: 0
+    }
+
+    await dirList._getExtendedInfo(folderPath, info)
+
+    return info
+  },
+
   /**
    * get files and dirs from dir, or error
    * @param {string} dir full path fs dir
@@ -22,8 +72,7 @@ const dirList = {
       return entries
     }
 
-    const limit = pLimit(4)
-    await Promise.all(files.map(filename => limit(async () => {
+    const worker = async (filename) => {
       let stats
       try {
         stats = await fs.stat(path.join(dir, filename))
@@ -33,62 +82,15 @@ const dirList = {
       const entry = { name: filename, stats }
       if (stats.isDirectory()) {
         if (options.extendedFolderInfo) {
-          entry.extendedInfo = await getExtendedInfo(path.join(dir, filename))
+          entry.extendedInfo = await dirList.getExtendedInfo(path.join(dir, filename))
         }
         entries.dirs.push(entry)
       } else {
         entries.files.push(entry)
       }
-    })))
-
-    async function getExtendedInfo (folderPath) {
-      const depth = folderPath.split(path.sep).length
-      let totalSize = 0
-      let fileCount = 0
-      let totalFileCount = 0
-      let folderCount = 0
-      let totalFolderCount = 0
-      let lastModified = 0
-
-      async function walk (dir) {
-        const files = await fs.readdir(dir)
-        const limit = pLimit(4)
-        await Promise.all(files.map(filename => limit(async () => {
-          const filePath = path.join(dir, filename)
-          let stats
-          try {
-            stats = await fs.stat(filePath)
-          } catch {
-            return
-          }
-
-          if (stats.isDirectory()) {
-            totalFolderCount++
-            if (filePath.split(path.sep).length === depth + 1) {
-              folderCount++
-            }
-            await walk(filePath)
-          } else {
-            totalSize += stats.size
-            totalFileCount++
-            if (filePath.split(path.sep).length === depth + 1) {
-              fileCount++
-            }
-            lastModified = Math.max(lastModified, stats.mtimeMs)
-          }
-        })))
-      }
-
-      await walk(folderPath)
-      return {
-        totalSize,
-        fileCount,
-        totalFileCount,
-        folderCount,
-        totalFolderCount,
-        lastModified
-      }
     }
+    const queue = fastq.promise(worker, fastqConcurrency)
+    await Promise.all(files.map(filename => queue.push(filename)))
 
     entries.dirs.sort((a, b) => a.name.localeCompare(b.name))
     entries.files.sort((a, b) => a.name.localeCompare(b.name))
@@ -115,6 +117,7 @@ const dirList = {
     } catch {
       return reply.callNotFound()
     }
+
     const format = reply.request.query.format || options.format
     if (format !== 'html') {
       if (options.jsonFormat !== 'extended') {
@@ -203,7 +206,6 @@ const dirList = {
       return new TypeError('The `list.render` option must be a function and is required with html format')
     }
   }
-
 }
 
 module.exports = dirList

package.json

@@ -35,8 +35,8 @@
     "@fastify/send": "^2.0.0",
     "content-disposition": "^0.5.3",
     "fastify-plugin": "^4.0.0",
-    "glob": "^10.3.4",
-    "p-limit": "^3.1.0"
+    "fastq": "^1.17.0",
+    "glob": "^10.3.4"
   },
   "devDependencies": {
     "@fastify/compress": "^7.0.0",