From adf03c631ff332392b15f7c30fc84ca1d4d575d8 Mon Sep 17 00:00:00 2001
From: Mila Votradovec <mila@snyk.io>
Date: Fri, 30 Nov 2018 14:54:32 +0000
Subject: [PATCH 1/2] perf: change Promise.all to for..of and await

Promise.all tries to spin all the subprocesses in parallel, but it prevents GC to act properly and results in too high memory consuption (~1GB for testing data). Switching to  loop with await dramatically lowers memory needs (~103MB with same data)
---
 lib/parsers/yarn-lock-parse.ts | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/lib/parsers/yarn-lock-parse.ts b/lib/parsers/yarn-lock-parse.ts
index 6f7059ba..a30859f6 100644
--- a/lib/parsers/yarn-lock-parse.ts
+++ b/lib/parsers/yarn-lock-parse.ts
@@ -79,14 +79,14 @@ export class YarnLockParser implements LockfileParser {
       return depTree;
     }
 
-    await Promise.all(topLevelDeps.map(async (dep) => {
-    if (/^file:/.test(dep.version)) {
-      depTree.dependencies[dep.name] = createPkgTreeFromDep(dep);
-    } else {
-      depTree.dependencies[dep.name] = await this.buildSubTreeRecursiveFromYarnLock(
-        dep, yarnLock, [], strict);
+    for (const dep of topLevelDeps) {
+      if (/^file:/.test(dep.version)) {
+        depTree.dependencies[dep.name] = createPkgTreeFromDep(dep);
+      } else {
+        depTree.dependencies[dep.name] = await this.buildSubTreeRecursiveFromYarnLock(
+          dep, yarnLock, [], strict);
       }
-    }));
+    }
 
     return depTree;
   }
@@ -123,7 +123,7 @@ export class YarnLockParser implements LockfileParser {
       depPath.push(depKey);
       const newDeps = _.entries({...dep.dependencies, ...dep.optionalDependencies});
 
-      await Promise.all(newDeps.map(async ([name, version]) => {
+      for (const [name, version] of newDeps) {
         const newDep: Dep = {
           dev: searchedDep.dev,
           name,
@@ -131,7 +131,7 @@ export class YarnLockParser implements LockfileParser {
         };
         depSubTree.dependencies[name] = await this.buildSubTreeRecursiveFromYarnLock(
           newDep, lockFile, [...depPath]);
-      }));
+      }
     }
 
     return depSubTree;

From 1be86db41b164598d2b0b0b3b2302512a829219d Mon Sep 17 00:00:00 2001
From: Mila Votradovec <mila@snyk.io>
Date: Fri, 30 Nov 2018 15:03:32 +0000
Subject: [PATCH 2/2] fix: Force event loop to tick

Whiles used in web service, yarn.lock parsing is blocking Node event loop and requires a lot of system resources in order to handler more requests. Forcing event loop to tick each 200ms allows other requests to be processed within the same Node process.
---
 lib/event-loop-spinner.ts      | 13 +++++++++++++
 lib/parsers/yarn-lock-parse.ts | 10 ++++++++++
 2 files changed, 23 insertions(+)
 create mode 100644 lib/event-loop-spinner.ts

diff --git a/lib/event-loop-spinner.ts b/lib/event-loop-spinner.ts
new file mode 100644
index 00000000..0726f93b
--- /dev/null
+++ b/lib/event-loop-spinner.ts
@@ -0,0 +1,13 @@
+export class EventLoopSpinner {
+  private lastSpin: number;
+  constructor(private thresholdMs: number = 100) {
+    this.lastSpin = Date.now();
+  }
+  public isStarving(): boolean {
+    return (Date.now() - this.lastSpin) > this.thresholdMs;
+  }
+  public async spin() {
+    this.lastSpin = Date.now();
+    return new Promise((resolve) => setImmediate(resolve));
+  }
+}
diff --git a/lib/parsers/yarn-lock-parse.ts b/lib/parsers/yarn-lock-parse.ts
index a30859f6..d4dcbbd3 100644
--- a/lib/parsers/yarn-lock-parse.ts
+++ b/lib/parsers/yarn-lock-parse.ts
@@ -2,6 +2,8 @@ import * as _ from 'lodash';
 import {LockfileParser, PkgTree, Dep, DepType, ManifestFile,
   getTopLevelDeps, Lockfile, LockfileType, createPkgTreeFromDep} from './';
 import getRuntimeVersion from '../get-node-runtime-version';
+import {setImmediatePromise} from '../set-immediate-promise';
+import { EventLoopSpinner } from '../event-loop-spinner';
 import {
   InvalidUserInputError,
   UnsupportedRuntimeError,
@@ -32,6 +34,7 @@ export interface YarnLockDep {
 export class YarnLockParser implements LockfileParser {
 
   private yarnLockfileParser;
+  private eventLoop: EventLoopSpinner;
 
   constructor() {
     // @yarnpkg/lockfile doesn't work with Node.js < 6 and crashes just after
@@ -42,6 +45,10 @@ export class YarnLockParser implements LockfileParser {
         'Node.js v6 and higher.');
     }
     this.yarnLockfileParser = require('@yarnpkg/lockfile');
+    // 200ms is an arbitrary value based on on testing "average request", which is
+    // processed in ~150ms. Idea is to let those average requests through in one
+    // tick and split only bigger ones.
+    this.eventLoop = new EventLoopSpinner(200);
   }
 
   public parseLockFile(lockFileContents: string): YarnLock {
@@ -134,6 +141,9 @@ export class YarnLockParser implements LockfileParser {
       }
     }
 
+    if (this.eventLoop.isStarving()) {
+        await this.eventLoop.spin();
+    }
     return depSubTree;
   }
 }