added memory comparison to tests

liabru · Nov 20, 2021 · bedf84c · bedf84c
1 parent cd289ec
commit bedf84c
Show file tree

Hide file tree

Showing 2 changed files with 26 additions and 9 deletions.
diff --git a/test/ExampleWorker.js b/test/ExampleWorker.js
@@ -30,8 +30,6 @@ const runExample = options => {
   const consoleOriginal = global.console;
   const logs = [];
 
-  global.gc();
-
   mock('matter-js', Matter);
   global.Matter = Matter;
 
@@ -47,6 +45,7 @@ const runExample = options => {
   const example = Example[options.name]();
   const engine = example.engine;
 
+  let totalMemory = 0;
   let totalDuration = 0;
   let overlapTotal = 0;
   let overlapCount = 0;
@@ -64,13 +63,17 @@ const runExample = options => {
     }
   }
 
+  global.gc();
+
   for (let i = 0; i < options.totalUpdates; i += 1) {
       const startTime = process.hrtime();
-
+      totalMemory += process.memoryUsage().heapUsed;
+
       Matter.Engine.update(engine, 1000 / 60);
 
       const duration = process.hrtime(startTime);
       totalDuration += duration[0] * 1e9 + duration[1];
+      totalMemory += process.memoryUsage().heapUsed;
 
       for (let p = 0; p < engine.pairs.list.length; p += 1) {
         const pair = engine.pairs.list[p];
@@ -93,6 +96,7 @@ const runExample = options => {
     name: options.name,
     duration: totalDuration,
     overlap: overlapTotal / (overlapCount || 1),
+    memory: totalMemory,
     logs,
     ...engineCapture(engine)
   };

diff --git a/test/TestTools.js b/test/TestTools.js
@@ -32,6 +32,13 @@ const colors = { Red: 31, Green: 32, Yellow: 33, White: 37, BrightWhite: 90, Bri
 const color = (text, number) => number ? `\x1b[${number}m${text}\x1b[0m` : text;
 const limit = (val, precision=3) => parseFloat(val.toPrecision(precision));
 const toPercent = val => (100 * val).toPrecision(3);
+const toPercentRound = val => Math.round(100 * val);
+
+const noiseThreshold = (val, threshold) => {
+    const sign = val < 0 ? -1 : 1;
+    const magnitude = Math.abs(val);
+    return sign * Math.max(0, magnitude - threshold) / (1 - threshold);
+};
 
 const engineCapture = (engine) => ({
     timestamp: limit(engine.timing.timestamp),
@@ -231,13 +238,19 @@ const comparisonReport = (capturesDev, capturesBuild, buildVersion, save) => {
     let totalTimeDev = 0;
     let totalOverlapBuild = 0;
     let totalOverlapDev = 0;
+    let totalMemoryBuild = 0;
+    let totalMemoryDev = 0;
 
     const capturePerformance = Object.entries(capturesDev).map(([name]) => {
         totalTimeBuild += capturesBuild[name].duration;
         totalTimeDev += capturesDev[name].duration;
+
         totalOverlapBuild += capturesBuild[name].overlap;
         totalOverlapDev += capturesDev[name].overlap;
 
+        totalMemoryBuild += capturesBuild[name].memory;
+        totalMemoryDev += capturesDev[name].memory;
+
         const changedIntrinsics = !equals(capturesDev[name].intrinsic, capturesBuild[name].intrinsic);
         if (changedIntrinsics) {
             capturesDev[name].changedIntrinsics = true;
@@ -254,10 +267,8 @@ const comparisonReport = (capturesDev, capturesBuild, buildVersion, save) => {
     capturePerformance.sort((a, b) => a.name.localeCompare(b.name));
     similarityEntries.sort((a, b) => a[1] - b[1]);
 
-    let perfChange = 1 - (totalTimeDev / totalTimeBuild);
-
-    const perfChangeThreshold = 0.01;
-    perfChange = Math.abs(perfChange) > perfChangeThreshold ? perfChange : 0;
+    let perfChange = noiseThreshold(1 - (totalTimeDev / totalTimeBuild), 0.01);
+    let memoryChange = noiseThreshold((totalMemoryDev / totalMemoryBuild) - 1, 0.01);
 
     let similarityAvg = 0;
     similarityEntries.forEach(([_, similarity]) => {
@@ -275,9 +286,11 @@ const comparisonReport = (capturesDev, capturesBuild, buildVersion, save) => {
         `\n\n${format('Similarity', colors.White)}`,
         `${format(toPercent(similarityAvg), similarityAvg === 1 ? colors.Green : colors.Yellow)}%`,
         `${format('Performance', colors.White)}`,
-        `${format((perfChange >= 0 ? '+' : '') + toPercent(perfChange), perfChange >= 0 ? colors.Green : colors.Red)}%`,
+        `${format((perfChange >= 0 ? '+' : '-') + toPercentRound(Math.abs(perfChange)), perfChange >= 0 ? colors.Green : colors.Red)}%`,
+        `${format('Memory', colors.White)}`,
+        `${format((memoryChange >= 0 ? '+' : '-') + toPercentRound(Math.abs(memoryChange)), memoryChange <= 0 ? colors.Green : colors.Red)}%`,
         `${format('Overlap', colors.White)}`,
-        `${format((overlapChange >= 0 ? '+' : '') + toPercent(overlapChange), overlapChange > 0 ? colors.Red : colors.Green)}%`,
+        `${format((overlapChange >= 0 ? '+' : '-') + toPercent(Math.abs(overlapChange)), overlapChange <= 0 ? colors.Green : colors.Red)}%`,
         capturePerformance.reduce((output, p, i) => {
             output += `${p.name} `;
             output += `${similarityRatings(similaritys[p.name])} `;