refactor: implement new payload cache

Author: fraxken

bin/index.js

@@ -124,7 +124,6 @@ prog
   .command("cache")
   .option("-l, --list", i18n.getTokenSync("cli.commands.cache.option_list"), false)
   .option("-c, --clear", i18n.getTokenSync("cli.commands.cache.option_clear"), false)
-  .option("-f, --full", i18n.getTokenSync("cli.commands.cache.option_full"), false)
   .describe(i18n.getTokenSync("cli.commands.cache.desc"))
   .action(commands.cache.main);
 

i18n/english.js

@@ -72,9 +72,7 @@ const cli = {
       missingAction: "No valid action specified. Use --help to see options.",
       option_list: "List cache files",
       option_clear: "Clear the cache",
-      option_full: "Clear or list the full cache, including payloads",
       cacheTitle: "NodeSecure Cache:",
-      scannedPayloadsTitle: "Scanned payloads available on disk:",
       cleared: "Cache cleared successfully!"
     },
     extractIntegrity: {

i18n/french.js

@@ -72,9 +72,7 @@ const cli = {
       missingAction: "Aucune action valide spécifiée. Utilisez --help pour voir les options.",
       option_list: "Lister les fichiers du cache",
       option_clear: "Nettoyer le cache",
-      option_full: "Nettoyer ou lister le cache complet, y compris les payloads",
       cacheTitle: "Cache NodeSecure:",
-      scannedPayloadsTitle: "Payloads scannés disponibles sur le disque:",
       cleared: "Cache nettoyé avec succès !"
     },
     extractIntegrity: {

public/components/views/search/search.css

@@ -289,7 +289,7 @@ body.dark .description {
   height: 20px;
 }
 
-.cache-packages, .recent-packages {
+.cache-packages {
   display: flex;
   flex-direction: column;
   align-items: center;
@@ -306,20 +306,20 @@ body.dark .description {
   overflow: auto;
 }
 
-body.dark .cache-packages, body.dark .recent-packages {
+body.dark .cache-packages {
   color: var(--secondary-darker);
 }
 
-.cache-packages h1, .recent-packages h1 {
+.cache-packages h1 {
   font-family: mononoki;
   color: #546884;
 }
 
-body.dark .cache-packages h1, body.dark .recent-packages h1 {
+body.dark .cache-packages h1 {
   color: white;
 }
 
-.cache-packages .package-cache-result:has(span:hover), .recent-packages .package-cache-result:has(span:hover) {
+.cache-packages .package-cache-result:has(span:hover) {
   color: var(--secondary-darker);
   background: #5468842a;
   cursor: pointer;

public/components/views/search/search.html

@@ -13,9 +13,6 @@
         >
       </div>
     </form>
-    <div class="recent-packages">
-      <h1>[[=z.token('search.recentPackages')]]</h1>
-    </div>
     <div class="cache-packages">
       <h1>[[=z.token('search.packagesCache')]]</h1>
     </div>

public/components/views/search/search.js

@@ -56,11 +56,7 @@ export class SearchView {
 
     this.#initializePackages(
       ".cache-packages",
-      window.scannedPackageCache
-    );
-    this.#initializePackages(
-      ".recent-packages",
-      window.recentPackageCache
+      window.cachedSpecs
     );
   }
 

public/core/search-nav.js

@@ -42,22 +42,25 @@ export function initSearchNav(data, options) {
   }
 }
 
-function initPackagesNavigation(data) {
+function initPackagesNavigation(
+  metadata
+) {
   const fragment = document.createDocumentFragment();
-  const packages = data.mru;
 
-  const hasAtLeast2Packages = packages.length > 1;
-  const hasExactly2Packages = packages.length === 2;
+  const hasAtLeast2Packages = metadata.length > 1;
+  const hasExactly2Packages = metadata.length === 2;
   const container = createDOMElement("div", {
     classList: ["packages"]
   });
 
-  if (packages.length === 0) {
+  if (metadata.length === 0) {
     return fragment;
   }
+  const current = metadata[0].spec;
 
-  for (const pkg of packages) {
-    const { name, version, local } = parseNpmSpec(pkg);
+  for (const { spec, scanType } of metadata) {
+    const local = scanType === "cwd";
+    const { name, version } = parseNpmSpec(spec);
 
     const childs = [
       createDOMElement("p", { text: name }),
@@ -70,18 +73,18 @@ function initPackagesNavigation(data) {
       classList: ["package"],
       childs
     });
-    pkgElement.dataset.name = pkg;
-    if (pkg === data.current) {
-      window.activePackage = pkg;
+    pkgElement.dataset.name = spec;
+    if (spec === current) {
+      window.activePackage = spec;
       pkgElement.classList.add("active");
     }
     pkgElement.addEventListener("click", () => {
-      if (window.activePackage !== pkg) {
-        window.socket.commands.search(pkg);
+      if (window.activePackage !== spec) {
+        window.socket.commands.search(spec);
       }
     });
 
-    if (hasAtLeast2Packages && pkg !== data.root) {
+    if (hasAtLeast2Packages) {
       pkgElement.appendChild(
         renderPackageRemoveButton(pkgElement.dataset.name, { hasExactly2Packages })
       );

public/main.js

@@ -26,15 +26,14 @@ let searchview;
 let packageInfoOpened = false;
 
 document.addEventListener("DOMContentLoaded", async() => {
-  window.scannedPackageCache = [];
-  window.recentPackageCache = [];
+  window.cachedSpecs = [];
   window.locker = null;
   window.settings = await new Settings().fetchUserConfig();
   window.i18n = await new i18n().fetch();
   window.navigation = new ViewNavigation();
   window.wiki = new Wiki();
 
-  await init();
+  await loadDataSet();
   window.dispatchEvent(
     new CustomEvent(EVENTS.SETTINGS_SAVED, {
       detail: window.settings.config
@@ -74,26 +73,12 @@ async function onSocketInitOrReload(event) {
   const data = event.detail;
   const { cache } = data;
 
-  window.scannedPackageCache = cache.availables;
-  window.recentPackageCache = cache.lru;
+  window.cachedSpecs = cache.map((metadata) => metadata.spec);
   console.log(
-    "[INFO] Older packages are loaded!",
-    window.scannedPackageCache
-  );
-  console.log(
-    "[INFO] Recent packages are loaded!",
-    window.recentPackageCache
+    "[INFO] Cached specs are loaded!",
+    window.cachedSpecs
   );
 
-  initSearchNav(cache, {
-    searchOptions: {
-      nsn,
-      secureDataSet
-    }
-  });
-  searchview.mount();
-  searchview.initialize();
-
   const nsnActivePackage = secureDataSet.linker.get(0);
   const nsnRootPackage = nsnActivePackage ?
     `${nsnActivePackage.name}@${nsnActivePackage.version}` :
@@ -105,34 +90,26 @@ async function onSocketInitOrReload(event) {
   ) {
     // it means we removed the previous active package, which is still active in network, so we need to re-init
     await init();
-
-    // FIXME: initSearchNav is called twice, we need to fix this
-    initSearchNav(cache, {
-      searchOptions: {
-        nsn,
-        secureDataSet
-      }
-    });
   }
+  else if (window.cachedSpecs.length === 0) {
+    await loadDataSet();
+  }
+
+  initSearchNav(cache, {
+    searchOptions: {
+      nsn,
+      secureDataSet
+    }
+  });
+  searchview.mount();
+  searchview.initialize();
 }
 
 async function init(options = {}) {
   const { navigateToNetworkView = false } = options;
 
-  secureDataSet = new NodeSecureDataSet({
-    flagsToIgnore: window.settings.config.ignore.flags,
-    warningsToIgnore: window.settings.config.ignore.warnings,
-    theme: window.settings.config.theme
-  });
-  await secureDataSet.init();
-
-  if (secureDataSet.data === null) {
-    window.navigation.hideMenu("network--view");
-    window.navigation.hideMenu("home--view");
-    window.navigation.setNavByName("search--view");
-
-    searchview ??= new SearchView(null, null);
-
+  const datasetLoaded = await loadDataSet();
+  if (!datasetLoaded) {
     return;
   }
 
@@ -190,6 +167,29 @@ async function init(options = {}) {
   console.log("[INFO] Node-Secure is ready!");
 }
 
+async function loadDataSet() {
+  const config = window.settings.config;
+
+  secureDataSet = new NodeSecureDataSet({
+    flagsToIgnore: config.ignore.flags,
+    warningsToIgnore: config.ignore.warnings,
+    theme: config.theme
+  });
+  await secureDataSet.init();
+
+  if (secureDataSet.data === null) {
+    window.navigation.hideMenu("network--view");
+    window.navigation.hideMenu("home--view");
+    window.navigation.setNavByName("search--view");
+
+    searchview ??= new SearchView(null, null);
+
+    return false;
+  }
+
+  return true;
+}
+
 async function updateShowInfoMenu(params) {
   if (params.nodes.length === 0) {
     window.networkNav.currentNodeParams = null;

src/commands/cache.js

@@ -1,17 +1,16 @@
 // Import Node.js Dependencies
 import { styleText } from "node:util";
-import { setImmediate } from "node:timers/promises";
 
 // Import Third-party Dependencies
 import prettyJson from "@topcli/pretty-json";
 import * as i18n from "@nodesecure/i18n";
-import { cache, config } from "@nodesecure/server";
+import { PayloadCache } from "@nodesecure/cache";
+import { config } from "@nodesecure/server";
 
 export async function main(options) {
   const {
     list,
-    clear,
-    full
+    clear
   } = options;
 
   await i18n.getLocalLang();
@@ -22,40 +21,35 @@ export async function main(options) {
   }
 
   if (list) {
-    listCache(full);
+    await listCache();
   }
   if (clear) {
-    await setImmediate();
-    await clearCache(full);
+    await clearCache();
   }
+  console.log();
 }
 
-async function listCache(full) {
-  const paylodsList = await cache.payloadsList();
-  console.log(styleText(["underline"], i18n.getTokenSync("cli.commands.cache.cacheTitle")));
-  prettyJson(paylodsList);
-
-  if (full) {
-    console.log(styleText(["underline"], i18n.getTokenSync("cli.commands.cache.scannedPayloadsTitle")));
-    try {
-      const payloads = cache.availablePayloads();
-      prettyJson(payloads);
-    }
-    catch {
-      prettyJson([]);
-    }
-  }
-}
+async function listCache() {
+  const cache = new PayloadCache();
+  await cache.load();
+
+  const metadata = Array.from(cache);
+  console.log(
+    styleText(["underline"], i18n.getTokenSync("cli.commands.cache.cacheTitle"))
+  );
 
-async function clearCache(full) {
-  if (full) {
-    cache.availablePayloads().forEach((pkg) => {
-      cache.removePayload(pkg);
-    });
+  for (const data of metadata) {
+    prettyJson(data);
   }
+}
 
+async function clearCache() {
   await config.setDefault();
-  await cache.initPayloadsList({ logging: false, reset: true });
 
-  console.log(styleText("green", i18n.getTokenSync("cli.commands.cache.cleared")));
+  const cache = new PayloadCache();
+  await cache.clear();
+
+  console.log(
+    styleText("green", i18n.getTokenSync("cli.commands.cache.cleared"))
+  );
 }

src/commands/http.js

@@ -1,15 +1,13 @@
 // Import Node.js Dependencies
 import fs from "node:fs";
 import path from "node:path";
-import crypto from "node:crypto";
 
 // Import Third-party Dependencies
 import kleur from "kleur";
 import open from "open";
 import * as SemVer from "semver";
 import * as i18n from "@nodesecure/i18n";
 import {
-  cache,
   logger,
   buildServer,
   WebSocketServerInstanciator
@@ -47,11 +45,8 @@ export async function start(
   if (runFromPayload) {
     assertScannerVersion(dataFilePath);
   }
-  else {
-    cache.prefix = crypto.randomBytes(4).toString("hex");
-  }
 
-  const httpServer = buildServer(dataFilePath, {
+  const { httpServer, cache } = await buildServer(dataFilePath, {
     port: httpPort,
     hotReload: enableDeveloperMode,
     runFromPayload,