Merge pull request lichess-org#15716 from schlawg/ui-hash-linked-assets

asset hash linking

.gitignore

@@ -14,14 +14,15 @@ project/metals.sbt
 project/project
 project/target
 public/compiled
+public/hashed
 public/npm
 public/lifat
 public/css/
+public/json
 target
 data/
 dist/
 node_modules/
-local/
 ui/common/css/theme/gen/*.scss
 ui/common/**/*.js
 ui/common/**/*.d.ts

modules/web/src/main/AssetManifest.scala

@@ -10,18 +10,24 @@ import java.nio.file.Files
 import lila.core.config.NetConfig
 
 case class SplitAsset(name: String, imports: List[String])
-case class AssetMaps(js: Map[String, SplitAsset], css: Map[String, String], modified: Instant)
+case class AssetMaps(
+    js: Map[String, SplitAsset],
+    css: Map[String, String],
+    hashed: Map[String, String],
+    modified: Instant
+)
 
 final class AssetManifest(environment: Environment, net: NetConfig)(using ws: StandaloneWSClient)(using
     Executor
 ) extends lila.ui.AssetManifest:
-  private var maps: AssetMaps = AssetMaps(Map.empty, Map.empty, java.time.Instant.MIN)
+  private var maps: AssetMaps = AssetMaps(Map.empty, Map.empty, Map.empty, java.time.Instant.MIN)
 
   private val filename = s"manifest.${if net.minifiedAssets then "prod" else "dev"}.json"
   private val logger   = lila.log("assetManifest")
 
   def js(key: String): Option[SplitAsset]    = maps.js.get(key)
   def css(key: String): Option[String]       = maps.css.get(key)
+  def hashed(path: String): Option[String]   = maps.hashed.get(path)
   def deps(keys: List[String]): List[String] = keys.flatMap { key => js(key).so(_.imports) }.distinct
   def lastUpdate: Instant                    = maps.modified
 
@@ -83,7 +89,15 @@ final class AssetManifest(environment: Environment, net: NetConfig)(using ws: St
         (k, s"$k.$hash.css")
       }
       .toMap
-    AssetMaps(js, css, nowInstant)
+    val hashed = (manifest \ "hashed")
+      .as[JsObject]
+      .value
+      .map { (k, asset) =>
+        val hashedName = (asset \ "hash").as[String]
+        (k, s"hashed/$hashedName")
+      }
+      .toMap
+    AssetMaps(js, css, hashed, nowInstant)
 
   private def fetchManifestJson(filename: String) =
     val resource = s"${net.assetBaseUrlInternal}/assets/compiled/$filename"

modules/web/src/main/helper/AssetFullHelper.scala

@@ -30,7 +30,8 @@ trait AssetFullHelper:
 
   def assetVersion = lila.core.net.AssetVersion.current
 
-  def assetUrl(path: String): String = s"$assetBaseUrl/assets/_$assetVersion/$path"
+  def assetUrl(path: String): String =
+    s"$assetBaseUrl/assets/${manifest.hashed(path).getOrElse(s"_$assetVersion/$path")}"
 
   private val dataCssKey = attr("data-css-key")
   def cssTag(key: String): Frag =

ui/.build/src/build.ts

@@ -7,7 +7,7 @@ import { sass, stopSass } from './sass';
 import { esbuild, stopEsbuild } from './esbuild';
 import { copies, stopCopies } from './copies';
 import { startMonitor, stopMonitor } from './monitor';
-import { initManifest, writeManifest } from './manifest';
+import { writeManifest } from './manifest';
 import { clean } from './clean';
 import { LichessModule, env, errorMark, colors as c } from './main';
 
@@ -32,11 +32,11 @@ export async function build(mods: string[]) {
   await Promise.allSettled([
     fs.promises.mkdir(env.jsDir),
     fs.promises.mkdir(env.cssDir),
+    fs.promises.mkdir(env.hashDir),
     fs.promises.mkdir(env.themeGenDir),
     fs.promises.mkdir(env.cssTempDir),
   ]);
 
-  await initManifest();
   startMonitor(mods);
   await Promise.all([sass(), copies(), esbuild(tsc())]);
 }

ui/.build/src/clean.ts

@@ -18,12 +18,13 @@ const allGlobs = [
   'public/compiled',
   'public/npm',
   'public/css',
+  'public/hashed',
 ];
 
-export async function clean(globs: string[] = allGlobs) {
-  if (!env.clean) return;
+export async function clean(globs?: string[]) {
+  if (!env.clean && !globs) return;
 
-  for (const glob of globs) {
+  for (const glob of globs ?? allGlobs) {
     env.log(`Cleaning '${c.cyan(glob)}'...`);
     for await (const f of fg.stream(glob, { cwd: env.rootDir, ...globOpts })) {
       if (f.includes('ui/.build') && !f.includes('dist/css')) continue;

ui/.build/src/copies.ts

@@ -1,6 +1,7 @@
 import * as fs from 'node:fs';
 import * as path from 'node:path';
-import { globArray } from './parse';
+import { globArray, globArrays } from './parse';
+import { hashedManifest, writeManifest } from './manifest';
 import { Sync, env, errorMark, colors as c } from './main';
 
 const syncWatch: fs.FSWatcher[] = [];
@@ -18,30 +19,47 @@ export async function copies() {
   const watched = new Map<string, Sync[]>();
   const updated = new Set<string>();
 
-  const fire = () => {
-    updated.forEach(d => watched.get(d)?.forEach(globSync));
-    updated.clear();
-    watchTimeout = undefined;
-  };
   for (const mod of env.building) {
-    if (!mod?.sync) continue;
-    for (const cp of mod.sync) {
+    for (const cp of mod.sync ?? []) {
       for (const src of await globSync(cp)) {
-        watched.set(src, [...(watched.get(src) ?? []), cp]);
+        if (env.watch) watched.set(src, [...(watched.get(src) ?? []), cp]);
       }
     }
     if (!env.watch) continue;
+    const sources = await globArrays(mod.hashGlobs, { cwd: env.outDir });
+
+    for (const src of sources.filter(isUnmanagedAsset)) {
+      if (!watched.has(path.dirname(src))) watched.set(path.dirname(src), []);
+    }
+  }
+  if (env.watch)
     for (const dir of watched.keys()) {
       const watcher = fs.watch(dir);
+
       watcher.on('change', () => {
         updated.add(dir);
         clearTimeout(watchTimeout);
-        watchTimeout = setTimeout(fire, 2000);
+
+        watchTimeout = setTimeout(() => {
+          Promise.all([...updated].flatMap(d => (watched.get(d) ?? []).map(x => globSync(x)))).then(() => {
+            hashedManifest();
+            writeManifest();
+          });
+          updated.clear();
+          watchTimeout = undefined;
+        }, 2000);
       });
       watcher.on('error', (err: Error) => env.error(err));
       syncWatch.push(watcher);
     }
-  }
+  hashedManifest();
+}
+
+export function isUnmanagedAsset(absfile: string) {
+  if (!absfile.startsWith(env.outDir)) return false;
+  const name = absfile.slice(env.outDir.length + 1);
+  if (['compiled/', 'hashed/', 'css/'].some(dir => name.startsWith(dir))) return false;
+  return true;
 }
 
 async function globSync(cp: Sync): Promise<Set<string>> {
@@ -54,7 +72,7 @@ async function globSync(cp: Sync): Promise<Set<string>> {
       ? cp.src.slice(0, globIndex - 1)
       : path.dirname(cp.src.slice(0, globIndex));
 
-  const srcs = await globArray(cp.src, { cwd: cp.mod.root, abs: false });
+  const srcs = await globArray(cp.src, { cwd: cp.mod.root, absolute: false });
 
   watchDirs.add(path.join(cp.mod.root, globRoot));
   env.log(`[${c.grey(cp.mod.name)}] - Sync '${c.cyan(cp.src)}' to '${c.cyan(cp.dest)}'`);

ui/.build/src/esbuild.ts

@@ -2,7 +2,7 @@ import * as path from 'node:path';
 import * as es from 'esbuild';
 import { preModule } from './build';
 import { env, errorMark, colors as c } from './main';
-import { js as jsManifest } from './manifest';
+import { jsManifest } from './manifest';
 
 const bundles = new Map<string, string>();
 const esbuildCtx: es.BuildContext[] = [];

ui/.build/src/main.ts

@@ -78,6 +78,7 @@ export interface LichessModule {
   post: string[][]; // post-bundle build steps from package.json scripts
   hasTsconfig?: boolean; // fileExists('tsconfig.json')
   bundles?: string[];
+  hashGlobs?: string[];
   sync?: Sync[]; // pre-bundle filesystem copies from package json
 }
 
@@ -157,6 +158,9 @@ class Env {
   get outDir(): string {
     return path.join(this.rootDir, 'public');
   }
+  get hashDir(): string {
+    return path.join(this.outDir, 'hashed');
+  }
   get themeDir(): string {
     return path.join(this.uiDir, 'common', 'css', 'theme');
   }

ui/.build/src/manifest.ts

@@ -4,44 +4,28 @@ import * as fs from 'node:fs';
 import * as crypto from 'node:crypto';
 import * as es from 'esbuild';
 import { env, colors as c, warnMark } from './main';
-import { globArray } from './parse';
+import { globArray, globArrays } from './parse';
+import { isUnmanagedAsset } from './copies';
 import { allSources } from './sass';
 
-type Manifest = { [key: string]: { hash?: string; imports?: string[] } };
+type Manifest = { [key: string]: { hash?: string; imports?: string[]; mtime?: number } };
 
-const current: { js: Manifest; css: Manifest; dirty: boolean } = { js: {}, css: {}, dirty: false };
-let writeTimer: NodeJS.Timeout;
+const current: { js: Manifest; css: Manifest; hashed: Manifest; dirty: boolean } = {
+  js: {},
+  css: {},
+  hashed: {},
+  dirty: false,
+};
 
-export async function initManifest() {
-  if (env.building.length === env.modules.size) return;
-  // we're building a subset of modules. reuse the previousl full manifest for
-  // a shot at changes viewable in the browser, otherwise punt.
-  if (!fs.existsSync(env.manifestFile)) return;
-  if (Object.keys(current.js).length && Object.keys(current.css).length) return;
-  const manifest = JSON.parse(await fs.promises.readFile(env.manifestFile, 'utf-8'));
-  delete manifest.js.manifest;
-  current.js = manifest.js;
-  current.css = manifest.css;
-}
+let writeTimer: NodeJS.Timeout;
 
-export async function writeManifest() {
+export function writeManifest() {
   if (!current.dirty) return;
   clearTimeout(writeTimer);
   writeTimer = setTimeout(write, 500);
 }
 
-export async function css() {
-  const files = await globArray(path.join(env.cssTempDir, '*.css'), { abs: true });
-  const css: { name: string; hash: string }[] = await Promise.all(files.map(hashMove));
-  const newCssManifest: Manifest = {};
-  for (const { name, hash } of css) newCssManifest[name] = { hash };
-  if (isEquivalent(newCssManifest, current.css)) return;
-  current.css = shallowSort({ ...current.css, ...newCssManifest });
-  current.dirty = true;
-  writeManifest();
-}
-
-export async function js(meta: es.Metafile) {
+export async function jsManifest(meta: es.Metafile) {
   const newJsManifest: Manifest = {};
   for (const [filename, info] of Object.entries(meta.outputs)) {
     const out = parsePath(filename);
@@ -64,28 +48,79 @@ export async function js(meta: es.Metafile) {
   current.dirty = true;
 }
 
+export async function cssManifest() {
+  const files = await globArray(path.join(env.cssTempDir, '*.css'));
+  const css: { name: string; hash: string }[] = await Promise.all(files.map(hashMoveCss));
+  const newCssManifest: Manifest = {};
+  for (const { name, hash } of css) newCssManifest[name] = { hash };
+  if (isEquivalent(newCssManifest, current.css)) return;
+  current.css = shallowSort({ ...current.css, ...newCssManifest });
+  current.dirty = true;
+  writeManifest();
+}
+
+export async function hashedManifest() {
+  const newHashLinks = new Map<string, number>();
+  const alreadyHashed = new Map<string, string>();
+  const sources: string[] = (
+    await globArrays(
+      env.building.flatMap(x => x.hashGlobs ?? []),
+      { cwd: env.outDir },
+    )
+  ).filter(isUnmanagedAsset);
+  const sourceStats = await Promise.all(sources.map(file => fs.promises.stat(file)));
+
+  for (const [i, stat] of sourceStats.entries()) {
+    const name = sources[i].slice(env.outDir.length + 1);
+
+    if (stat.mtimeMs === current.hashed[name]?.mtime) alreadyHashed.set(name, current.hashed[name].hash!);
+    else newHashLinks.set(name, stat.mtimeMs);
+  }
+
+  await Promise.allSettled(
+    [...alreadyHashed].map(([name, hash]) =>
+      fs.promises.symlink(path.join(env.outDir, name), path.join(env.hashDir, hash)),
+    ),
+  );
+
+  for (const { name, hash } of await Promise.all([...newHashLinks.keys()].map(hashLink))) {
+    current.hashed[name] = Object.defineProperty({ hash }, 'mtime', { value: newHashLinks.get(name) });
+  }
+
+  if (newHashLinks.size === 0 && alreadyHashed.size === Object.keys(current.hashed).length) return;
+
+  for (const name of Object.keys(current.hashed)) {
+    if (!sources.some(x => x.endsWith(name))) delete current.hashed[name];
+  }
+  current.dirty = true;
+  writeManifest();
+}
+
 async function write() {
   if (!env.manifestOk || !(await isComplete())) return;
   const commitMessage = cps
     .execSync('git log -1 --pretty=%s', { encoding: 'utf-8' })
     .trim()
     .replace(/'/g, '&#39;')
     .replace(/"/g, '&quot;');
+
   const clientJs: string[] = [
     'if (!window.site) window.site={};',
     'if (!window.site.info) window.site.info={};',
     `window.site.info.commit='${cps.execSync('git rev-parse -q HEAD', { encoding: 'utf-8' }).trim()}';`,
     `window.site.info.message='${commitMessage}';`,
     `window.site.debug=${env.debug};`,
-    'const m=window.site.manifest={css:{},js:{}};',
+    'const m=window.site.manifest={css:{},js:{},hashed:{}};',
   ];
   for (const [name, info] of Object.entries(current.js)) {
     if (!/common\.[A-Z0-9]{8}/.test(name)) clientJs.push(`m.js['${name}']='${info.hash}';`);
   }
   for (const [name, info] of Object.entries(current.css)) {
     clientJs.push(`m.css['${name}']='${info.hash}';`);
   }
-
+  for (const [path, info] of Object.entries(current.hashed)) {
+    clientJs.push(`m.hashed[${JSON.stringify(path)}]='${info.hash}';`);
+  }
   const hashable = clientJs.join('\n');
   const hash = crypto.createHash('sha256').update(hashable).digest('hex').slice(0, 8);
   // add the date after hashing
@@ -94,7 +129,11 @@ async function write() {
     `\nwindow.site.info.date='${
       new Date(new Date().toUTCString()).toISOString().split('.')[0] + '+00:00'
     }';\n`;
-  const serverManifest = { js: { manifest: { hash }, ...current.js }, css: { ...current.css } };
+  const serverManifest = {
+    js: { manifest: { hash }, ...current.js },
+    css: { ...current.css },
+    hashed: { ...current.hashed },
+  };
 
   await Promise.all([
     fs.promises.writeFile(path.join(env.jsDir, `manifest.${hash}.js`), clientManifest),
@@ -107,7 +146,7 @@ async function write() {
   env.log(`Manifest hash ${c.green(hash)}`);
 }
 
-async function hashMove(src: string) {
+async function hashMoveCss(src: string) {
   const content = await fs.promises.readFile(src, 'utf-8');
   const hash = crypto.createHash('sha256').update(content).digest('hex').slice(0, 8);
   const basename = path.basename(src, '.css');
@@ -118,6 +157,19 @@ async function hashMove(src: string) {
   return { name: path.basename(src, '.css'), hash };
 }
 
+async function hashLink(name: string) {
+  const src = path.join(env.outDir, name);
+  const hash =
+    crypto
+      .createHash('sha256')
+      .update(await fs.promises.readFile(src))
+      .digest('base64url')
+      .slice(0, 8) + path.extname(src);
+  const link = path.join(env.hashDir, hash);
+  fs.promises.symlink(path.join('..', name), link).catch(() => {});
+  return { name, hash };
+}
+
 async function isComplete() {
   for (const bundle of [...env.modules.values()].map(x => x.bundles ?? []).flat()) {
     const name = path.basename(bundle, '.ts');