Performance fixes

package/scripts/common/quarto

@@ -155,4 +155,9 @@ export DENO_NO_UPDATE_CHECK=1
 # Be sure to include any already defined QUARTO_DENO_OPTIONS
 QUARTO_DENO_OPTIONS="--unstable --no-config ${QUARTO_CACHE_OPTIONS} --allow-read --allow-write --allow-run --allow-env --allow-net --allow-ffi ${QUARTO_DENO_OPTIONS}"
 
-"${QUARTO_DENO}" ${QUARTO_ACTION} ${QUARTO_DENO_OPTIONS} ${QUARTO_DENO_EXTRA_OPTIONS} ${QUARTO_IMPORT_ARGMAP} "${QUARTO_TARGET}" "$@"
+if [ "$QUARTO_TS_PROFILE" != "" ]; then
+  QUARTO_DENO_EXTRA_OPTIONS="--inspect-brk ${QUARTO_DENO_EXTRA_OPTIONS}"
+  QUARTO_TS_PROFILE=true "${QUARTO_DENO}" ${QUARTO_ACTION} ${QUARTO_DENO_OPTIONS} ${QUARTO_DENO_EXTRA_OPTIONS} "${QUARTO_IMPORT_ARGMAP}" "${QUARTO_TARGET}" "$@"
+else
+  "${QUARTO_DENO}" ${QUARTO_ACTION} ${QUARTO_DENO_OPTIONS} ${QUARTO_DENO_EXTRA_OPTIONS} ${QUARTO_IMPORT_ARGMAP} "${QUARTO_TARGET}" "$@" 
+fi

src/command/render/render-contexts.ts

@@ -207,9 +207,15 @@ export async function renderContexts(
   options: RenderOptions,
   forExecute: boolean,
   project?: ProjectContext,
+  cloneOptions: boolean = true,
 ): Promise<Record<string, RenderContext>> {
-  // clone options (b/c we will modify them)
-  options = ld.cloneDeep(options) as RenderOptions;
+  if (cloneOptions) {
+    // clone options (b/c we will modify them)
+    // we make it optional because some of the callers have
+    // actually just cloned it themselves and don't need to preserve
+    // the original
+    options = ld.cloneDeep(options) as RenderOptions;
+  }
 
   const { engine, target } = await fileExecutionEngineAndTarget(
     file.path,

src/command/render/render-files.ts

@@ -307,6 +307,7 @@ export async function renderFiles(
           options,
           true,
           project,
+          false,
         );
       } catch (e) {
         // bad YAML can cause failure before validation. We

src/core/cache.ts

@@ -0,0 +1,29 @@
+/*
+ * cache.ts
+ *
+ * provides a simple cache for expensive operations
+ *
+ * Copyright (C) 2023 Posit Software, PBC
+ */
+
+export function cache<T>(f: () => Promise<T>): () => Promise<T> {
+  let value: T | undefined = undefined;
+  return async () => {
+    if (value === undefined) {
+      value = await f();
+    }
+    return value;
+  };
+}
+
+export function cacheMap<K, V>(
+  f: (key: K) => Promise<V>,
+): (key: K) => Promise<V> {
+  const map = new Map<K, V>();
+  return async (key: K) => {
+    if (!map.has(key)) {
+      map.set(key, await f(key));
+    }
+    return map.get(key)!;
+  };
+}

src/core/jupyter/kernels.ts

@@ -1,9 +1,8 @@
 /*
-* kerenels.ts
-*
-* Copyright (C) 2020-2022 Posit Software, PBC
-*
-*/
+ * kernels.ts
+ *
+ * Copyright (C) 2020-2022 Posit Software, PBC
+ */
 
 import { basename, join } from "path/mod.ts";
 import { existsSync, walkSync } from "fs/mod.ts";
@@ -43,8 +42,19 @@ export async function jupyterKernelspec(
   return kernelspecs.get(name);
 }
 
+let kJupyterKernelspecs: Map<string, JupyterKernelspec> | undefined = undefined;
 export async function jupyterKernelspecs(): Promise<
   Map<string, JupyterKernelspec>
+> {
+  if (kJupyterKernelspecs) {
+    return kJupyterKernelspecs;
+  }
+  kJupyterKernelspecs = await computeJupyterKernelspecs();
+  return kJupyterKernelspecs;
+}
+
+async function computeJupyterKernelspecs(): Promise<
+  Map<string, JupyterKernelspec>
 > {
   try {
     const result = await execProcess(

src/core/language.ts

@@ -1,9 +1,8 @@
 /*
-* language.ts
-*
-* Copyright (C) 2020-2022 Posit Software, PBC
-*
-*/
+ * language.ts
+ *
+ * Copyright (C) 2020-2022 Posit Software, PBC
+ */
 
 import { existsSync, expandGlobSync } from "fs/mod.ts";
 import { extname, join } from "path/mod.ts";
@@ -20,31 +19,50 @@ import { mergeConfigs } from "./config.ts";
 import { readAndValidateYamlFromFile } from "./schema/validated-yaml.ts";
 import { RenderFlags } from "../command/render/types.ts";
 import { getSchemaDefinition } from "./lib/yaml-validation/schema.ts";
+import { cacheMap } from "./cache.ts";
 
-export async function readLanguageTranslations(
-  translationFile: string,
-  lang?: string,
-): Promise<{ language: FormatLanguage; files: string[] }> {
-  // read and parse yaml if it exists (track files read)
-  const files: string[] = [];
-  const maybeReadYaml = async (file: string) => {
+type TranslationCacheValue = {
+  file?: string;
+  result: FormatLanguage;
+};
+
+const translationCache = cacheMap(
+  async (file: string) => {
     if (existsSync(file)) {
-      files.push(normalizePath(file));
       const errMsg = "Validation of format language object failed.";
       const formatLanguageSchema = getSchemaDefinition("format-language");
       const result = await readAndValidateYamlFromFile(
         file,
         formatLanguageSchema,
         errMsg,
       );
-      return result as FormatLanguage;
+      return {
+        file: normalizePath(file),
+        language: result as FormatLanguage,
+      };
     } else {
-      return {} as FormatLanguage;
+      return {
+        language: {} as FormatLanguage,
+      };
     }
-  };
+  },
+);
+
+export async function readLanguageTranslations(
+  translationFile: string,
+  lang?: string,
+): Promise<{ language: FormatLanguage; files: string[] }> {
+  // read and parse yaml if it exists (track files read)
+  const files: string[] = [];
 
   // read the original file
-  const language = await maybeReadYaml(translationFile);
+  const {
+    file,
+    language,
+  } = await translationCache(translationFile);
+  if (file) {
+    files.push(file);
+  }
 
   // determine additional variations to read
   const ext = extname(translationFile);
@@ -78,9 +96,16 @@ export async function readLanguageTranslations(
 
   // read the variations
   for (const variation of variations) {
-    const translations = await maybeReadYaml(
-      join(dir, stem + "-" + variation + ext),
-    );
+    const {
+      file: variationFile,
+      language: translations,
+    } = await translationCache(join(dir, stem + "-" + variation + ext));
+    if (variationFile) {
+      files.push(variationFile);
+    }
+    // const translations = await maybeReadYaml(
+    //   join(dir, stem + "-" + variation + ext),
+    // );
     Object.keys(translations).forEach((key) => {
       // top level entries use the variation key
       if (kLanguageDefaultsKeys.includes(key)) {

src/core/lib/yaml-intelligence/annotated-yaml.ts

@@ -1,9 +1,8 @@
 /*
-* annotated-yaml.ts
-*
-* Copyright (C) 2021-2022 Posit Software, PBC
-*
-*/
+ * annotated-yaml.ts
+ *
+ * Copyright (C) 2021-2022 Posit Software, PBC
+ */
 
 import { lineColToIndex } from "../text.ts";
 import { AnnotatedParse, JSONValue } from "../yaml-schema/types.ts";
@@ -48,10 +47,8 @@ function postProcessAnnotation(parse: AnnotatedParse): AnnotatedParse {
   ) {
     return postProcessAnnotation(parse.components[0]);
   } else {
-    return {
-      ...parse,
-      components: parse.components.map(postProcessAnnotation),
-    };
+    parse.components = parse.components.map(postProcessAnnotation);
+    return parse;
   }
 }
 

src/core/performance/metrics.ts

@@ -0,0 +1,53 @@
+/*
+ * metrics.ts
+ *
+ * Copyright (C) 2020-2023 Posit Software, PBC
+ */
+
+import { inputTargetIndexCacheMetrics } from "../../project/project-index.ts";
+
+export function quartoPerformanceMetrics() {
+  return {
+    inputTargetIndexCache: inputTargetIndexCacheMetrics,
+  };
+}
+
+export function reportPeformanceMetrics() {
+  console.log("---");
+  console.log("Performance metrics");
+  console.log("Quarto:");
+  console.log(JSON.stringify(quartoPerformanceMetrics(), null, 2));
+  console.log();
+  // denoMetrics is some kind of fancy object that doesn't respond
+  // to a bunch of the normal methods.  So we have to do this
+  // the JSON-round-trip way.
+  console.log("Deno:");
+  const denoMetrics = JSON.parse(JSON.stringify(Deno.metrics() as any));
+  denoMetrics.ops = Object.fromEntries(
+    Object.entries(denoMetrics.ops).map(
+      ([key, opMetrics]: any) => {
+        for (const key of Object.keys(opMetrics)) {
+          if (opMetrics[key] === 0) {
+            delete opMetrics[key];
+          }
+        }
+        return [key, opMetrics];
+      },
+    ).filter(([_key, opMetrics]: any) => Object.keys(opMetrics).length > 0)
+      .map(([key, opMetrics]: any) => {
+        if (
+          (opMetrics.opsDispatched === opMetrics.opsDispatchedSync &&
+            opMetrics.opsDispatched === opMetrics.opsCompleted &&
+            opMetrics.opsDispatched === opMetrics.opsCompletedSync) ||
+          (opMetrics.opsDispatched === opMetrics.opsDispatchedAsync &&
+            opMetrics.opsDispatched === opMetrics.opsCompleted &&
+            opMetrics.opsDispatched === opMetrics.opsCompletedAsync)
+        ) {
+          return [key, opMetrics.opsDispatched];
+        } else {
+          return [key, opMetrics];
+        }
+      }),
+  );
+  console.log(JSON.stringify(denoMetrics, null, 2));
+}

src/project/project-index.ts

@@ -179,20 +179,48 @@ export function inputTargetIsEmpty(index: InputTargetIndex) {
   return true;
 }
 
+const inputTargetIndexCache = new Map<string, InputTargetIndex>();
+export const inputTargetIndexCacheMetrics = {
+  hits: 0,
+  misses: 0,
+  invalidations: 0,
+};
+
 function readInputTargetIndexIfStillCurrent(projectDir: string, input: string) {
   const inputFile = join(projectDir, input);
   const indexFile = inputTargetIndexFile(projectDir, input);
-  if (existsSync(indexFile)) {
+  try {
     const inputMod = Deno.statSync(inputFile).mtime;
     const indexMod = Deno.statSync(indexFile).mtime;
     if (
-      inputMod && indexMod && (indexMod >= inputMod)
+      inputMod && indexMod
     ) {
-      try {
-        return JSON.parse(Deno.readTextFileSync(indexFile)) as InputTargetIndex;
-      } catch {
-        return undefined;
+      if (inputMod > indexMod) {
+        inputTargetIndexCacheMetrics.invalidations++;
+        inputTargetIndexCache.delete(indexFile);
       }
+
+      if (inputTargetIndexCache.has(indexFile)) {
+        inputTargetIndexCacheMetrics.hits++;
+        return inputTargetIndexCache.get(indexFile);
+      } else {
+        inputTargetIndexCacheMetrics.misses++;
+        try {
+          const result = JSON.parse(
+            Deno.readTextFileSync(indexFile),
+          ) as InputTargetIndex;
+          inputTargetIndexCache.set(indexFile, result);
+          return result;
+        } catch {
+          return undefined;
+        }
+      }
+    }
+  } catch (e) {
+    if (e instanceof Deno.errors.NotFound) {
+      return undefined;
+    } else {
+      throw e;
     }
   }
 }

src/quarto.ts

@@ -158,14 +158,12 @@ if (import.meta.main) {
 
     await cleanupLogger();
 
-    // print out metrics
-    // reportPeformanceMetrics();
-    // console.log(
-    //   "Done running, sleeping for 10 seconds... Stop your profiler now!",
-    // );
-    // await Deno.run({
-    //   cmd: ["sleep", "10"],
-    // }).status();
+    // if profiling, wait for 10 seconds before quitting
+    if (Deno.env.get("QUARTO_TS_PROFILE") !== undefined) {
+      console.log("Program finished. Turn off the Chrome profiler now!");
+      console.log("Waiting for 10 seconds ...");
+      await new Promise((resolve) => setTimeout(resolve, 10000));
+    }
 
     // exit
     exitWithCleanup(0);

src/resources/extensions/quarto/kbd/kbd.lua

@@ -33,7 +33,7 @@ return {
       return pandoc.RawInline('html', '<kbd ' .. kwargs_str .. '>' .. default_arg_str .. '</kbd>')
     elseif quarto.doc.isFormat("asciidoc") and args and #args == 1 then
       -- get the 'first' kbd shortcut as we can only produce on shortcut in asciidoc
-      local shortcutText = pandoc.utils.stringify(args[1][1]):gsub('-', '+')
+      local shortcutText = pandoc.utils.stringify(args[1]):gsub('-', '+')
       return pandoc.RawInline("asciidoc", "kbd:[" .. shortcutText .. "]")
     else
       -- example shortcodes

src/resources/extensions/quarto/video/video.lua

@@ -288,6 +288,7 @@ return {
     end
 
     if isEmpty(srcValue) then
+
       if #raw_args > 0 then
         srcValue = pandoc.utils.stringify(raw_args[1])
       else
@@ -298,7 +299,7 @@ return {
 
     if quarto.doc.is_format("html:js") then
       return htmlVideo(srcValue, heightValue, widthValue, titleValue, startValue, aspectRatio)
-    elseif quarto.doc.isFormat("asciidoc") then
+    elseif quarto.doc.is_format("asciidoc") then
       return asciidocVideo(srcValue, heightValue, widthValue, titleValue, startValue, aspectRatio)
     else
       -- Fall-back to a link of the source