feat!: use markdown for projects assets (#89)

Switches projects assets to publish markdown docs instead of compiled
web assets.

Related: dfinity/ci-tools#52

Author: ilbertt

.github/CONTRIBUTING.md

@@ -133,6 +133,9 @@ To serve the root docs project, run:
 deno task docs:start
 ```
 
+> Note: You need to run `deno task docs:prebuild` before to ensure the projects
+> files are added to the docs.
+
 ### Build root docs project
 
 To build the root docs project, run:
@@ -141,6 +144,16 @@ To build the root docs project, run:
 deno task docs:build
 ```
 
+### Update projects schema
+
+If you make changes to the [`projects-schema.json`](../projects-schema.json)
+file, you need to update the [`projects-schema.d.ts`](../projects-schema.d.ts)
+file. To do this, run:
+
+```shell
+deno task types:projects
+```
+
 ### Serve all docs using Juno
 
 To start the Juno emulator, run:

.gitignore

@@ -9,3 +9,13 @@ node_modules/
 
 # MacOS
 .DS_Store
+
+# Ignore all unzipped content from projects
+docs/src/content/docs/**
+# Track any file you want to keep manually here
+!docs/src/content/docs/index.mdx
+
+# Ignore all unzipped html pages from projects
+docs/src/pages/**
+# Track any file you want to keep manually here
+# e.g. !docs/src/pages/index.astro

README.md

@@ -11,6 +11,8 @@ This repository contains the documentation for the ICP JavaScript SDK hosted on
    ```json
    [
      {
+       "title": "Core",
+       "description": "Base library for Internet Computer apps.",
        "repository": "dfinity/icp-js-core",
        "subdirectory": "core"
      },
@@ -23,7 +25,7 @@ This repository contains the documentation for the ICP JavaScript SDK hosted on
    ```
 
 2. Add a link and short description for the sub-project in the
-   [`index.astro`](./docs/src/pages/index.astro) file:
+   [`index.mdx`](./docs/src/content/docs/index.mdx) file:
 
    ```html
    <CardGrid stagger>

deno.json

@@ -1,15 +1,28 @@
 {
   "tasks": {
-    "predeploy": "deno run --no-prompt --allow-read --allow-write=\"./dist\" scripts/predeploy.ts",
     "pull-project-docs": "deno run --no-prompt --allow-read --allow-net=\"github.com,codeload.github.com\" --allow-write=\"./public,$TMPDIR\" scripts/pull-project-docs.ts",
     "fetch-project-commit": "deno run --no-prompt --allow-read --allow-env --allow-net=\"github.com,api.github.com\" --allow-write=\"$TMPDIR\" scripts/fetch-project-commit.ts",
-    "docs:build": "astro build --root ./docs",
+    "docs:prebuild": "deno run --no-prompt --allow-read --allow-write=\"./docs/src/content/docs,./docs/src/pages\" scripts/prebuild.ts",
+    "docs:build": "deno task docs:prebuild && astro build --root ./docs",
+    "docs:preview": "astro preview --root ./docs",
     "docs:start": "astro dev --root ./docs",
+    "types:projects": "deno run --no-prompt --allow-read --allow-write=\"projects-schema.d.ts\" --allow-env --allow-sys=\"cpus\" json-schema-to-typescript --input projects-schema.json --output projects-schema.d.ts",
     "juno": "juno"
   },
   "imports": {
+    "@astrojs/react": "npm:@astrojs/react@^4.3.0",
+    "@pagefind/default-ui": "npm:@pagefind/default-ui@^1.4.0",
+    "@radix-ui/react-navigation-menu": "npm:@radix-ui/react-navigation-menu@^1.2.14",
+    "@types/react": "npm:@types/react@^19.1.12",
+    "astro-integration-kit": "npm:astro-integration-kit@^0.19.0",
     "fs": "node:fs",
+    "json-schema-to-typescript": "npm:json-schema-to-typescript@^15.0.4",
+    "pagefind": "npm:pagefind@^1.4.0",
+    "react": "npm:react@^19.1.1",
+    "react-dom": "npm:react-dom@^19.1.1",
+    "unist-util-visit": "npm:unist-util-visit@^5.0.0",
     "url": "node:url",
+    "util": "node:util",
     "path": "node:path",
     "@actions/core": "npm:@actions/core@^1.11.1",
     "@actions/github": "npm:@actions/github@^6.0.1",

deno.lock

@@ -11,7 +11,7 @@
   "canisters": {
     "docs": {
       "type": "assets",
-      "build": ["deno task docs:build", "deno task predeploy"],
+      "build": ["deno task docs:build"],
       "source": ["dist"],
       "frontend": {}
     }

dfx.json

@@ -1,30 +1,54 @@
 // @ts-check
 import { defineConfig, passthroughImageService } from "astro/config";
+import react from "@astrojs/react";
 import starlight from "@astrojs/starlight";
+import { multiSidebarPlugin } from "./plugins/multi-sidebar/index.ts";
+import { markdownUrlsPlugin } from "./plugins/markdown-urls/index.ts";
+import { getProjectsConfig, getSidebarsFromProjects } from "./projects.ts";
+
+const projectsConfig = getProjectsConfig();
 
 // https://astro.build/config
 export default defineConfig({
   site: "https://js.icp.build/",
   image: {
     service: passthroughImageService(),
   },
+  output: "static",
+  vite: {
+    server: {
+      fs: {
+        strict: false,
+      },
+    },
+    ssr: {
+      // See https://docs.astro.build/en/guides/troubleshooting/#adding-dependencies-to-astro-in-a-monorepo
+      noExternal: ["@astrojs/react"],
+    },
+  },
   integrations: [
+    react(),
     starlight({
-      title: "ICP JavaScript SDK Docs",
+      title: "ICP JS SDK Docs",
       logo: {
         src: "./src/assets/icp.svg",
         alt: "Internet Computer Logo",
-        replacesTitle: true,
       },
+      favicon: "/favicon.png",
       customCss: [
         "./src/assets/layers.css",
         "./src/assets/theme.css",
         "./src/assets/overrides.css",
         "./src/assets/elements.css",
       ],
       pagefind: false,
-      tableOfContents: false,
       sidebar: [],
+      plugins: [
+        multiSidebarPlugin({
+          sidebars: getSidebarsFromProjects(projectsConfig),
+        }),
+        markdownUrlsPlugin(),
+      ],
     }),
   ],
 });

docs/astro.config.mjs

@@ -0,0 +1,13 @@
+import { type StarlightPlugin } from "@astrojs/starlight/types";
+import { markdownUrlsIntegration } from "./integration.ts";
+
+export function markdownUrlsPlugin(): StarlightPlugin {
+  return {
+    name: "starlight-markdown-urls-plugin",
+    hooks: {
+      "config:setup": (ctx) => {
+        ctx.addIntegration(markdownUrlsIntegration());
+      },
+    },
+  };
+}

docs/plugins/markdown-urls/index.ts

@@ -0,0 +1,76 @@
+import { defineIntegration } from "astro-integration-kit";
+import { type AstroIntegrationLogger, type RemarkPlugins } from "astro";
+import { visit } from "unist-util-visit";
+import path from "path";
+
+const currentDir = import.meta.dirname!;
+const docsDir = path.join(currentDir, "../../src/content/docs");
+
+type RemarkPlugin = RemarkPlugins[number];
+
+const markdownUrlsRemarkPlugin: RemarkPlugin = ([logger, site]: [
+  AstroIntegrationLogger,
+  string,
+]) =>
+(tree, file) => {
+  const currentFileDir = path.dirname(file.path);
+
+  visit(tree, "link", (node) => {
+    const url = node.url;
+
+    // take full URLs to the current site and make them relative
+    if (url.startsWith(site)) {
+      node.url = new URL(url).pathname;
+      return;
+    }
+
+    // skip any other full URLs
+    if (
+      url.startsWith("https://") ||
+      url.startsWith("/") ||
+      url.startsWith("http://") ||
+      url.startsWith("mailto:") ||
+      url.startsWith("#")
+    ) {
+      logger.debug(`Skipping URL: ${url}`);
+      return;
+    }
+
+    const resolvedUrl = path.resolve(currentFileDir, url);
+    const relativeToDocs = path.relative(docsDir, resolvedUrl);
+    const nodeUrl = `/${
+      relativeToDocs
+        .replace(/(index)?\.mdx?(#.*)?$/, "$2")
+        .toLowerCase()
+    }`;
+    logger.debug(`Normalizing URL: ${url} -> ${nodeUrl}`);
+
+    node.url = nodeUrl;
+  });
+};
+
+export const markdownUrlsIntegration = defineIntegration({
+  name: "starlight-markdown-urls-integration",
+  setup: () => {
+    return {
+      hooks: {
+        "astro:config:setup": ({ updateConfig, config, logger }) => {
+          updateConfig({
+            markdown: {
+              remarkPlugins: [
+                ...config.markdown.remarkPlugins,
+                [
+                  markdownUrlsRemarkPlugin,
+                  [
+                    logger,
+                    config.site,
+                  ],
+                ],
+              ],
+            },
+          });
+        },
+      },
+    };
+  },
+});

docs/plugins/markdown-urls/integration.ts

@@ -0,0 +1,3 @@
+throw new Error(
+  "There are no exports in the root module of this package. Import from submodules instead. Use intellisense or the docs to find the available submodules.",
+);