Devdocs: offload Registry on to CAS

.github/workflows/main.yml

@@ -61,23 +61,22 @@ jobs:
         with:
           ssh-private-key: ${{ secrets.NEXTJOURNAL_CI_SSH_KEY }}
 
-      - name: Build Notebooks
-        run: clojure -X:nextjournal/devdocs
-
-      - name: Compile ClojureScript (snapshot devcards/stories build)
+      - name: Compile ClojureScript (snapshot build)
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: bb release:cljs
 
       - name: Compile CSS
         run: bb build:css
 
-      - name: Build static devcards
-        run: ./bin/build_static_snapshot
+      - name: Build Notebooks
+        run: bb build:devdocs
+
+      - name: Copy Asseets
+        run: bb build:copy-assets
 
-      - name: Copy build to bucket under SHA
-        run: |
-          gsutil cp -r build gs://nextjournal-snapshots/viewers/build/${{ github.sha }}
+      - name: Store in CAS
+        run: echo "cas_url=$(bb devdocs:cas)" >> $GITHUB_ENV
 
       - name: Add status flag to the devcards build
         uses: Sibz/github-status-action@v1
@@ -87,4 +86,4 @@ jobs:
           description: 'Ready'
           state: 'success'
           sha: ${{github.event.pull_request.head.sha || github.sha}}
-          target_url: https://storage.googleapis.com/nextjournal-snapshots/viewers/build/${{ github.sha }}/index.html
+          target_url: "${{ env.cas_url }}"

.gitignore

@@ -23,3 +23,4 @@ node_modules
 # VSCode/Calva
 .calva/
 /build/
+public/*.edn

bb.edn

@@ -1,6 +1,8 @@
-{:tasks
- {:init (def tailwind-build-cmd "npx tailwindcss --input resources/css/viewer.css --output public/css/viewer.css")
-
+{:deps {io.github.nextjournal/cas-client {:git/sha "e34901f4a29d54954544313ae1a0bb859fe3931f"}}
+ :tasks
+ {:init (do (def tailwind-build-cmd "npx tailwindcss --input resources/css/viewer.css --output public/css/viewer.css")
+            (def devdocs-build-path "build/devdocs"))
+  :requires ([babashka.fs :as fs] [nextjournal.cas-client :as cas])
   yarn-install {:doc "Installs and updates npm dependencies"
                 :task (shell "yarn install")}
 
@@ -16,7 +18,23 @@
              :task (shell tailwind-build-cmd)
              :depends [yarn-install]}
 
-  -dev {:depends [watch:cljs watch:css]}
+  build:copy-assets {:doc "Moves assets into devdocs out path prior to uploading them"
+                     :task (do
+                             (fs/create-dirs (str devdocs-build-path "/js"))
+                             (fs/create-dirs (str devdocs-build-path "/css"))
+                             (fs/copy "public/index.html" (str devdocs-build-path "/index.html") {:replace-existing true})
+                             (fs/copy "public/js/viewer.js" (str devdocs-build-path "/js/") {:replace-existing true})
+                             (fs/copy "public/css/viewer.css" (str devdocs-build-path "/css/") {:replace-existing true}))}
+
+  build:devdocs {:doc "Build Devdocs" :task (clojure "-X:nextjournal/devdocs")}
+
+  build:devdocs:dev {:doc "Build Devdocs to be served by shadow dev http server"
+                     :task (clojure "-X:nextjournal/devdocs :out-path '\"public\"'")}
+
+  devdocs:cas {:doc "Store devdocs into CAS"
+               :task (print (get-in (cas/put {:path devdocs-build-path}) ["manifest" "index.html"]))}
+
+  -dev {:depends [watch:cljs watch:css build:devdocs:dev]}
   dev {:doc "Start app in dev mode, compiles cljs and css, starts an nrepl "
        :task (run '-dev {:parallel true})}
 

examples/examples.cljs

@@ -2,6 +2,7 @@
   (:require ["react-dom/client" :as react-client]
             [clojure.string :as str]
             [nextjournal.devcards.routes :as devcards.routes]
+            [nextjournal.devdocs :as devdocs]
             [nextjournal.devdocs.demo :as devdocs.demo]
             [nextjournal.devcards-ui :as devcards-ui]
             [nextjournal.clerk-sci-env] ;; sets up clerk sci env extensions
@@ -83,6 +84,8 @@
     (react-client/createRoot el)))
 
 (defn ^:export ^:dev/after-load init []
-  (re-frame/dispatch-sync [:init-commands (commands.state/empty-db!)])
-  (rfe/start! router #(reset! match %1) {:use-fragment true})
-  (.render react-root (reagent/as-element [view])))
+  (.then (devdocs/init!)
+         (fn [_]
+           (re-frame/dispatch-sync [:init-commands (commands.state/empty-db!)])
+           (rfe/start! router #(reset! match %1) {:use-fragment true})
+           (.render react-root (reagent/as-element [view])))))

examples/nextjournal/devdocs/demo.cljs

@@ -4,13 +4,6 @@
             [reitit.frontend :as rf])
   (:require-macros [nextjournal.devdocs :as devdocs]))
 
-(reset! devdocs/registry (assoc (devdocs/build-registry {:paths ["README.md"
-                                                                 "docs/**.{clj,md}"]})
-                                :navbar/theme
-                                {:back "text-[12px] text-slate-300 hover:bg-white/10 font-normal px-5 py-1"
-                                 :icon "text-slate-400"}))
-
-
 (defonce router (rf/router ["/docs" ["/*path" {:name :devdocs/show}]]))
 
 (defn view [match]

modules/devdocs/deps.edn

@@ -4,7 +4,6 @@
         io.github.nextjournal/clerk.render {:git/url "https://github.com/nextjournal/clerk"
                                             :git/sha "711a1b2fae3c212d2c8c2323cf4d53c178766114"
                                             :deps/root "render"}
-        io.github.nextjournal/cas-client {:git/sha "84ab35c3321c1e51a589fddbeee058aecd055bf8"}
         re-frame/re-frame {:git/url "https://github.com/nextjournal/freerange"
                            :git/sha "8cf68c30722a4c6f8f948a134c900d7a656ecad4"}
         com.lambdaisland/deja-fu {:mvn/version "0.3.33"}}}

modules/devdocs/src/nextjournal/devdocs.clj

@@ -5,7 +5,6 @@
             [clojure.java.shell :as sh]
             [clojure.stacktrace :as stacktrace]
             [clojure.string :as str]
-            [nextjournal.cas-client :as cas]
             [nextjournal.clerk :as clerk]
             [nextjournal.clerk.builder :as builder]
             [nextjournal.clerk.eval :as eval]
@@ -15,13 +14,11 @@
 
 
 (def build-path "build/devdocs")
-(defn doc-path->edn-path [path]
-  (str build-path "/" (str/replace (str path) fs/file-separator "|") ".edn"))
+(defn doc-path->edn-path [path] (str (str/replace (str path) fs/file-separator "|") ".edn"))
 
 #_(doc-path->edn-path "docs/clerk/clerk.md")
 #_(doc-path->edn-path "docs/clerk/clerk.clj")
 #_(doc-path->edn-path "manifest")
-#_(str (fs/relativize build-path (doc-path->edn-path "docs/clerk/clerk.clj")))
 
 (defn path->title [path]
   (-> path fs/file-name (str/replace #"\.(clj(c?)|md)$" "") (str/split #"_")
@@ -56,18 +53,14 @@
 (defn guard [p? val] (when (p? val) val))
 (defn assoc-when-missing [m k v] (cond-> m (not (contains? m k)) (assoc k v)))
 
-(defn file->doc-info [{:keys [cas-manifest]} path]
+(defn file->doc-info [path]
   (-> (parser/parse-file {:doc? true} (fs/file path))
       (select-keys [:title :doc])
       (assoc-when-missing :title (path->title path))
       (assoc :path (str path)
+             :edn-url (doc-path->edn-path path)
              :last-modified (when-some [ts (-> (sh/sh "git" "log" "-1" "--format=%ct" (str path)) :out str/trim not-empty)]
-                              (* (Long/parseLong ts) 1000)))
-      (as-> info
-        (if-some [cas-url (get cas-manifest (str (fs/relativize build-path (doc-path->edn-path path))))]
-          (assoc info :edn-cas-url cas-url)
-          (do (println (str "No EDN data url found for notebook: '" path "'. Falling back to notebook viewer data with no cell results."))
-              (assoc info :edn-doc (doc-viewer-edn {:path path})))))))
+                              (* (Long/parseLong ts) 1000)))))
 
 #_(file->doc-info "docs/clerk/clerk.clj")
 #_(file->doc-info "docs/clerk/missing_title.clj")
@@ -106,53 +99,50 @@
                  "README.md"
                  "modules/devdocs/src/nextjournal/devdocs.clj"])
 
-(defmacro build-registry
+(defn build-registry
   "Populates a nested registry of parsed notebooks given a set of paths."
   [{:keys [paths]}]
-  (let [cas-manifest (when (fs/exists? (doc-path->edn-path "manifest"))
-                       (clojure.edn/read-string (slurp (doc-path->edn-path "manifest"))))]
-    (->> paths
-         expand-paths
-         (map (partial file->doc-info {:cas-manifest cas-manifest}))
-         (group-by (comp str fs/parent :path))
-         (sort-by first)
-         (reduce add-collection {}))))
-
-#_(letfn [(strip-edn [coll] (-> coll
-                                (update :devdocs (partial into [] (map #(dissoc % :edn-doc))))
-                                (update :collections (partial into [] (map strip-edn)))))]
-    (strip-edn
-     (build-registry {:paths ["docs/**/*.{clj,md}"
-                              "README.md"
-                              "modules/devdocs/src/nextjournal/devdocs.clj"]})))
-
-(defn write-edn-results [_opts docs]
+  (->> paths
+       expand-paths
+       (map file->doc-info)
+       (group-by (comp str fs/parent :path))
+       (sort-by first)
+       (reduce add-collection {})))
+
+(defn write-edn-results [{:keys [out-path]} docs]
   (doseq [{:as _doc :keys [viewer file]} docs]
-    (let [edn-path (doc-path->edn-path file)]
+    (let [edn-path (str (fs/path out-path (doc-path->edn-path file)))]
       (when-not (fs/exists? (fs/parent edn-path)) (fs/create-dirs (fs/parent edn-path)))
       (spit edn-path (viewer/->edn viewer)))))
 
 (defn build!
   "Expand paths and evals resulting notebooks with clerk. Persists EDN results to fs at conventional path (see `doc-path->cached-edn-path`)."
-  [{:as opts :keys [paths]}]
+  [{:as opts :keys [paths out-path] :navbar/keys [theme] :or {out-path build-path}}]
+  ;; build Clerk static app (in EDN format) one file per notebook
   (with-redefs [builder/write-static-app! write-edn-results]
-    (builder/build-static-app! {:paths (expand-paths paths)}))
-  ;; store in CAS
-  (let [{:as response :strs [manifest]} (cas/put (-> opts (select-keys [:cas-host]) (assoc :path build-path)))]
-    (when-not manifest
-      (throw (ex-info "Unable to upload to CAS" {:response response})))
-    (spit (doc-path->edn-path "manifest") (pr-str manifest))))
+    (builder/build-static-app! {:paths (expand-paths paths) :out-path out-path}))
+  ;; Assemble a registry respecting fs hierarchy
+  (println "🗂 Building devdocs registry...")
+  (spit (fs/file out-path "registry.edn")
+        (pr-str (cond-> (build-registry {:paths paths})
+                  theme (assoc :navbar/theme theme))))
+  (println "done."))
 
 (comment
   (shadow.cljs.devtools.api/repl :browser)
   (fs/delete-tree "build/devdocs")
   (fs/list-dir "build/devdocs")
-  (cas/put {:path "build/devdocs"})
+
   ;; get manifest
-  (clojure.edn/read-string (slurp (doc-path->edn-path "manifest")))
+  (clojure.edn/read-string (slurp (str build-path "/registry.edn")))
   (clerk/clear-cache!)
+
   ;; build devdocs for results to appear in notebooks
-  (build! {:paths ["docs/**.{clj,md}"]})
+  (build! {:paths ["docs/**.{clj,md}"]
+           ;; #_#_
+           :out-path "public"
+           :navbar/theme {:back "text-[12px] text-slate-300 hover:bg-white/10 font-normal px-5 py-1"
+                          :icon "text-slate-400"}})
 
   ;; registry
   (build-registry {:paths ["docs/**.{clj,md}"]}))

modules/devdocs/src/nextjournal/devdocs.cljs

@@ -21,6 +21,13 @@
 
 (defonce registry (reagent/atom []))
 
+(defn init!
+  ([] (init! {}))
+  ([{:keys [registry-url] :or {registry-url "registry.edn"}}]
+   (.. (js/fetch registry-url)
+       (then #(.text %))
+       (then #(reset! registry (render/read-string %))))))
+
 ;; TODO: maybe compile into reitit router
 (defn find-coll [reg path] (some #(and (str/starts-with? path (:path %)) %) (:items reg)))
 (defn find-doc [{:keys [items]} path] (some #(and (= path (:path %)) %) items))
@@ -42,11 +49,11 @@
 
 (declare collection-inner-view)
 
-(defn item-view [{:as item :keys [title edn-cas-url edn-doc path last-modified items]}]
+(defn item-view [{:as item :keys [title edn-url path last-modified items]}]
   [:div
    (cond
      ;; doc
-     (or edn-cas-url edn-doc)
+     edn-url
      [:div.mb-2
       [:a.hover:underline.font-bold
        {:href (rfe/href :devdocs/show {:path path}) :title path}
@@ -75,11 +82,8 @@
     [:h1.pt-8 "Devdocs"]
     [collection-inner-view collection]]])
 
-(defn devdoc-view [{:as doc :keys [edn-doc edn-cas-url fragment]}]
-  (let [edn (render.hooks/use-promise
-             (if edn-cas-url
-               (.then (js/fetch edn-cas-url) #(.text %))
-               (js/Promise.resolve edn-doc)))]
+(defn devdoc-view [{:as doc :keys [edn-url fragment]}]
+  (let [edn (render.hooks/use-promise (.then (js/fetch edn-url) #(.text %)))]
     [:div.overflow-y-auto.bg-white.dark:bg-gray-900.flex-auto.relative.font-sans
      (cond-> {:style {:padding-top 45 :padding-bottom 70}}
        fragment (assoc :ref #(scroll-to-fragment fragment)))
@@ -121,8 +125,8 @@
      [navbar/panel !state [navbar/navbar !state]]
      (if (or (nil? path) (contains? #{"" "/"} path))
        [collection-view @registry]
-       (let [{:as node :keys [edn-doc edn-cas-url]} (lookup @registry path)]
-         (when (or edn-cas-url edn-doc)
+       (let [{:as node :keys [edn-url]} (lookup @registry path)]
+         (when edn-url
            ^{:key path} [devdoc-view node])))]))
 
 (defn devdoc-commands