New CLI and bootdata system (#143)

* shell: fix assignment error

* boot: create new loader file generation from boot dir in new CLI

* wanix: make loader OS agnostic

`filepath` package uses different path separators depending on the target OS.
On Windows this causes errors since embedFS expects Unix path separators.

We should really be using the `path` package everywhere unless specifically
interacting with the host OS.

* rename references to old initfs to bootfs

initfs, now bootfs, only contains files necessary to boot the kernel.
What is now called initfs will contain userspace files to setup the dev environment.
Think of it like GNU CoreUtils, preinstalled on most Unix OSes but
unnecessary to actually boot.

* boot: fetch `wanix-initfs.gz` and unpack into `/sys/*` on first boot

This commit makes some changes to the structure of `boot/initfs/`
and how initfs files are copied. Instead of hardcoding which files go
where, `fs.go` just unpacks the contents of initfs into `/sys/*`.

It's basically running the equivalent of:
```sh
cd /sys/ && \
get wanix-initfs.gz && \
gzip -d wanix-initfs.gz && \
tar -xf wanix-initfs
```

This gives us direct and simple control of file placement.
And if in the future we want to put things outside of `/sys`,
we can just unpack to `/` and update the build paths. Though
it makes sense to me that initfs would only include system related files.

---------

Co-authored-by: Parzival-3141 <29632054+Parzival-3141@users.noreply.github.com>

.gitignore

@@ -4,3 +4,9 @@
 external/jazz/node_modules
 /local
 /build/pkg.zip
+/boot/kernel.gz
+/boot/initfs.gz
+/boot/initfs
+/wanix-bootloader.js
+/wanix-initfs.gz
+/wanix-kernel.gz

Makefile

@@ -1,36 +1,56 @@
-.PHONY: boot dev kernel shell dev bundle micro hugo local/bin/kernel
+.PHONY: boot dev kernel shell dev bundle micro hugo wanix boot/kernel.gz initfsDirs
 
 VERSION=0.2dev
 DEBUG?=false
 
-all: kernel shell build micro
+all: wanix kernel shell build micro
 
 dev: all
-	go run ./dev
-
-bundle: local/bin/kernel
-	cp local/bin/kernel local/wanix-kernel
-	gzip -f -9 local/wanix-kernel
-	go run -tags bundle ./dev
-
-kernel: local/bin/kernel
-local/bin/kernel: kernel/bin/shell kernel/bin/micro kernel/bin/build
-	cd kernel && GOOS=js GOARCH=wasm go build -ldflags="-X 'main.Version=${VERSION}' -X 'tractor.dev/wanix/kernel/fs.DebugLog=${DEBUG}'" -o ../local/bin/kernel .
-
-shell: kernel/bin/shell
-kernel/bin/shell: shell/main.go
-	cd shell && GOOS=js GOARCH=wasm go build -o ../kernel/bin/shell .
-
-micro: kernel/bin/micro
-kernel/bin/micro: external/micro/
+	./local/bin/wanix dev
+
+loader: all
+	cd ./local && ./bin/wanix loader
+
+wanix: local/bin/wanix
+local/bin/wanix: kernel initfs
+	go build -o ./local/bin/ ./cmd/wanix
+
+kernel: boot/kernel.gz
+boot/kernel.gz: 
+	cd kernel && GOOS=js GOARCH=wasm go build -ldflags="-X 'main.Version=${VERSION}' -X 'tractor.dev/wanix/kernel/fs.DebugLog=${DEBUG}'" -o ../boot/kernel .
+	gzip -f -9 ./boot/kernel
+
+initfs: boot/initfs.gz
+boot/initfs.gz: boot/initfs
+	tar -cf ./boot/initfs.tar  -C ./boot/initfs .
+	gzip -f -9 ./boot/initfs.tar
+	mv ./boot/initfs.tar.gz ./boot/initfs.gz
+
+boot/initfs: initfsDirs shell micro build
+	cp -r ./shell ./boot/initfs/cmd/
+	cp internal/export/exportapp.sh ./boot/initfs/cmd/
+	cp internal/export/main.go ./boot/initfs/export/
+
+initfsDirs:
+	mkdir -p ./boot/initfs
+	mkdir -p ./boot/initfs/bin
+	mkdir -p ./boot/initfs/cmd
+	mkdir -p ./boot/initfs/export
+
+shell: boot/initfs/bin/shell.wasm
+boot/initfs/bin/shell.wasm: shell/main.go
+	cd shell && GOOS=js GOARCH=wasm go build -o ../boot/initfs/bin/shell.wasm .
+
+micro: boot/initfs/cmd/micro.wasm
+boot/initfs/cmd/micro.wasm: external/micro/
 	make -C external/micro build
 
-hugo: external/hugo/
-	make -C external/hugo build
-
 build/pkg.zip: build/build-pkgs/imports/imports.go build/build-pkgs/main.go
 	cd build && go run ./build-pkgs/main.go ./build-pkgs/imports ./pkg.zip
 
-build: kernel/bin/build
-kernel/bin/build: build/main.go build/pkg.zip
-	cd build && GOOS=js GOARCH=wasm go build -o ../kernel/bin/build .
+build: boot/initfs/cmd/build.wasm
+boot/initfs/cmd/build.wasm: build/main.go build/pkg.zip
+	cd build && GOOS=js GOARCH=wasm go build -o ../boot/initfs/cmd/build.wasm .
+
+hugo: external/hugo/
+	make -C external/hugo build

dev/initdata.tmpl → boot/bootloader.tmpl

@@ -1,5 +1,6 @@
-globalThis.initdata = {
-{{range .}}
+{{.Loader}}
+globalThis.bootdata = {
+{{range .Files }}
   "{{.Name}}": {type: "{{.Type}}", mtimeMs: {{.Mtime}}, data: "{{.Data}}"},
 {{end}}
 }

boot/embed.go

@@ -0,0 +1,6 @@
+package boot
+
+import "embed"
+
+//go:embed *
+var Dir embed.FS

dev/bootloader.js → boot/loader.js

@@ -2,22 +2,9 @@ if (!globalThis["ServiceWorkerGlobalScope"]) {
   const basePath = window.location.pathname.replace("index.html", "");
 
   // registers Service Worker using this file (see bottom) if none is registered,
-  // and sets up a mechanism to fullfill requests from initfs or kernel
+  // and sets up a mechanism to fullfill requests from bootfs or kernel
   async function setupServiceWorker() {
     const timeout = (ms) => new Promise((resolve, reject) => setTimeout(() => reject(new Error("Timeout")), ms));
-    const unzip = async (b64data) => {
-      const gzipData = atob(b64data);
-      const gzipBuf = new Uint8Array(gzipData.length);
-      for (let i = 0; i < gzipData.length; i++) {
-        gzipBuf[i] = gzipData.charCodeAt(i);
-      }
-      const gzipBlob = new Blob([gzipBuf], { type: 'application/gzip' });
-      const ds = new DecompressionStream('gzip');
-      const out = gzipBlob.stream().pipeThrough(ds);
-      const response = new Response(out);
-      const buf = await response.arrayBuffer();
-      return new Uint8Array(buf);
-    }
 
     let registration = await navigator.serviceWorker.getRegistration();
     if (!registration) {
@@ -54,16 +41,6 @@ if (!globalThis["ServiceWorkerGlobalScope"]) {
         return;
       }
 
-      // handle requests for compressed embedded initfs files if present
-      if (globalThis.initdata && req.path.startsWith(`${basePath}~init/`)) {
-        const f = globalThis.initdata[req.path.replace(`${basePath}~init/`, "")];
-        if (f) {
-          const data = await unzip(f.data);
-          registration.active.postMessage({response: { reqId: req.id, body: data, headers: {"content-type": f.type}}});
-          return;
-        }
-      }
-
       if (!globalThis.sys) {
         registration.active.postMessage({response: { reqId: req.id,  error: `kernel not loaded yet for ${req.path}` }});
         return;
@@ -92,44 +69,54 @@ if (!globalThis["ServiceWorkerGlobalScope"]) {
     const ds = new DecompressionStream('gzip');
     const out = gzipBlob.stream().pipeThrough(ds);
     const response = new Response(out);
-    globalThis.initfs["kernel"] = { mtimeMs: Date.now(), blob: await response.blob() };
+    globalThis.bootfs["kernel"] = { mtimeMs: Date.now(), blob: await response.blob() };
+  }
+
+  const unzipb64 = async (b64data) => {
+    const gzipData = atob(b64data);
+    const gzipBuf = new Uint8Array(gzipData.length);
+    for (let i = 0; i < gzipData.length; i++) {
+      gzipBuf[i] = gzipData.charCodeAt(i);
+    }
+    const gzipBlob = new Blob([gzipBuf], { type: 'application/gzip' });
+    const ds = new DecompressionStream('gzip');
+    const out = gzipBlob.stream().pipeThrough(ds);
+    const response = new Response(out);
+    const buf = await response.arrayBuffer();
+    return new Uint8Array(buf);
   }
 
   // bootloader starts here
   globalThis.bootWanix = (async function() {
     console.log("Wanix booting...")
 
-    globalThis.initfs = {};
+    globalThis.bootfs = {};
     const kernelReady = fetchKernel();
     await setupServiceWorker();
 
-    const load = async (name, file) => {
-      // Determine if file contains a path to fetch or embedded file contents to load.
-      if(file.type === "text/plain") {
-        globalThis.initfs[name] = { mtimeMs: file.mtimeMs, blob: await (await fetch(`./sys/dev/${file.data}`)).blob() };
-      } else {
-        globalThis.initfs[name] = { mtimeMs: file.mtimeMs, blob: await (await fetch(`./~init/${name}`)).blob() };
-      }
-    };
-
     // allow loading concurrently
     let loads = [];
-    for(const property in globalThis.initdata) {
-      loads.push(load(property, globalThis.initdata[property]));
+    for(const filename in globalThis.bootdata) {
+      loads.push((async () => {
+        console.log(`Loading ${filename}...`)
+        const file = globalThis.bootdata[filename];
+        const data = await unzipb64(file.data);
+        globalThis.bootfs[filename] = { mtimeMs: file.mtimeMs, blob: new Blob([data], { type: file.type }) };
+      })());
     }
     await Promise.all(loads);
 
-    globalThis.duplex = await import(URL.createObjectURL(initfs["duplex.js"].blob));
-    globalThis.task = await import(URL.createObjectURL(initfs["task.js"].blob));
+    globalThis.duplex = await import(URL.createObjectURL(bootfs["duplex.js"].blob));
+    globalThis.task = await import(URL.createObjectURL(bootfs["task.js"].blob));
 
     await kernelReady;
-    globalThis.sys = new task.Task(initfs);
+    globalThis.sys = new task.Task(bootfs);
 
     console.log("Starting kernel...")
     await sys.exec("kernel");
 
     // load host API
-    await import(URL.createObjectURL(initfs["host.js"].blob));
+    await import(URL.createObjectURL(bootfs["host.js"].blob));
   });
 }
 
@@ -164,6 +151,7 @@ if (globalThis["ServiceWorkerGlobalScope"] && self instanceof ServiceWorkerGloba
       url.pathname.startsWith(`${basePath}index.html`) ||
       url.pathname.startsWith(`${basePath}loading.gif`) ||
       url.pathname.startsWith(`${basePath}wanix-kernel.gz`) ||
+      url.pathname.startsWith(`${basePath}wanix-initfs.gz`) ||
       url.pathname.startsWith("/auth") ||
       url.hostname !== location.hostname ||
       !host) return;