zstd-codec support for decompressing files

d.txt

@@ -0,0 +1,14 @@
+On branch nxp
+Your branch is ahead of 'nxp/master' by 2 commits.
+  (use "git push" to publish your local commits)
+
+Changes not staged for commit:
+  (use "git add <file>..." to update what will be committed)
+  (use "git restore <file>..." to discard changes in working directory)
+	modified:   webusb/package.json
+
+Untracked files:
+  (use "git add <file>..." to include in what will be committed)
+	d.txt
+
+no changes added to commit (use "git add" and/or "git commit -a")

webusb/notes.txt

@@ -0,0 +1,5 @@
+errors you keep running into: 
+
+must init git submodule for zstd
+must start the emsdk thing to use emcc in terminal
+updating node version to do serve -s build

webusb/src/App.js

@@ -1,7 +1,8 @@
 import React, { useState } from 'react';
 import Combined from "./components/Combined";
+import Decompress2 from "./components/Decompress";
 import Decompress from "./components/Decompress";
-import Decompress2 from "./components/Decompress2"
+import DecompressAlt from "./components/DecompressAlt"
 import usePopup from "./logic/usePopup";
 
 import './App.css'
@@ -15,12 +16,9 @@ const App = () => {
     <div className="App">
         <div className="u-flex u-column">
             <div>
-                Decompress (stream)
+                <Decompress2 /> (2)
                 <Decompress />
-            </div>
-            <div>
-                Decompress (callback)
-                <Decompress2 />
+                <DecompressAlt />
             </div>
             <div className="u-row">
                 <span className="link-container">bootloader [optional]: </span>
@@ -43,15 +41,17 @@ const App = () => {
             <div className="popup-button">
                 <button onClick = {showPopup}>USB_down</button>
             </div>
-
-            <Combined bootFile={bootFile} flashFile={flashFile}/>
-
+
+            {/* for testing purpose, just let you input files without waiting */}
             {/* <div className="popup-loc ">
                 <div className="popup-container" style={show?{}:{display:'none'}}>
                     <Combined bootFile={bootFile} flashFile={flashFile}/>
                     <button onClick={()=>closePopup()}>close</button>
                 </div>
             </div> */}
+
+            <Combined bootFile={bootFile} flashFile={flashFile}/>
+
             <span>{error}</span>
         </div>
 

webusb/src/components/Decompress.js

@@ -1,12 +1,13 @@
 import { useEffect, useState } from "react"
-import useDecompress from '../logic/useDecompress'
+// import useDecompress from '../logic/useDecompress'
+import useDecompress2 from '../logic/useDecompress2'
 import { DATA_SZ, BLK_SZ, PACKET_SZ } from "../helper/sparse";
 
 const Decompress = () => {
     const [flashFile, setFlashFile] = useState();
     const [{
         requestUSBDevice
-    }] = useDecompress(flashFile);
+    }] = useDecompress2(flashFile);
 
     useEffect(()=> {
         console.log(ZstdCodec)

webusb/src/components/Decompress2.js

@@ -1,14 +1,13 @@
 import { useEffect, useState } from "react"
-import useDecompress4 from '../logic/useDecompress2'
+import useDecompress2 from '../logic/useDecompress2'
 import { DATA_SZ, BLK_SZ, PACKET_SZ } from "../helper/sparse";
 
 const Decompress2 = () => {
     const [flashFile, setFlashFile] = useState();
 
     const [{
-        requestUSBDevice,
-        decompressFileToTransform
-    }] = useDecompress4(flashFile);
+        // requestUSBDevice,
+    }] = useDecompress2(flashFile);
 
     useEffect(()=> {
         console.log(ZstdCodec)
@@ -47,7 +46,7 @@ const Decompress2 = () => {
             file to decompress:
             <input type="file" onChange={(e) => setFlashFile(e.target.files[0])}/>
 
-            <button onClick={requestUSBDevice}>Choose USB Dev </button>
+            {/* <button onClick={requestUSBDevice}>Choose USB Dev </button> */}
         </div>
     )
 }

webusb/src/components/DecompressAlt.js

@@ -0,0 +1,27 @@
+import { useEffect, useState } from "react"
+// import useDecompress from '../logic/useDecompress'
+import useDecompress from '../logic/useDecompress3'
+import { DATA_SZ, BLK_SZ, PACKET_SZ } from "../helper/sparse";
+
+const DecompressAlt = () => {
+    const [flashFile, setFlashFile] = useState();
+    const [{
+        requestUSBDevice
+    }] = useDecompress(flashFile);
+
+    useEffect(()=> {
+        console.log(ZstdCodec)
+        console.log(window.binding);
+    }, [])
+
+    return (
+        <div>
+            file to decompress (3):
+            <input type="file" onChange={(e) => setFlashFile(e.target.files[0])}/>
+
+            <button onClick={requestUSBDevice}>Pair USBDevice </button>
+        </div>
+    )
+}
+
+export default DecompressAlt

webusb/src/logic/flashMachine.js

@@ -0,0 +1,87 @@
+class ChunkProcessor 
+{
+    #chunk_size
+    #buffer
+    #on_chunk
+
+    /**
+     * This callback is displayed as part of the Requester class.
+     * @callback dataCallback
+     * @param {buffer} Uint8Array
+     */
+
+    /**
+     * @param {number} chunk_size 
+     * @param {dataCallback} on_chunk
+     */
+    constructor(chunk_size, on_chunk) 
+    {
+        this.#buffer = new Uint8Array();
+        this.#chunk_size = chunk_size;
+        this.#on_chunk = on_chunk;
+    }
+
+    /**
+     * @param {Uint8Array} new_buffer 
+     */
+    #append_buffer(new_buffer) {
+        const temp = new Uint8Array(this.#buffer.length + new_buffer.length);
+        temp.set(this.#buffer, 0);
+        temp.set(new_buffer, this.#buffer.length);
+        this.#buffer = temp;
+    }
+
+    /**
+     * Adds new data to the buffer, and consumes chunks
+     * @param {Uint8Array} new_buffer 
+     */
+    async push_data(new_buffer) {
+        if(this.#buffer.length + new_buffer.length < this.#chunk_size) {
+            this.#append_buffer(new_buffer);
+            return;
+        }
+
+        // process old data
+        const chunkPadCount = this.#chunk_size - this.#buffer.length;
+        this.#append_buffer(new_buffer.slice(0, chunkPadCount));
+        await this.#on_chunk(this.#buffer); //TODO:: is this await necessary?
+
+        // remove the data already processed
+        new_buffer = new_buffer.slice(chunkPadCount);
+        while(true) {
+            const curr_slice = new_buffer.slice(0, this.#chunk_size); // dont care about OOB, js handles it for us
+            if(curr_slice.length === this.#chunk_size) {
+                await this.#on_chunk(curr_slice); //TODO: same with this await?
+                new_buffer = new_buffer.slice(this.#chunk_size);
+            } else {
+                this.#buffer = curr_slice;
+                break;
+            }
+        }
+    }
+
+    /**
+     * flushes anything currently being buffered
+     * @param {dataCallback=} callback 
+     */
+    flush(callback) {
+        if(!callback) {
+            callback = this.#on_chunk;
+        }
+
+        callback(this.#buffer);
+        this.#buffer = new Uint8Array();
+    }
+
+    /**
+     * processes an entire array, including remaining data
+     * @param {Uint8Array} new_buffer 
+     * @param {dataCallback=} callback 
+     */
+    process_entire_arr(new_buffer, callback) {
+        this.push_data(new_buffer);
+        this.flush(callback);
+    }
+}
+
+export default ChunkProcessor

webusb/src/logic/notes.js

@@ -1,3 +1,4 @@
+<<<<<<< HEAD
 let buffer = new Uint8Array();
 
 let decompress_sz = 1024*64;
@@ -74,3 +75,33 @@ function sleep(ms) {
     console.log("End");
   })();
 
+=======
+async function callback() {
+
+    await send_data();
+
+    callback();
+}
+
+function caller () {
+    stream.transform(chunk_offset, slice, callback);
+}
+
+///
+
+function transform (chunk, callback){
+    // fill up a src_bytes worth from chunk
+    decompress(src_bytes);
+}
+
+function decompress(src_bytes) {
+    // loop over src_bytes
+
+    while (condition){
+        let decompressed = zstd_decompress(src_bytes);
+        callback(decompressed)
+    }
+
+    src_bytes.clear();
+}
+>>>>>>> 8ab6186ba1f75467c69e0196e4293e4c0bde5a9c

webusb/src/logic/useDecompress.js

@@ -1,4 +1,4 @@
-
+// stream reader version
 import {useEffect, useState} from 'react';
 import {str_to_arr, ab_to_str} from '../helper/functions.js'
 import {CHUNK_SZ, BLK_SZ, CHUNK_TYPE_RAW, CHUNK_TYPE_DONT_CARE, build_sparse_header, build_chunk_header} from '../helper/sparse.js'
@@ -151,13 +151,12 @@ const useDecompress = (flashFile) => {
         }
 
         const stream = new binding.ZstdDecompressStreamBinding();
-        // const transform = new TransformStream();
-        // const writer = transform.writable.getWriter();
+        const transform = new TransformStream();
+        const writer = transform.writable.getWriter();
 
         const callback = (decompressed) => {
-            // totalBytes += decompressed.length;
-            // writer.write(decompressed);
-            console.log(decompressed);
+            totalBytes += decompressed.length;
+            writer.write(decompressed);
         }
 
         if (!stream.begin()) {
@@ -186,10 +185,10 @@ const useDecompress = (flashFile) => {
             return null;
         }
 
-        // console.log("finishing unzipping");
+        console.log("finishing unzipping");
 
-        // const readable = transform.readable;
-        // doFlash(readable, PACKET_SZ, totalBytes);
+        const readable = transform.readable;
+        doFlash(readable, PACKET_SZ, totalBytes);
     }
 
     const doFlash = async(readable, size, totalSize) => {