Update column paths when the column set changes in perspective-viewer-datagrid

packages/perspective-viewer-datagrid/src/js/custom_elements/datagrid.js

@@ -123,6 +123,14 @@ export class HTMLPerspectiveViewerDatagridPluginElement extends HTMLElement {
     async update(view) {
         if (this.model === undefined) {
             await this.draw(view);
+        } else if (this.model._config.split_by?.length > 0) {
+            const dimensions = await view.dimensions();
+            this.model._num_rows = dimensions.num_view_rows;
+            // if (this.model._column_paths.length !== dimensions.num_view_columns) {
+            // await this.draw(view);
+            // } else {
+            await this.regular_table.draw();
+            // }
         } else {
             this.model._num_rows = await view.num_rows();
             await this.regular_table.draw();

packages/perspective-viewer-datagrid/src/js/data_listener/index.js

@@ -45,9 +45,45 @@ export function createDataListener(viewer) {
                 id: true,
             };
 
-            columns = JSON.parse(
-                await this._view.to_columns_string(new_window)
-            );
+            if (this._config.split_by?.length > 0) {
+                const [x, y] = await Promise.all([
+                    this._view.to_columns_string(new_window),
+                    this._view.column_paths(new_window),
+                ]);
+
+                columns = JSON.parse(x);
+                const new_col_paths = y.filter(
+                    (x) => x !== "__ROW_PATH__" && x !== "__ID__"
+                );
+
+                const old_length = this._column_paths.length;
+                this._column_paths.splice(
+                    new_window.start_col,
+                    new_col_paths.length,
+                    ...new_col_paths
+                );
+
+                if (this._column_paths.length !== old_length) {
+                    const [a, b] = await Promise.all([
+                        this._view.schema(),
+                        this._view.expression_schema(),
+                    ]);
+
+                    this._schema = { ...a, ...b };
+                    for (let i = 0; i < new_col_paths.length; i++) {
+                        const column_path_parts = new_col_paths[i].split("|");
+                        const column =
+                            column_path_parts[this._config.split_by.length];
+
+                        this._column_types[i + new_window.start_col] =
+                            this._schema[column];
+                    }
+                }
+            } else {
+                columns = JSON.parse(
+                    await this._view.to_columns_string(new_window)
+                );
+            }
 
             this._last_window = new_window;
             this._ids = columns.__ID__;
@@ -73,6 +109,10 @@ export function createDataListener(viewer) {
 
         const is_settings_open = viewer.hasAttribute("settings");
 
+        // if (this._config.split_by?.length > 0) {
+        //     this._column_paths
+        // }
+
         // for (const path of this._column_paths.slice(x0, x1)) {
         for (
             let ipath = x0;

rust/generate-metadata/main.rs

@@ -32,8 +32,8 @@ use std::fs;
 
 use perspective_client::config::*;
 use perspective_client::{
-    DeleteOptions, OnUpdateData, OnUpdateOptions, SystemInfo, TableInitOptions, UpdateOptions,
-    ViewWindow,
+    ColumnWindow, DeleteOptions, OnUpdateData, OnUpdateOptions, SystemInfo, TableInitOptions,
+    UpdateOptions, ViewWindow,
 };
 use perspective_viewer::config::ViewerConfigUpdate;
 use ts_rs::TS;
@@ -64,6 +64,7 @@ fn generate_exprtk_docs() -> Result<(), Box<dyn Error>> {
 #[doc(hidden)]
 pub fn generate_type_bindings_js() -> Result<(), Box<dyn Error>> {
     let path = std::env::current_dir()?.join("../perspective-js/src/ts/ts-rs");
+    ColumnWindow::export_all_to(&path)?;
     ViewWindow::export_all_to(&path)?;
     TableInitOptions::export_all_to(&path)?;
     ViewConfigUpdate::export_all_to(&path)?;
@@ -72,7 +73,7 @@ pub fn generate_type_bindings_js() -> Result<(), Box<dyn Error>> {
     UpdateOptions::export_all_to(&path)?;
     DeleteOptions::export_all_to(&path)?;
     ViewWindow::export_all_to(&path)?;
-    SystemInfo::export_all_to(&path)?;
+    SystemInfo::<f64>::export_all_to(&path)?;
     Ok(())
 }
 

rust/perspective-client/Cargo.toml

@@ -65,6 +65,7 @@ itertools = { version = "0.10.1" }
 paste = { version = "1.0.12" }
 prost-types = { version = "0.12.3" }
 getrandom = { version = "0.3", features = ["wasm_js"] }
+num-traits = "0.2.19"
 rand = { version = "=0.9.1" }
 rand-unique = { version = "0.2.0" }
 serde = { version = "1.0", features = ["derive"] }

rust/perspective-client/perspective.proto

@@ -420,7 +420,10 @@ message ViewToArrowResp {
     bytes arrow = 1;
 }
 
-message ViewColumnPathsReq {}
+message ViewColumnPathsReq {
+    optional uint32 start_col = 1;
+    optional uint32 end_col = 2;
+}
 
 // // TODO This is a better paths representations but its not compatible with
 // // the legacy API. Let's do this when we can fix the API.

rust/perspective-client/src/rust/client.rs

@@ -37,12 +37,12 @@ use crate::{OnUpdateMode, OnUpdateOptions, asyncfn, clone};
 
 /// Metadata about the engine runtime (such as total heap utilization).
 #[derive(Clone, Debug, Serialize, Deserialize, TS)]
-pub struct SystemInfo {
+pub struct SystemInfo<T = u64> {
     /// Total available bytes for allocation on the [`Server`].
-    pub heap_size: u64,
+    pub heap_size: T,
 
     /// Bytes allocated for use on the [`Server`].
-    pub used_size: u64,
+    pub used_size: T,
 
     /// Wall-clock time spent processing requests on the [`Server`], in
     /// milliseconds (estimated). This does not properly account for the
@@ -56,15 +56,34 @@ pub struct SystemInfo {
     /// Timestamp (POSIX) this request was made. This field may be omitted
     /// for wasm due to `perspective-client` lacking a dependency on
     /// `wasm_bindgen`.
-    pub timestamp: Option<u64>,
+    pub timestamp: Option<T>,
 
     /// Total available bytes for allocation on the [`Client`]. This is only
     /// available if `trace-allocator` is enabled.
-    pub client_heap: Option<u64>,
+    pub client_heap: Option<T>,
 
     /// Bytes allocated for use on the [`Client`].  This is only
     /// available if `trace-allocator` is enabled.
-    pub client_used: Option<u64>,
+    pub client_used: Option<T>,
+}
+
+impl<U: Copy + 'static> SystemInfo<U> {
+    /// Convert the numeric representation for `T` to something else, which is
+    /// useful for JavaScript where there is no `u64` native type.
+    pub fn cast<T: Copy + 'static>(&self) -> SystemInfo<T>
+    where
+        U: num_traits::AsPrimitive<T>,
+    {
+        SystemInfo {
+            heap_size: self.heap_size.as_(),
+            used_size: self.used_size.as_(),
+            cpu_time: self.cpu_time,
+            cpu_time_epoch: self.cpu_time_epoch,
+            timestamp: self.timestamp.map(|x| x.as_()),
+            client_heap: self.client_heap.map(|x| x.as_()),
+            client_used: self.client_used.map(|x| x.as_()),
+        }
+    }
 }
 
 /// Metadata about what features are supported by the `Server` to which this

rust/perspective-client/src/rust/lib.rs

@@ -53,7 +53,9 @@ pub use crate::table::{
     DeleteOptions, ExprValidationResult, Table, TableInitOptions, TableReadFormat, UpdateOptions,
 };
 pub use crate::table_data::{TableData, UpdateData};
-pub use crate::view::{OnUpdateData, OnUpdateMode, OnUpdateOptions, View, ViewWindow};
+pub use crate::view::{
+    ColumnWindow, OnUpdateData, OnUpdateMode, OnUpdateOptions, View, ViewWindow,
+};
 
 pub type ClientError = utils::ClientError;
 pub type ExprValidationError = crate::proto::table_validate_expr_resp::ExprValidationError;

rust/perspective-client/src/rust/view.rs

@@ -69,6 +69,15 @@ pub struct Dimensions {
     pub num_table_columns: usize,
 }
 
+#[derive(Clone, Debug, Default, Deserialize, Serialize, TS, PartialEq)]
+pub struct ColumnWindow {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub start_col: Option<f32>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub end_col: Option<f32>,
+}
+
 /// Options for serializing a window of data from a [`View`].
 ///
 /// Some fields of [`ViewWindow`] are only applicable to specific methods of
@@ -239,8 +248,12 @@ impl View {
     ///
     /// A column path shows the columns that a given cell belongs to after
     /// pivots are applied.
-    pub async fn column_paths(&self) -> ClientResult<Vec<String>> {
-        let msg = self.client_message(ClientReq::ViewColumnPathsReq(ViewColumnPathsReq {}));
+    pub async fn column_paths(&self, window: ColumnWindow) -> ClientResult<Vec<String>> {
+        let msg = self.client_message(ClientReq::ViewColumnPathsReq(ViewColumnPathsReq {
+            start_col: window.start_col.map(|x| x as u32),
+            end_col: window.end_col.map(|x| x as u32),
+        }));
+
         match self.client.oneshot(&msg).await? {
             ClientResp::ViewColumnPathsResp(ViewColumnPathsResp { paths }) => {
                 // Ok(paths.into_iter().map(|x| x.path).collect())

rust/perspective-js/build.js

@@ -142,9 +142,6 @@ async function build_all() {
         "inherit",
         "inherit"
     );
-
-    await cpy("target/themes/*", "dist/css");
-    await cpy("target/themes/*", "dist/css");
 }
 
 build_all();

rust/perspective-js/src/rust/client.rs

@@ -477,7 +477,7 @@ impl Client {
             .map(|x| x.now() as u64);
 
         info.timestamp = timestamp;
-        let record = JsValue::from_serde_ext(&info)?;
+        let record = JsValue::from_serde_ext(&info.cast::<f64>())?;
         Ok(record.unchecked_into())
     }
 }

rust/perspective-js/src/rust/lib.rs

@@ -42,6 +42,7 @@ pub use crate::table_data::*;
 #[wasm_bindgen(typescript_custom_section)]
 const TS_APPEND_CONTENT: &'static str = r#"
 export type * from "../../src/ts/ts-rs/ViewWindow.d.ts";
+export type * from "../../src/ts/ts-rs/ColumnWindow.d.ts";
 export type * from "../../src/ts/ts-rs/TableInitOptions.d.ts";
 export type * from "../../src/ts/ts-rs/ViewConfigUpdate.d.ts";
 export type * from "../../src/ts/ts-rs/ViewOnUpdateResp.d.ts";
@@ -50,6 +51,7 @@ export type * from "../../src/ts/ts-rs/UpdateOptions.d.ts";
 export type * from "../../src/ts/ts-rs/DeleteOptions.d.ts";
 export type * from "../../src/ts/ts-rs/SystemInfo.d.ts";
 
+import type {ColumnWindow} from "../../src/ts/ts-rs/ColumnWindow.d.ts";
 import type {ViewWindow} from "../../src/ts/ts-rs/ViewWindow.d.ts";
 import type {TableInitOptions} from "../../src/ts/ts-rs/TableInitOptions.d.ts";
 import type {ViewConfigUpdate} from "../../src/ts/ts-rs/ViewConfigUpdate.d.ts";

rust/perspective-js/src/rust/utils/serde.rs

@@ -32,7 +32,6 @@ where
     {
         t.serialize(
             &serde_wasm_bindgen::Serializer::new()
-                .serialize_large_number_types_as_bigints(true)
                 .serialize_maps_as_objects(true)
                 .serialize_missing_as_null(true),
         )

rust/perspective-js/src/rust/view.rs

@@ -11,7 +11,9 @@
 // ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
 
 use js_sys::{Array, ArrayBuffer, Function, Object};
-use perspective_client::{OnUpdateData, OnUpdateOptions, ViewWindow, assert_view_api};
+use perspective_client::{
+    ColumnWindow, OnUpdateData, OnUpdateOptions, ViewWindow, assert_view_api,
+};
 use wasm_bindgen::prelude::*;
 use wasm_bindgen_futures::spawn_local;
 
@@ -25,6 +27,10 @@ unsafe extern "C" {
     #[derive(Clone)]
     pub type JsViewWindow;
 
+    #[wasm_bindgen(typescript_type = "ColumnWindow")]
+    #[derive(Clone)]
+    pub type JsColumnWindow;
+
     #[wasm_bindgen(method, setter, js_name = "formatted")]
     pub fn set_formatted(this: &JsViewWindow, x: bool);
 
@@ -79,8 +85,9 @@ impl View {
     /// A column path shows the columns that a given cell belongs to after
     /// pivots are applied.
     #[wasm_bindgen]
-    pub async fn column_paths(&self) -> ApiResult<JsValue> {
-        let columns = self.0.column_paths().await?;
+    pub async fn column_paths(&self, window: Option<JsColumnWindow>) -> ApiResult<JsValue> {
+        let window = window.into_serde_ext::<Option<ColumnWindow>>()?;
+        let columns = self.0.column_paths(window.unwrap_or_default()).await?;
         Ok(JsValue::from_serde_ext(&columns)?)
     }
 

rust/perspective-js/src/ts/wasm/engine.ts

@@ -148,7 +148,7 @@ export class PerspectiveSession {
             polled,
             async (msg: ApiResponse) => {
                 if (msg.client_id === 0) {
-                    console.error("Poll error");
+                    await this.client_map.get(this.client_id)!(msg.data);
                 } else {
                     await this.client_map.get(msg.client_id)!(msg.data);
                 }

rust/perspective-js/test/js/aggregates.spec.js

@@ -152,7 +152,7 @@ const std = (nums) => {
                 },
             });
             const paths = await view.column_paths();
-            expect(paths).toEqual(["__ROW_PATH__", "y", "z"]);
+            expect(paths).toEqual(["y", "z"]);
             const answer = [
                 { __ROW_PATH__: [], y: "c", z: true },
                 { __ROW_PATH__: [false], y: "d", z: false },
@@ -174,7 +174,7 @@ const std = (nums) => {
                 },
             });
             const paths = await view.column_paths();
-            expect(paths).toEqual(["__ROW_PATH__", "y", "z"]);
+            expect(paths).toEqual(["y", "z"]);
             const answer = [
                 { __ROW_PATH__: [], y: 4, z: 4 },
                 { __ROW_PATH__: [false], y: 2, z: 2 },

rust/perspective-js/test/js/expressions/functionality.spec.js

@@ -2385,14 +2385,7 @@ import perspective from "../perspective_client";
             });
 
             let paths = await view.column_paths();
-            expect(paths).toEqual([
-                "__ROW_PATH__",
-                "w",
-                "x",
-                "y",
-                "z",
-                "column",
-            ]);
+            expect(paths).toEqual(["w", "x", "y", "z", "column"]);
 
             await view.delete();
 
@@ -2404,7 +2397,6 @@ import perspective from "../perspective_client";
 
             for (const expected of expected_paths) {
                 const output = expected.slice();
-                output.unshift("__ROW_PATH__");
                 view = await table.view({
                     group_by: ["y"],
                     expressions: { column: '"w" + "x"' },
@@ -2417,7 +2409,6 @@ import perspective from "../perspective_client";
 
             for (const expected of expected_paths) {
                 const output = expected.slice();
-                output.unshift("__ROW_PATH__");
                 view = await table.view({
                     group_by: ["column"],
                     expressions: { column: '"w" + "x"' },
@@ -2448,7 +2439,7 @@ import perspective from "../perspective_client";
             let view = await table.view(config);
 
             let paths = await view.column_paths();
-            expect(paths).toEqual(["__ROW_PATH__", "w", "x", "y", "z", "1234"]);
+            expect(paths).toEqual(["w", "x", "y", "z", "1234"]);
 
             await view.delete();
 
@@ -2460,7 +2451,6 @@ import perspective from "../perspective_client";
 
             for (const expected of expected_paths) {
                 const output = expected.slice();
-                output.unshift("__ROW_PATH__");
                 view = await table.view({
                     ...config,
                     columns: expected,
@@ -2472,7 +2462,6 @@ import perspective from "../perspective_client";
 
             for (const expected of expected_paths) {
                 const output = expected.slice();
-                output.unshift("__ROW_PATH__");
                 view = await table.view({
                     ...config,
                     columns: expected,