feat(datastore): support complex column types

console/package.json

@@ -97,6 +97,7 @@
     "styletron-engine-atomic": "^1.4.8",
     "styletron-react": "^6.0.1",
     "ts-auto-mock": "^3.5.0",
+    "tsc": "^2.0.4",
     "tsconfig-paths-webpack-plugin": "^3.5.2",
     "typescript": "^4.6.4",
     "uuid": "^8.3.2",
@@ -108,6 +109,7 @@
     "xterm": "^4.14.1",
     "xterm-addon-fit": "^0.5.0",
     "xterm-addon-web-links": "^0.4.0",
+    "yarn": "^1.22.19",
     "zustand": "v4.1.1"
   },
   "scripts": {
@@ -120,7 +122,8 @@
     "cy:open": "cypress open",
     "start": "vite --host 0.0.0.0",
     "build": "NODE_OPTIONS='--max-old-space-size=5120' vite build",
-    "serve": "vite preview"
+    "serve": "vite preview",
+    "test": "jest"
   },
   "eslintConfig": {
     "extends": [
@@ -220,4 +223,4 @@
     "framer-motion": "4.1.17",
     "react-virtualized": "git+https://git@github.com/remorses/react-virtualized-fixed-import.git#9.22.3"
   }
-}
+}

console/src/domain/dataset/sdk.ts

@@ -1,5 +1,4 @@
 import isObject from 'lodash/isObject'
-import Typer, { IDataType } from '../datastore/sdk'
 import { tableDataLink } from '../datastore/utils'
 
 export type IBBox = [x: number, y: number, width: number, height: number]
@@ -74,8 +73,6 @@ export class DatasetObject {
 
     public data: IObjectImage
 
-    public columnTypes: Record<string, IDataType>
-
     // annotations
     public cocos: IAnnotationCOCOObject[]
 
@@ -85,14 +82,13 @@ export class DatasetObject {
 
     public objects: any[]
 
-    constructor(data: any, columnTypes: any) {
+    constructor(data: any) {
         this.src = data?.src ?? 0
-        this.size = data?.data_size ?? 0
-        this.offset = data?.data_offset ?? 0
+        this.size = Number(data?.data_size ?? 0)
+        this.offset = Number(data?.data_offset ?? 0)
         this.uri = data?.data_uri ?? ''
         this.authName = data?.auth_name ?? ''
         this.id = data?.id ?? ''
-        this.columnTypes = columnTypes
         this.mimeType = ''
         this.type = ''
         this.data = {} as any
@@ -156,8 +152,8 @@ export class DatasetObject {
         const src = tableDataLink(projectId, datasetVersionName, datasetVersionVersionName, {
             uri: this.uri,
             authName: this.authName,
-            offset: Typer?.[this.columnTypes.data_offset]?.encode(this.offset),
-            size: Typer?.[this.columnTypes.data_size]?.encode(this.size),
+            offset: this.offset.toString(16),
+            size: this.size.toString(16),
             Authorization: token as string,
         })
         this.src = src

console/src/domain/datastore/hooks/useParseDatastore.ts

@@ -1,12 +1,11 @@
-import omit from 'lodash/omit'
 import keyBy from 'lodash/keyBy'
 import React from 'react'
 import { RecordListVO } from '../schemas/datastore'
 
 export function useParseConfusionMatrix(data: RecordListVO = {}) {
     const labels = React.useMemo(() => {
         const { columnTypes } = data
-        return Object.keys(omit(columnTypes, 'id')).sort()
+        return (columnTypes?.map((column) => column.name)?.filter((name) => name !== 'id') ?? []).sort()
     }, [data])
 
     const binarylabel = React.useMemo(() => {

console/src/domain/datastore/schemas/datastore.tsx

@@ -1,6 +1,9 @@
 export interface ColumnSchemaDesc {
-    name?: string
-    type?: string
+    name: string
+    type: string
+    pythonType?: string
+    elementType?: ColumnSchemaDesc
+    attributes?: ColumnSchemaDesc[]
 }
 
 export interface RecordDesc {
@@ -49,11 +52,8 @@ export interface TableDesc {
 }
 
 export interface RecordListVO {
-    columnTypes?: Record<
-        string,
-        'UNKNOWN' | 'BOOL' | 'INT8' | 'INT16' | 'INT32' | 'INT64' | 'FLOAT32' | 'FLOAT64' | 'STRING' | 'BYTES'
-    >
-    records?: Record<string, string>[]
+    columnTypes?: ColumnSchemaDesc[]
+    records?: Record<string, any>[]
     lastKey?: string
 }
 

console/src/pages/Dataset/DatasetVersionOverviewFiles.tsx

@@ -160,17 +160,13 @@ export default function DatasetVersionFiles() {
         // eslint-disable-next-line react-hooks/exhaustive-deps
     }, [page, rowCount])
 
-    const columnTypes = React.useMemo(() => {
-        return tables.data?.columnTypes ?? {}
-    }, [tables.data])
-
     const [layoutKey, setLayoutKey] = React.useState('1')
     const [isFullscreen, setIsFullscreen] = React.useState(false)
 
     const datasets = React.useMemo(
         () =>
             tables?.data?.records?.map((record) => {
-                const dObj = new DatasetObject(record, columnTypes)
+                const dObj = new DatasetObject(record)
                 dObj.setDataSrc(
                     projectId,
                     datasetVersion?.name as string,
@@ -179,7 +175,7 @@ export default function DatasetVersionFiles() {
                 )
                 return dObj ?? []
             }) ?? [],
-        [tables?.data, columnTypes, projectId, datasetVersion, token]
+        [tables?.data, projectId, datasetVersion, token]
     )
 
     const Records = React.useMemo(() => {

console/src/pages/Evaluation/EvaluationListCard.tsx

@@ -144,27 +144,27 @@ export default function EvaluationListCard() {
 
         if (!summaryTable?.data) return columnsWithAttrs
 
-        Object.entries(summaryTable?.data?.columnTypes ?? {}).forEach(([name, type]) => {
-            if (name === 'id') return
-            switch (type) {
+        summaryTable?.data?.columnTypes?.forEach((column) => {
+            if (column.name === 'id') return
+            switch (column.type) {
                 case 'UNKNOWN':
                 case 'BYTES':
                     break
                 case 'STRING':
                     columnsWithAttrs.push(
                         StringColumn({
-                            key: name,
-                            title: name,
+                            key: column.name,
+                            title: column.name,
                             filterType: 'string',
-                            mapDataToValue: (data: any): string => data.attributes?.[name] ?? '-',
+                            mapDataToValue: (data: any): string => data.attributes?.[column.name] ?? '-',
                         })
                     )
                     break
                 default:
                     columnsWithAttrs.push(
                         CustomColumn({
-                            key: name,
-                            title: name,
+                            key: column.name,
+                            title: column.name,
                             sortable: true,
                             filterType: 'number',
                             sortFn: (a: any, b: any) => {
@@ -181,7 +181,7 @@ export default function EvaluationListCard() {
                                 if (props?.value === undefined) return '-'
                                 return <p title={props?.value}>{parseDecimal(props?.value, 4)}</p>
                             },
-                            mapDataToValue: (data: any): string => data.attributes?.[name] ?? undefined,
+                            mapDataToValue: (data: any): string => data.attributes?.[column.name] ?? undefined,
                         })
                     )
                     break

console/src/pages/Evaluation/EvaluationListCompare.tsx

@@ -180,11 +180,11 @@ export default function EvaluationListCompare({
     const $rowWithAttrs = useMemo(() => {
         const rowWithAttrs = [...$rows]
 
-        Object.entries(attrs ?? {}).forEach(([name]) => {
+        attrs?.forEach((attr) => {
             rowWithAttrs.push({
-                key: name,
-                title: name,
-                values: rows.map((data: any) => data.attributes?.[name] ?? '-'),
+                key: attr.name,
+                title: attr.name,
+                values: rows.map((data: any) => data.attributes?.[attr.name] ?? '-'),
                 renderCompare: NumberCompareCell,
             })
         })

console/src/pages/Evaluation/EvaluationResults.tsx

@@ -106,13 +106,9 @@ function EvaluationViewer({ table, filter }: { table: string; filter?: Record<st
 
     const info = useQueryDatastore(query, true)
 
-    const columnTypes = React.useMemo(() => {
-        if (!info.data) return {}
-        return info.data?.columnTypes ?? {}
-    }, [info])
     const columns = React.useMemo(() => {
-        return Object.keys(columnTypes).sort((a) => (a === 'id' ? -1 : 1)) ?? []
-    }, [columnTypes])
+        return info.data?.columnTypes?.map((column) => column.name)?.sort((a) => (a === 'id' ? -1 : 1)) ?? []
+    }, [info])
     const data = React.useMemo(() => {
         if (!info.data) return []