Milab 4412 support mtx files

.changeset/free-pets-turn.md

@@ -0,0 +1,7 @@
+---
+'@platforma-open/milaboratories.samples-and-data.workflow': minor
+'@platforma-open/milaboratories.samples-and-data.model': minor
+'@platforma-open/milaboratories.samples-and-data.ui': minor
+---
+
+Added support for mtx files

model/src/args.ts

@@ -166,13 +166,26 @@ export const DatasetContentTaggedXsv = z
   .strict();
 export type DatasetContentTaggedXsv = z.infer<typeof DatasetContentTaggedXsv>;
 
+export const DatasetContentMtx = z
+  .object({
+    type: z.literal('Mtx'),
+    gzipped: z.boolean(),
+    data: z.record(
+      PlId,
+      ImportFileHandleSchema.nullable() /* null means sample is added to the dataset, but file is not yet set */
+    )
+  })
+  .strict();
+export type DatasetContentMtx = z.infer<typeof DatasetContentMtx>;
+
 export const DatasetContent = z.discriminatedUnion('type', [
   DatasetContentFastq,
   DatasetContentMultilaneFastq,
   DatasetContentTaggedFastq,
   DatasetContentFasta,
   DatasetContentXsv,
-  DatasetContentTaggedXsv
+  DatasetContentTaggedXsv,
+  DatasetContentMtx
 ]);
 export type DatasetContent = z.infer<typeof DatasetContent>;
 
@@ -197,6 +210,8 @@ export const DatasetXsv = Dataset(DatasetContentXsv);
 export type DatasetXsv = z.infer<typeof DatasetXsv>;
 export const DatasetTaggedXsv = Dataset(DatasetContentTaggedXsv);
 export type DatasetTaggedXsv = z.infer<typeof DatasetTaggedXsv>;
+export const DatasetMtx = Dataset(DatasetContentMtx);
+export type DatasetMtx = z.infer<typeof DatasetMtx>;
 
 export type DatasetAny = z.infer<typeof DatasetAny>;
 export type DatasetType = DatasetAny['content']['type'];

ui/src/DatasetPage.vue

@@ -20,6 +20,7 @@ import FastaDatasetPage from './FastaDatasetPage.vue';
 import FastqDatasetPage from './FastqDatasetPage.vue';
 import { argsModel } from './lens';
 import MultilaneFastqDatasetPage from './MultilaneFastqDatasetPage.vue';
+import MtxDatasetPage from './MtxDatasetPage.vue';
 import TaggedFastqDatasetPage from './TaggedFastqDatasetPage.vue';
 import TaggedXsvDatasetPage from './TaggedXsvDatasetPage.vue';
 import { UpdateDatasetDialog } from './UpdateDatasetDialog';
@@ -51,14 +52,14 @@ const readIndicesOptions: SimpleOption<string>[] = [
 
 const currentReadIndices = computed(() =>
   JSON.stringify(
-    (dataset.value.content.type === 'Fasta' || dataset.value.content.type === 'TaggedXsv' || dataset.value.content.type === 'Xsv') ? undefined : dataset.value.content.readIndices
+    (dataset.value.content.type === 'Fasta' || dataset.value.content.type === 'TaggedXsv' || dataset.value.content.type === 'Xsv' || dataset.value.content.type === 'Mtx') ? undefined : dataset.value.content.readIndices
   )
 );
 
 function setReadIndices(newIndices: string) {
   const indicesArray = ReadIndices.parse(JSON.parse(newIndices));
   dataset.update((ds) => {
-    if (ds.content.type !== 'Fasta' && ds.content.type !== 'TaggedXsv' && ds.content.type !== 'Xsv') ds.content.readIndices = indicesArray;
+    if (ds.content.type !== 'Fasta' && ds.content.type !== 'TaggedXsv' && ds.content.type !== 'Xsv' && ds.content.type !== 'Mtx') ds.content.readIndices = indicesArray;
     else throw new Error("Can't set read indices for fasta dataset.");
   });
 }
@@ -79,6 +80,7 @@ const datasetTypeOptions: ListOption<DatasetType>[] = [
   { value: 'TaggedFastq', label: "Tagged FASTQ" },
   { value: 'Xsv', label: "XSV" },
   { value: 'TaggedXsv', label: "Tagged XSV" },
+  { value: 'Mtx', label: "MTX" },
 ]
 </script>
 
@@ -113,6 +115,9 @@ const datasetTypeOptions: ListOption<DatasetType>[] = [
     <template v-else-if="dataset.value.content.type === 'TaggedXsv'">
       <TaggedXsvDatasetPage />
     </template>
+    <template v-else-if="dataset.value.content.type === 'Mtx'">
+      <MtxDatasetPage />
+    </template>
   </PlBlockPage>
 
   <!-- Settings panel -->
@@ -124,7 +129,7 @@ const datasetTypeOptions: ListOption<DatasetType>[] = [
       @update:model-value="(v) => dataset.update((ds) => (ds.content.gzipped = v))">
       Gzipped
     </PlCheckbox>
-    <PlBtnGroup v-if="dataset.value.content.type !== 'Fasta' && dataset.value.content.type !== 'TaggedXsv' && dataset.value.content.type !== 'Xsv'" :model-value="currentReadIndices"
+    <PlBtnGroup v-if="dataset.value.content.type !== 'Fasta' && dataset.value.content.type !== 'TaggedXsv' && dataset.value.content.type !== 'Xsv' && dataset.value.content.type !== 'Mtx'" :model-value="currentReadIndices"
       @update:model-value="setReadIndices" :options="readIndicesOptions" />
   </PlSlideModal>
 

ui/src/ImportDatasetDialog/ImportDatasetDialog.vue

@@ -3,6 +3,7 @@ import {
   BlockArgs,
   DatasetContentFasta,
   DatasetContentFastq,
+  DatasetContentMtx,
   DatasetContentMultilaneFastq,
   DatasetContentTaggedFastq,
   DatasetContentTaggedXsv,
@@ -126,6 +127,20 @@ function addFastaDatasetContent(args: BlockArgs, contentData: DatasetContentFast
   }
 }
 
+function addMtxDatasetContent(args: BlockArgs, contentData: DatasetContentMtx['data']) {
+  const getOrCreateSample = createGetOrCreateSample(args);
+
+  if (compiledPattern.value?.hasLaneMatcher || compiledPattern.value?.hasReadIndexMatcher)
+    throw new Error('Dataset has read or lane matcher, trying to add mtx dataset');
+
+  for (const f of parsedFiles.value) {
+    if (!f.match) continue;
+    const sample = f.match.sample.value;
+    const sampleId = getOrCreateSample(sample);
+    contentData[sampleId] = f.handle;
+  }
+}
+
 function addFastqDatasetContent(args: BlockArgs, contentData: DatasetContentFastq['data']) {
   const getOrCreateSample = createGetOrCreateSample(args);
 
@@ -281,6 +296,7 @@ async function addToExistingDataset() {
       addMultilaneFastqDatasetContent(args, dataset.content.data);
     else if (dataset.content.type === 'Xsv') addXsvDatasetContent(args, dataset.content.data);
     else if (dataset.content.type === 'TaggedXsv') addTaggedXsvDatasetContent(args, dataset.content.data);
+    else if (dataset.content.type === 'Mtx') addMtxDatasetContent(args, dataset.content.data);
     else throw new Error('Unknown dataset type');
   });
   await app.navigateTo(`/dataset?id=${datasetId}`);
@@ -291,6 +307,10 @@ const isXsv = () => {
   return data.files.map(f => getFileNameFromHandle(f)).every(f => f.endsWith('.csv') || f.endsWith('.tsv'))
 }
 
+const isMtx = () => {
+  return data.files.map(f => getFileNameFromHandle(f)).every(f => f.endsWith('.mtx') || f.endsWith('.mtx.gz'))
+}
+
 const xsvType = (): 'csv' | 'tsv' => {
   const fileNames = data.files.map(f => getFileNameFromHandle(f));
   if (fileNames.every(f => f.endsWith('.csv') || f.endsWith('.csv.gz'))) return 'csv';
@@ -330,6 +350,20 @@ async function createNewDataset() {
         }
       });
     } 
+    else if (isMtx()) {
+      const contentData: DatasetContentMtx['data'] = {};
+      addMtxDatasetContent(args, contentData);
+
+      args.datasets.push({
+        label: data.newDatasetLabel,
+        id: newDatasetId,
+        content: {
+          type: 'Mtx',
+          gzipped: data.gzipped,
+          data: contentData
+        }
+      });
+    }
     else if (data.readIndices.length === 0 /* fasta */) {
       const contentData: DatasetContentFasta['data'] = {};
       addFastaDatasetContent(args, contentData);
@@ -398,8 +432,10 @@ const canCreateOrAdd = computed(
   () =>
     hasMatchedFiles.value &&
     (data.mode === 'create-new-dataset' || data.targetAddDataset !== undefined) &&
-    // This prevents selecting fasta as type while having read index matcher in pattern
+    // This prevents selecting fasta/mtx/xsv as type while having read index matcher in pattern
     (data.readIndices.length !== 0 ||
+      isMtx() ||
+      isXsv() ||
       (compiledPattern.value?.hasReadIndexMatcher === false &&
         compiledPattern.value?.hasLaneMatcher === false))
 );

ui/src/MtxDatasetPage.vue

@@ -0,0 +1,195 @@
+<script setup lang="ts">
+import {
+  ClientSideRowModelModule,
+  ColDef,
+  GridApi,
+  GridOptions,
+  IRichCellEditorParams,
+  IRowNode,
+  MenuModule,
+  ModuleRegistry,
+  RichSelectModule
+} from 'ag-grid-enterprise';
+
+import { AgGridVue } from 'ag-grid-vue3';
+
+import { DatasetMtx, PlId } from '@platforma-open/milaboratories.samples-and-data.model';
+import { ImportFileHandle } from '@platforma-sdk/model';
+import { AgGridTheme, makeRowNumberColDef, PlAgCellFile } from '@platforma-sdk/ui-vue';
+import { computed } from 'vue';
+import { useApp } from './app';
+import { argsModel } from './lens';
+import { agSampleIdComparator } from './util';
+
+ModuleRegistry.registerModules([ClientSideRowModelModule, RichSelectModule, MenuModule]);
+
+const app = useApp();
+const datasetId = app.queryParams.id;
+
+type MtxDatasetRow = {
+  // undefined for an empty row at the end of the table
+  readonly sample: PlId | '';
+  readonly data?: ImportFileHandle | null;
+};
+
+function nullToUndefined<T>(value: T | undefined | null): T | undefined {
+  return value === null ? undefined : value;
+}
+
+function undefinedToNull<T>(value: T | undefined): T | null {
+  return value === undefined ? null : value;
+}
+
+const dataset = argsModel(app, {
+  get: (args) => args.datasets.find((ds) => ds.id === datasetId) as DatasetMtx,
+  onDisconnected: () => app.navigateTo('/')
+});
+
+const unusedIds = () => {
+  const usedIds = new Set(Object.keys(dataset.value.content.data));
+  return app.model.args.sampleIds.filter((id) => !usedIds.has(id));
+};
+
+const rowData = computed(() => {
+  const result: MtxDatasetRow[] = Object.entries(dataset.value.content.data).map(
+    ([sampleId, data]) => ({
+      sample: sampleId as PlId,
+      data
+    })
+  );
+  if (unusedIds().length > 0) result.push({ sample: '' });
+  return result;
+});
+
+const defaultColDef: ColDef = {
+  suppressHeaderMenuButton: true
+};
+
+const columnDefs = computed(() => {
+  const sampleLabels = app.model.args.sampleLabels as Record<string, string>;
+  const sampleIdComparator = agSampleIdComparator(sampleLabels);
+  const res: ColDef<MtxDatasetRow>[] = [
+    makeRowNumberColDef(),
+    {
+      headerName: app.model.args.sampleLabelColumnLabel,
+      flex: 1,
+      valueGetter: (params) => params.data?.sample,
+      editable: (params) => {
+        // only creating new records
+        return params.data?.sample === '';
+      },
+      valueSetter: (params) => {
+        if (params.oldValue !== '') throw new Error('Unexpected edit');
+        if (!params.newValue) return false;
+        dataset.update((ds) => (ds.content.data[params.newValue] = null));
+        return true;
+      },
+      cellEditor: 'agRichSelectCellEditor',
+      refData: { ...sampleLabels, '': '+ add sample' },
+      singleClickEdit: true,
+      cellEditorParams: {
+        values: unusedIds
+      } satisfies IRichCellEditorParams<MtxDatasetRow>,
+      pinned: 'left',
+      lockPinned: true,
+      comparator: sampleIdComparator
+    }
+  ];
+
+  res.push({
+    headerName: 'MTX file',
+    flex: 2,
+    cellStyle: { padding: 0 },
+
+    cellRendererParams: {
+      resolveProgress: (fileHandle: ImportFileHandle | undefined) => {
+        const progresses = app.progresses;
+        if (!fileHandle) return undefined;
+        else return progresses[fileHandle];
+      }
+    },
+
+    cellRendererSelector: (params) =>
+      params.data?.sample
+        ? {
+            component: 'PlAgCellFile',
+            params: {
+              extensions: dataset.value.content.gzipped ? ['mtx.gz'] : ['mtx']
+            }
+          }
+        : undefined,
+    valueGetter: (params) =>
+      params.data?.sample
+        ? nullToUndefined(dataset.value.content.data[params.data.sample])
+        : undefined,
+    valueSetter: (params) => {
+      const sample = params.data.sample;
+      if (sample === '') return false;
+      dataset.update((ds) => (ds.content.data[sample] = nullToUndefined(params.newValue)));
+      return true;
+    }
+  } as ColDef<MtxDatasetRow, ImportFileHandle>);
+
+  return res;
+});
+
+function isPlId(v: PlId | ''): v is PlId {
+  return v !== '';
+}
+
+function getSelectedSamples(
+  api: GridApi<MtxDatasetRow>,
+  node: IRowNode<MtxDatasetRow> | null
+): PlId[] {
+  const samples = api
+    .getSelectedRows()
+    .map((row) => row.sample)
+    .filter(isPlId);
+  if (samples.length !== 0) return samples;
+  const sample = node?.data?.sample;
+  if (!sample) return [];
+  return [sample];
+}
+
+const gridOptions: GridOptions<MtxDatasetRow> = {
+  getRowId: (row) => row.data.sample ?? 'new',
+  rowSelection: {
+    mode: 'multiRow',
+    checkboxes: false,
+    headerCheckbox: false
+  },
+  rowHeight: 45,
+  getMainMenuItems: (params) => {
+    return [];
+  },
+  getContextMenuItems: (params) => {
+    if (getSelectedSamples(params.api, params.node).length === 0) return [];
+    return [
+      {
+        name: 'Delete',
+        action: (params) => {
+          const samplesToDelete = getSelectedSamples(params.api, params.node);
+          dataset.update((ds) => {
+            for (const s of samplesToDelete) delete ds.content.data[s];
+          });
+        }
+      }
+    ];
+  },
+  components: {
+    PlAgCellFile
+  }
+};
+</script>
+
+<template>
+  <AgGridVue
+    :theme="AgGridTheme"
+    :style="{ height: '100%' }"
+    :rowData="rowData"
+    :defaultColDef="defaultColDef"
+    :columnDefs="columnDefs"
+    :gridOptions="gridOptions"
+  />
+</template>
+

ui/src/datasets.ts

@@ -49,6 +49,9 @@ export function getDsReadIndices(ds: DatasetAny): string[] {
     case 'Xsv':
     case 'TaggedXsv':
     case 'Fasta':
+    case 'Mtx':
+      return [];
+    default:
       return [];
   }
 }