[POC- DO NOT MERGE] Add heatmap for fingerprinting recordings

package.json

@@ -54,6 +54,7 @@
     "@microsoft/applicationinsights-web": "^3.0.0",
     "@sentry/svelte": "^7.100.1",
     "@tensorflow/tfjs": "^4.4.0",
+    "@tensorflow/tfjs-vis": "^1.5.1",
     "@types/w3c-web-serial": "^1.0.6",
     "@types/w3c-web-usb": "^1.0.6",
     "bowser": "^2.11.0",

src/__tests__/ml.test.ts

@@ -8,14 +8,22 @@
  */
 
 import * as tf from '@tensorflow/tfjs';
-import { makeInputs, trainModel } from '../script/ml';
+import { ModelSettings, makeInputs, trainModel } from '../script/ml';
 import { gestures } from '../script/stores/Stores';
 import gestureData from './fixtures/gesture-data.json';
 import gestureDataBadLabels from './fixtures/gesture-data-bad-labels.json';
 import testdataShakeStill from './fixtures/test-data-shake-still.json';
 import { PersistantGestureData } from '../script/domain/Gestures';
+import { get } from 'svelte/store';
+import { settings } from '../script/stores/mlStore';
 
 let tensorFlowModel: tf.LayersModel | void;
+const mlSettings = get(settings);
+const modelSettings = {
+  axes: mlSettings.includedAxes,
+  filters: mlSettings.includedFilters,
+};
+
 beforeAll(async () => {
   // No webgl in tests running in node.
   tf.setBackend('cpu');
@@ -34,7 +42,7 @@ const getModelResults = (data: PersistantGestureData[]) => {
   const numActions = data.length;
   data.forEach((action, index) => {
     action.recordings.forEach(recording => {
-      x.push(makeInputs(recording.data));
+      x.push(makeInputs(modelSettings, recording.data));
       const label = new Array(numActions);
       label.fill(0, 0, numActions);
       label[index] = 1;

src/components/Fingerprint.svelte

@@ -0,0 +1,60 @@
+<!--
+  (c) 2024, Center for Computational Thinking and Design at Aarhus University and contributors
+
+  SPDX-License-Identifier: MIT
+ -->
+
+<script lang="ts">
+  import * as tfvis from '@tensorflow/tfjs-vis';
+  import { onMount } from 'svelte';
+  import { get } from 'svelte/store';
+  import { makeInputs, ModelSettings } from '../script/ml';
+  import { RecordingData, settings } from '../script/stores/mlStore';
+
+  export let recordingData: RecordingData;
+  export let gestureName: string;
+
+  let surface: undefined | tfvis.Drawable;
+
+  const mlSettings = get(settings);
+  const modelSettings: ModelSettings = {
+    axes: mlSettings.includedAxes,
+    filters: mlSettings.includedFilters,
+  };
+
+  const filtersLabels: string[] = [];
+  modelSettings.filters.forEach(filter => {
+    filtersLabels.push(`${filter}-x`, `${filter}-y`, `${filter}-z`);
+  });
+
+  const processedData = makeInputs(
+    modelSettings,
+    recordingData.data,
+    'computeNormalizedOutput',
+  );
+
+  const chartData = {
+    values: [processedData],
+    xTickLabels: filtersLabels,
+    yTickLabels: [gestureName],
+  };
+
+  onMount(() => {
+    if (surface) {
+      tfvis.render.heatmap(surface, chartData, {
+        colorMap: 'viridis',
+        height: 20,
+        width: 204,
+        rowMajor: true,
+        domain: [0, 1],
+        fontSize: 0,
+      });
+    }
+  });
+</script>
+
+<div class="relative h-20px w-full">
+  <div class="absolute h-20px w-full -bottom-8px -left-8px">
+    <div bind:this={surface}></div>
+  </div>
+</div>

src/components/Gesture.svelte

@@ -366,7 +366,11 @@
         </div>
         {#if hasRecordings}
           {#each $gesture.recordings as recording (String($gesture.ID) + String(recording.ID))}
-            <Recording {recording} onDelete={deleteRecording} on:focus={selectGesture} />
+            <Recording
+              gestureName={$nameBind}
+              {recording}
+              onDelete={deleteRecording}
+              on:focus={selectGesture} />
           {/each}
         {/if}
       </div>

src/components/Recording.svelte

@@ -5,20 +5,22 @@
  -->
 
 <script lang="ts">
-  import { fade } from 'svelte/transition';
+  import CloseIcon from 'virtual:icons/ri/close-line';
+  import { t } from '../i18n';
   import type { RecordingData } from '../script/stores/mlStore';
+  import Fingerprint from './Fingerprint.svelte';
   import RecordingGraph from './graphs/RecordingGraph.svelte';
   import IconButton from './IconButton.svelte';
-  import { t } from '../i18n';
-  import CloseIcon from 'virtual:icons/ri/close-line';
 
   // get recording from mother prop
   export let recording: RecordingData;
+  export let gestureName: string;
   export let onDelete: (recording: RecordingData) => void;
 </script>
 
-<div class="h-full w-40 relative">
+<div class="h-full flex flex-col w-40 relative overflow-hidden">
   <RecordingGraph data={recording.data} />
+  <Fingerprint {gestureName} recordingData={recording} />
 
   <div class="absolute right-0 top-0 z-2">
     <IconButton

src/pages/ProcessDataPage.svelte

@@ -0,0 +1,137 @@
+<!--
+  (c) 2024, Center for Computational Thinking and Design at Aarhus University and contributors
+
+  SPDX-License-Identifier: MIT
+ -->
+
+<script lang="ts">
+  import * as tfvis from '@tensorflow/tfjs-vis';
+  import { onMount } from 'svelte';
+  import { gestures } from '../script/stores/Stores';
+  import { get } from 'svelte/store';
+  import { getPrevData, settings } from '../script/stores/mlStore';
+  import { makeInputs, ModelSettings } from '../script/ml';
+  import TabView from '../views/TabView.svelte';
+  import { state } from '../script/stores/uiStore';
+  import BottomPanel from '../components/bottom/BottomPanel.svelte';
+  import LoadingAnimation from '../components/LoadingBlobs.svelte';
+
+  let surfaceAll: undefined | tfvis.Drawable;
+  let surfaceCurrent: undefined | tfvis.Drawable;
+  $: prevData = getPrevData();
+  const filters = Array.from(get(settings).includedFilters);
+
+  const mlSettings = get(settings);
+  const modelSettings: ModelSettings = {
+    axes: mlSettings.includedAxes,
+    filters: mlSettings.includedFilters,
+  };
+
+  const filtersLabels: string[] = [];
+  filters.forEach(filter => {
+    filtersLabels.push(`${filter}-x`, `${filter}-y`, `${filter}-z`);
+  });
+  const allData = $gestures.map(g => {
+    const inputData: number[][] = [];
+    g.recordings.forEach(r => {
+      inputData.push(makeInputs(modelSettings, r.data, 'computeNormalizedOutput'));
+    });
+    const averagedData: number[] = [];
+    for (let i = 0; i < filters.length * 3; i++) {
+      let filterValues: number[] = [];
+      inputData.forEach(d => {
+        filterValues.push(d[i]);
+      });
+      averagedData.push(filterValues.reduce((a, b) => a + b, 0) / filterValues.length);
+    }
+    return averagedData;
+  });
+
+  const data = {
+    values: allData,
+    xTickLabels: filtersLabels,
+    yTickLabels: $gestures.map(g => g.name),
+  };
+
+  onMount(() => {
+    const renderAllDataHeatmap = () => {
+      if (surfaceAll) {
+        tfvis.render.heatmap(surfaceAll, data, {
+          colorMap: 'viridis',
+          height: 250,
+          rowMajor: true,
+          domain: [0, 1],
+        });
+      }
+    };
+
+    renderAllDataHeatmap();
+
+    window.addEventListener('resize', renderAllDataHeatmap);
+
+    const interval = setInterval(() => {
+      prevData = getPrevData();
+
+      if (surfaceCurrent && prevData) {
+        const currentGestureData = makeInputs(
+          modelSettings,
+          prevData,
+          'computeNormalizedOutput',
+        );
+        const currentData = {
+          values: [currentGestureData],
+          xTickLabels: filtersLabels,
+          yTickLabels: ['Live'],
+        };
+        tfvis.render.heatmap(surfaceCurrent, currentData, {
+          colorMap: 'viridis',
+          height: 120,
+          rowMajor: true,
+          domain: [0, 1],
+        });
+      }
+    });
+    return () => {
+      clearInterval(interval);
+      window.removeEventListener('resize', renderAllDataHeatmap);
+    };
+  });
+</script>
+
+<div class="flex flex-col h-full inline-block w-full bg-backgrounddark">
+  <TabView />
+  <main class="contents">
+    <h1 class="sr-only">Process data</h1>
+    <div class="flex flex-col flex-grow items-center h-0 overflow-y-auto">
+      <div class="p-5 flex-grow flex flex-col gap-5 w-3/4">
+        <div class="flex flex-col gap-2">
+          <h2 class="font-semibold">All recordings</h2>
+          <p>The mean of the filters applied to all recordings for each action</p>
+          <div class="bg-white p-5 rounded-lg w-full">
+            <div bind:this={surfaceAll}></div>
+          </div>
+        </div>
+        <div class="flex flex-col gap-2">
+          <h2 class="font-semibold">Current action</h2>
+          <p>Filters applied to the live data</p>
+          <div class="bg-white p-5 min-h-160px rounded-lg w-full">
+            {#if $state.isInputConnected && prevData}
+              <div bind:this={surfaceCurrent}></div>
+            {:else if $state.isInputConnected && !prevData}
+              <div class="flex justify-center items-center h-full">
+                <LoadingAnimation />
+              </div>
+            {:else}
+              <div class="flex justify-center items-center h-full">
+                <p>Connect your micro:bit view live data</p>
+              </div>
+            {/if}
+          </div>
+        </div>
+      </div>
+    </div>
+    <div class="h-160px w-full">
+      <BottomPanel />
+    </div>
+  </main>
+</div>

src/router/Router.svelte

@@ -16,6 +16,7 @@
   import TrainingPage from '../pages/training/TrainingPage.svelte';
   import { currentPageComponent } from '../views/currentComponentStore';
   import { currentPath, isValidPath, navigate, Paths, PathType } from './paths';
+  import FingerprintPage from '../pages/ProcessDataPage.svelte';
 
   const stripLeadingSlash = (s: string): string => (s.startsWith('/') ? s.slice(1) : s);
 
@@ -33,6 +34,8 @@
         return DataPage;
       case Paths.TRAINING:
         return TrainingPage;
+      case Paths.PROCESS:
+        return FingerprintPage;
       case Paths.MODEL:
         return ModelPage;
       case Paths.FILTERS:

src/router/paths.ts

@@ -14,6 +14,7 @@ export const Paths = {
   GET_STARTED: 'resources/get-started',
   DATA: 'add-data',
   TRAINING: 'train-model',
+  PROCESS: 'process-data',
   MODEL: 'test-model',
   FILTERS: 'training/filters',
 } as const;
@@ -45,6 +46,9 @@ export const getTitle = (path: PathType, t: MessageFormatter) => {
     case Paths.TRAINING: {
       return `${t('content.index.toolProcessCards.train.title')} | ${appName}`;
     }
+    case Paths.PROCESS: {
+      return `Process data | ${appName}`;
+    }
     case Paths.MODEL: {
       return `${t('content.index.toolProcessCards.model.title')} | ${appName}`;
     }