Add OpenCV DNN pipeline

photon-client/src/components/dashboard/CameraAndPipelineSelectCard.vue

@@ -24,6 +24,9 @@ const changeCurrentCameraIndex = (index: number) => {
     case PipelineType.Aruco:
       pipelineType.value = WebsocketPipelineType.Aruco;
       break;
+    case PipelineType.Dnn:
+      pipelineType.value = WebsocketPipelineType.Dnn;
+      break;
   }
 };
 
@@ -152,7 +155,8 @@ const pipelineTypesWrapper = computed<{ name: string; value: number }[]>(() => {
     { name: "Reflective", value: WebsocketPipelineType.Reflective },
     { name: "Colored Shape", value: WebsocketPipelineType.ColoredShape },
     { name: "AprilTag", value: WebsocketPipelineType.AprilTag },
-    { name: "Aruco", value: WebsocketPipelineType.Aruco }
+    { name: "Aruco", value: WebsocketPipelineType.Aruco },
+    { name: "DNN", value: WebsocketPipelineType.Dnn }
   ];
 
   if (useCameraSettingsStore().isDriverMode) {
@@ -208,6 +212,9 @@ useCameraSettingsStore().$subscribe((mutation, state) => {
     case PipelineType.Aruco:
       pipelineType.value = WebsocketPipelineType.Aruco;
       break;
+    case PipelineType.Dnn:
+      pipelineType.value = WebsocketPipelineType.Dnn;
+      break;
   }
 });
 </script>
@@ -354,7 +361,8 @@ useCameraSettingsStore().$subscribe((mutation, state) => {
               { name: 'Reflective', value: WebsocketPipelineType.Reflective },
               { name: 'Colored Shape', value: WebsocketPipelineType.ColoredShape },
               { name: 'AprilTag', value: WebsocketPipelineType.AprilTag },
-              { name: 'Aruco', value: WebsocketPipelineType.Aruco }
+              { name: 'Aruco', value: WebsocketPipelineType.Aruco },
+              { name: 'Dnn', value: WebsocketPipelineType.Dnn }
             ]"
           />
         </v-card-text>

photon-client/src/components/dashboard/ConfigOptions.vue

@@ -12,6 +12,7 @@ import OutputTab from "@/components/dashboard/tabs/OutputTab.vue";
 import TargetsTab from "@/components/dashboard/tabs/TargetsTab.vue";
 import PnPTab from "@/components/dashboard/tabs/PnPTab.vue";
 import Map3DTab from "@/components/dashboard/tabs/Map3DTab.vue";
+import DnnTab from "@/components/dashboard/tabs/DnnTab.vue";
 import { WebsocketPipelineType } from "@/types/WebsocketDataTypes";
 
 interface ConfigOption {
@@ -55,6 +56,10 @@ const allTabs = Object.freeze({
   map3dTab: {
     tabName: "3D",
     component: Map3DTab
+  },
+  dnnTab: {
+    tabName: "DNN",
+    component: DnnTab
   }
 });
 
@@ -75,21 +80,29 @@ const getTabGroups = (): ConfigOption[][] => {
         allTabs.contoursTab,
         allTabs.apriltagTab,
         allTabs.arucoTab,
+        allTabs.dnnTab,
         allTabs.outputTab
       ],
       [allTabs.targetsTab, allTabs.pnpTab, allTabs.map3dTab]
     ];
   } else if (lgAndDown) {
     return [
       [allTabs.inputTab],
-      [allTabs.thresholdTab, allTabs.contoursTab, allTabs.apriltagTab, allTabs.arucoTab, allTabs.outputTab],
+      [
+        allTabs.thresholdTab,
+        allTabs.contoursTab,
+        allTabs.apriltagTab,
+        allTabs.arucoTab,
+        allTabs.dnnTab,
+        allTabs.outputTab
+      ],
       [allTabs.targetsTab, allTabs.pnpTab, allTabs.map3dTab]
     ];
   } else if (xl) {
     return [
       [allTabs.inputTab],
       [allTabs.thresholdTab],
-      [allTabs.contoursTab, allTabs.apriltagTab, allTabs.arucoTab, allTabs.outputTab],
+      [allTabs.contoursTab, allTabs.apriltagTab, allTabs.arucoTab, allTabs.dnnTab, allTabs.outputTab],
       [allTabs.targetsTab, allTabs.pnpTab, allTabs.map3dTab]
     ];
   }
@@ -103,16 +116,18 @@ const tabGroups = computed<ConfigOption[][]>(() => {
   const allow3d = useCameraSettingsStore().currentPipelineSettings.solvePNPEnabled;
   const isAprilTag = useCameraSettingsStore().currentWebsocketPipelineType === WebsocketPipelineType.AprilTag;
   const isAruco = useCameraSettingsStore().currentWebsocketPipelineType === WebsocketPipelineType.Aruco;
+  const isDNN = useCameraSettingsStore().currentWebsocketPipelineType === WebsocketPipelineType.Dnn;
 
   return getTabGroups()
     .map((tabGroup) =>
       tabGroup.filter(
         (tabConfig) =>
           !(!allow3d && tabConfig.tabName === "3D") && //Filter out 3D tab any time 3D isn't calibrated
           !((!allow3d || isAprilTag || isAruco) && tabConfig.tabName === "PnP") && //Filter out the PnP config tab if 3D isn't available, or we're doing AprilTags
-          !((isAprilTag || isAruco) && tabConfig.tabName === "Threshold") && //Filter out threshold tab if we're doing AprilTags
-          !((isAprilTag || isAruco) && tabConfig.tabName === "Contours") && //Filter out contours if we're doing AprilTags
+          !((isAprilTag || isAruco || isDNN) && tabConfig.tabName === "Threshold") && //Filter out threshold tab if we're doing AprilTags
+          !((isAprilTag || isAruco || isDNN) && tabConfig.tabName === "Contours") && //Filter out contours if we're doing AprilTags
           !(!isAprilTag && tabConfig.tabName === "AprilTag") && //Filter out apriltag unless we actually are doing AprilTags
+          !(!isDNN && tabConfig.tabName === "DNN") && //Filter out dnn unless we actually are doing DNN things
           !(!isAruco && tabConfig.tabName === "Aruco") //Filter out aruco unless we actually are doing Aruco
       )
     )

photon-client/src/components/dashboard/tabs/DnnTab.vue

@@ -0,0 +1,48 @@
+<script setup lang="ts">
+import { useCameraSettingsStore } from "@/stores/settings/CameraSettingsStore";
+import { PipelineType } from "@/types/PipelineTypes";
+import PvSelect from "@/components/common/pv-select.vue";
+import PvSlider from "@/components/common/pv-slider.vue";
+import PvSwitch from "@/components/common/pv-switch.vue";
+import { computed, getCurrentInstance } from "vue";
+import { useStateStore } from "@/stores/StateStore";
+
+// TODO fix pipeline typing in order to fix this, the store settings call should be able to infer that only valid pipeline type settings are exposed based on pre-checks for the entire config section
+// Defer reference to store access method
+const currentPipelineSettings = useCameraSettingsStore().currentPipelineSettings;
+
+const interactiveCols = computed(
+  () =>
+    (getCurrentInstance()?.proxy.$vuetify.breakpoint.mdAndDown || false) &&
+    (!useStateStore().sidebarFolded || useCameraSettingsStore().isDriverMode)
+)
+  ? 9
+  : 8;
+
+const models = () => {
+  return ["a", "bcd"];
+};
+</script>
+
+<template>
+  <div v-if="currentPipelineSettings.pipelineType === PipelineType.Dnn">
+    <pv-select
+      v-model="currentPipelineSettings.modelIndex"
+      label="Model Index"
+      :items="models"
+      :select-cols="interactiveCols"
+      @input="(value) => useCameraSettingsStore().changeCurrentPipelineSetting({ tagFamily: value }, false)"
+    />
+    <pv-slider
+      v-model="currentPipelineSettings.confidence"
+      class="pt-2"
+      :slider-cols="interactiveCols"
+      label="Confidence"
+      tooltip="asdf"
+      :min="0"
+      :max="1"
+      :step="-1"
+      @input="(value) => useCameraSettingsStore().changeCurrentPipelineSetting({ confidence: value }, false)"
+    />
+  </div>
+</template>

photon-client/src/components/dashboard/tabs/TargetsTab.vue

@@ -48,6 +48,10 @@ const resetCurrentBuffer = () => {
               >
                 Fiducial ID
               </th>
+              <template v-if="currentPipelineSettings.pipelineType === PipelineType.Dnn">
+                <th class="text-center white--text">Class ID</th>
+                <th class="text-center white--text">Confidence</th>
+              </template>
               <template v-if="!useCameraSettingsStore().currentPipelineSettings.solvePNPEnabled">
                 <th class="text-center white--text">Pitch &theta;&deg;</th>
                 <th class="text-center white--text">Yaw &theta;&deg;</th>
@@ -85,6 +89,12 @@ const resetCurrentBuffer = () => {
               >
                 {{ target.fiducialId }}
               </td>
+              <td v-if="currentPipelineSettings.pipelineType === PipelineType.Dnn" class="text-center white--text">
+                {{ target.classId }}
+              </td>
+              <td v-if="currentPipelineSettings.pipelineType === PipelineType.Dnn" class="text-center white--text">
+                {{ target.confidence.toFixed(2) }}
+              </td>
               <template v-if="!useCameraSettingsStore().currentPipelineSettings.solvePNPEnabled">
                 <td class="text-center">{{ target.pitch.toFixed(2) }}&deg;</td>
                 <td class="text-center">{{ target.yaw.toFixed(2) }}&deg;</td>

photon-client/src/types/PhotonTrackingTypes.ts

@@ -56,6 +56,9 @@ export interface PhotonTarget {
   fiducialId: number;
   // undefined if 3d isn't enabled
   pose?: Transform3d;
+
+  confidence: number;
+  classId: number;
 }
 
 export interface MultitagResult {

photon-client/src/types/PipelineTypes.ts

@@ -5,7 +5,8 @@ export enum PipelineType {
   Reflective = 2,
   ColoredShape = 3,
   AprilTag = 4,
-  Aruco = 5
+  Aruco = 5,
+  Dnn = 6
 }
 
 export enum AprilTagFamily {
@@ -237,6 +238,28 @@ export const DefaultAprilTagPipelineSettings: AprilTagPipelineSettings = {
   doSingleTargetAlways: false
 };
 
+export interface DnnPipelineSettings extends PipelineSettings {
+  pipelineType: PipelineType.Dnn;
+  modelIndex: number;
+  confidence: number;
+}
+export type ConfigurableDnnPipelineSettings = Partial<Omit<DnnPipelineSettings, "pipelineType">> &
+  ConfigurablePipelineSettings;
+export const DefaultDnnPipelineSettings: DnnPipelineSettings = {
+  ...DefaultPipelineSettings,
+  // Need these for boilerplate
+  cameraGain: 75,
+  outputShowMultipleTargets: true,
+  targetModel: TargetModel.AprilTag6in_16h5,
+  cameraExposure: -1,
+  cameraAutoExposure: true,
+  ledMode: false,
+  pipelineType: PipelineType.Dnn,
+  // actual stuff
+  modelIndex: -1,
+  confidence: 0.3
+};
+
 export interface ArucoPipelineSettings extends PipelineSettings {
   pipelineType: PipelineType.Aruco;
 
@@ -285,10 +308,12 @@ export type ActivePipelineSettings =
   | ReflectivePipelineSettings
   | ColoredShapePipelineSettings
   | AprilTagPipelineSettings
-  | ArucoPipelineSettings;
+  | ArucoPipelineSettings
+  | DnnPipelineSettings;
 
 export type ActiveConfigurablePipelineSettings =
   | ConfigurableReflectivePipelineSettings
   | ConfigurableColoredShapePipelineSettings
   | ConfigurableAprilTagPipelineSettings
-  | ConfigurableArucoPipelineSettings;
+  | ConfigurableArucoPipelineSettings
+  | ConfigurableDnnPipelineSettings;

photon-client/src/types/WebsocketDataTypes.ts

@@ -101,5 +101,6 @@ export enum WebsocketPipelineType {
   Reflective = 0,
   ColoredShape = 1,
   AprilTag = 2,
-  Aruco = 3
+  Aruco = 3,
+  Dnn = 4
 }

photon-core/src/main/java/org/photonvision/vision/opencv/PhotonNet.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright (C) Photon Vision.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.photonvision.vision.opencv;
+
+import org.opencv.dnn.Net;
+
+// Hack so we can see the delete function
+public class PhotonNet extends Net implements Releasable {
+    private Net net;
+
+    public PhotonNet(Net net) {
+        super(net.getNativeObjAddr());
+        // And keep net around so the GC doesn't try to eat it
+        this.net = net;
+    }
+
+    @Override
+    public void release() {
+        // This relies on opencv calling their private delete from finalize
+        try {
+            finalize();
+        } catch (Throwable e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        }
+    }
+}

photon-core/src/main/java/org/photonvision/vision/pipe/CVPipe.java

@@ -33,6 +33,10 @@ public void setParams(P params) {
         this.params = params;
     }
 
+    public P getParams() {
+        return this.params;
+    }
+
     /**
      * Runs the process for the pipe.
      *

photon-core/src/main/java/org/photonvision/vision/pipe/impl/NeuralNetworkPipeResult.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright (C) Photon Vision.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.photonvision.vision.pipe.impl;
+
+import org.opencv.core.Rect2d;
+
+public class NeuralNetworkPipeResult {
+    public NeuralNetworkPipeResult(Rect2d box2, Integer integer, Float float1) {
+        box = box2;
+        classIdx = integer;
+        confidence = float1;
+    }
+
+    public final int classIdx;
+    public final Rect2d box;
+    public final double confidence;
+}