Fix draw 2d for dnn

photon-client/src/components/dashboard/ConfigOptions.vue

@@ -120,6 +120,7 @@ const tabGroups = computed<ConfigOption[][]>(() => {
           !((isAprilTag || isAruco || isDNN) && tabConfig.tabName === "Threshold") && //Filter out threshold tab if we're doing AprilTags
           !((isAprilTag || isAruco || isDNN) && tabConfig.tabName === "Contours") && //Filter out contours if we're doing AprilTags
           !(!isAprilTag && tabConfig.tabName === "AprilTag") && //Filter out apriltag unless we actually are doing AprilTags
+          !(!isDNN && tabConfig.tabName === "DNN") && //Filter out dnn unless we actually are doing DNN things
           !(!isAruco && tabConfig.tabName === "Aruco") //Filter out aruco unless we actually are doing Aruco
       )
     )

photon-client/src/components/dashboard/tabs/DnnTab.vue

@@ -28,8 +28,8 @@ const models = () => {
   <div v-if="currentPipelineSettings.pipelineType === PipelineType.Dnn">
     <pv-select
       v-model="currentPipelineSettings.modelIndex"
-      label="Target family"
-      :items = models
+      label="Model Index"
+      :items = "models"
       :select-cols="interactiveCols"
       @input="(value) => useCameraSettingsStore().changeCurrentPipelineSetting({ tagFamily: value }, false)"
     />
@@ -42,7 +42,7 @@ const models = () => {
       :min="0"
       :max="1"
       :step="-1"
-      @input="(value) => useCameraSettingsStore().changeCurrentPipelineSetting({ decimate: value }, false)"
+      @input="(value) => useCameraSettingsStore().changeCurrentPipelineSetting({ confidence: value }, false)"
     />
   </div>
 </template>

photon-client/src/components/dashboard/tabs/TargetsTab.vue

@@ -48,6 +48,12 @@ const resetCurrentBuffer = () => {
               >
                 Fiducial ID
               </th>
+                <template
+                v-if="currentPipelineSettings.pipelineType === PipelineType.Dnn" 
+                >
+                <th class="text-center white--text">Class ID</th>
+                <th class="text-center white--text">Confidence</th>
+              </template>
               <template v-if="!useCameraSettingsStore().currentPipelineSettings.solvePNPEnabled">
                 <th class="text-center white--text">Pitch &theta;&deg;</th>
                 <th class="text-center white--text">Yaw &theta;&deg;</th>
@@ -85,6 +91,8 @@ const resetCurrentBuffer = () => {
               >
                 {{ target.fiducialId }}
               </td>
+              <td v-if="currentPipelineSettings.pipelineType === PipelineType.Dnn" class="text-center white--text"> {{ target.classId }} </td>
+              <td v-if="currentPipelineSettings.pipelineType === PipelineType.Dnn" class="text-center white--text"> {{ target.confidence.toFixed(2) }} </td>
               <template v-if="!useCameraSettingsStore().currentPipelineSettings.solvePNPEnabled">
                 <td class="text-center">{{ target.pitch.toFixed(2) }}&deg;</td>
                 <td class="text-center">{{ target.yaw.toFixed(2) }}&deg;</td>

photon-client/src/types/PhotonTrackingTypes.ts

@@ -55,7 +55,10 @@ export interface PhotonTarget {
   // -1 if not set
   fiducialId: number;
   // undefined if 3d isn't enabled
-  pose?: Transform3d;
+  pose?: Transform3d,
+
+  confidence: number,
+  classId: number;
 }
 
 export interface MultitagResult {

photon-client/src/types/PipelineTypes.ts

@@ -240,12 +240,24 @@ export const DefaultAprilTagPipelineSettings: AprilTagPipelineSettings = {
 
 export interface DnnPipelineSettings extends PipelineSettings {
   pipelineType: PipelineType.Dnn;
+  modelIndex: number
+  confidence: number
 }
 export type ConfigurableDnnPipelineSettings = Partial<Omit<DnnPipelineSettings, "pipelineType">> &
   ConfigurablePipelineSettings;
 export const DefaultDnnPipelineSettings: DnnPipelineSettings = {
   ...DefaultPipelineSettings,
-  pipelineType: PipelineType.Dnn
+  // Need these for boilerplate
+  cameraGain: 75,
+  outputShowMultipleTargets: true,
+  targetModel: TargetModel.AprilTag6in_16h5,
+  cameraExposure: -1,
+  cameraAutoExposure: true,
+  ledMode: false,
+  pipelineType: PipelineType.Dnn,
+  // actual stuff
+  modelIndex: -1,
+  confidence: 0.3
 };
 
 export interface ArucoPipelineSettings extends PipelineSettings {

photon-core/src/main/java/org/photonvision/vision/opencv/PhotonNet.java

@@ -21,8 +21,12 @@
 
 // Hack so we can see the delete function
 public class PhotonNet extends Net implements Releasable {
-    public PhotonNet(long nativeObjAddr) {
-        super(nativeObjAddr);
+    private Net net;
+
+    public PhotonNet(Net net) {
+        super(net.getNativeObjAddr());
+        // And keep net around so the GC doesn't try to eat it
+        this.net = net;
     }
 
     @Override

photon-core/src/main/java/org/photonvision/vision/pipe/impl/OpencvDnnPipe.java

@@ -111,13 +111,14 @@ protected List<NeuralNetworkPipeResult> process(CVMat in) {
 
     public OpencvDnnPipe() {
         super();
-        this.params = new OpencvDnnPipeParams(null);
+        this.params = new OpencvDnnPipeParams(null, 0.5f);
     }
 
     @Override
     public void setParams(OpencvDnnPipeParams newParams) {
         // Avoid extra reset if we don't have to
         if (params.modelPath != null && params.modelPath.equals(newParams.modelPath)) {
+            super.setParams(newParams);
             return;
         }
         super.setParams(newParams);
@@ -135,8 +136,8 @@ public void setParams(OpencvDnnPipeParams newParams) {
                     Dnn.readNetFromDarknet(
                             "/home/matt/Downloads/yolov4-csp-swish.cfg",
                             "/home/matt/Downloads/yolov4-csp-swish.weights");
-            this.net = new PhotonNet(net.getNativeObjAddr());
-            Core.setNumThreads(1);
+            this.net = new PhotonNet(net);
+            Core.setNumThreads(6);
         } catch (Exception e) {
             System.out.println(e);
         }
@@ -256,8 +257,9 @@ public static class OpencvDnnPipeParams {
 
         public float confidence;
 
-        public OpencvDnnPipeParams(String modelPath) {
+        public OpencvDnnPipeParams(String modelPath, float confidence) {
             this.modelPath = modelPath;
+            this.confidence = confidence;
         }
     }
 }

photon-core/src/main/java/org/photonvision/vision/pipeline/DnnPipelineSettings.java

@@ -18,8 +18,9 @@
 package org.photonvision.vision.pipeline;
 
 public class DnnPipelineSettings extends AdvancedPipelineSettings {
-    String modelPath;
-    double confidence;
+    public String modelPath;
+    public double confidence;
+    public int modelIndex;
 
     public DnnPipelineSettings() {
         super();
@@ -33,5 +34,7 @@ public DnnPipelineSettings() {
 
         this.modelPath = "some_path";
         this.confidence = 0.3;
+
+        this.modelIndex = -1;
     }
 }

photon-core/src/main/java/org/photonvision/vision/pipeline/OpencvDnnPipeline.java

@@ -50,7 +50,7 @@ public OpencvDnnPipeline(DnnPipelineSettings settings) {
 
     @Override
     protected void setPipeParamsImpl() {
-        dnnPipe.setParams(new OpencvDnnPipeParams(settings.modelPath));
+        dnnPipe.setParams(new OpencvDnnPipeParams(settings.modelPath, (float)settings.confidence));
     }
 
     @Override
@@ -70,26 +70,29 @@ protected CVPipelineResult process(Frame input_frame, DnnPipelineSettings settin
         List<TrackedTarget> targets = new ArrayList<>();
 
         // This belongs in a collect & draw pipe but I'm lazy
+        int i = 0;
         for (var t : targetList) {
-            Imgproc.rectangle(
-                    input_frame.processedImage.getMat(), t.box.tl(), t.box.br(), new Scalar(0, 0, 255), 2);
-
-            var name = String.format("%s (%f)", names.get(t.classIdx), t.confidence);
-
-            Imgproc.putText(
-                    input_frame.processedImage.getMat(),
-                    name,
-                    new Point(t.box.x + t.box.width / 2.0, t.box.y + t.box.height / 2.0),
-                    0,
-                    0.6,
-                    ColorHelper.colorToScalar(java.awt.Color.white),
-                    2);
+            // Imgproc.rectangle(
+            //         input_frame.processedImage.getMat(), t.box.tl(), t.box.br(), new Scalar(0, 0, 255), 2);
+
+            // Let the draw pipeline deal with this all for us
+            // var name = String.format("%d (%f)", i, t.confidence);
+            // Imgproc.putText(
+            //         input_frame.processedImage.getMat(),
+            //         name,
+            //         new Point(t.box.x + t.box.width / 2.0, t.box.y + t.box.height / 2.0),
+            //         0,
+            //         0.6,
+            //         ColorHelper.colorToScalar(java.awt.Color.white),
+            //         2);
 
             targets.add(
                     new TrackedTarget(
                             t,
                             new TargetCalculationParameters(
                                     false, null, null, null, null, frameStaticProperties)));
+
+            i++;
         }
 
         var fpsResult = calculateFPSPipe.run(null);

photon-core/src/main/java/org/photonvision/vision/target/TrackedTarget.java

@@ -67,6 +67,10 @@ public class TrackedTarget implements Releasable {
 
     private Mat m_cameraRelativeTvec, m_cameraRelativeRvec;
 
+    // ML specific params
+    private int m_classId = -1;
+    private double m_confidence = -1;
+
     public TrackedTarget(
             PotentialTarget origTarget, TargetCalculationParameters params, CVShape shape) {
         this.m_mainContour = origTarget.m_mainContour;
@@ -229,16 +233,19 @@ public TrackedTarget(
 
     public TrackedTarget(
             Rect2d box, int class_id, double confidence, TargetCalculationParameters params) {
-        m_targetOffsetPoint = new Point(box.x, box.y);
+        var centerX = box.x + box.width / 2.0;
+        var centerY = box.y + box.height / 2.0;
+
+        m_targetOffsetPoint = new Point(centerX, centerY);
         m_robotOffsetPoint = new Point();
 
         var yawPitch =
                 TargetCalculations.calculateYawPitch(
                         params.cameraCenterPoint.x,
-                        box.x + box.width / 2.0,
+                        centerX,
                         params.horizontalFocalLength,
                         params.cameraCenterPoint.y,
-                        box.y + box.height / 2.0,
+                        centerY,
                         params.verticalFocalLength);
         m_yaw = yawPitch.getFirst();
         m_pitch = yawPitch.getSecond();
@@ -259,7 +266,8 @@ public TrackedTarget(
         m_mainContour = new Contour(contourMat);
         m_area = m_mainContour.getArea() / params.imageArea * 100;
 
-        m_fiducialId = class_id;
+        m_classId = class_id;
+        m_confidence = confidence;
     }
 
     public TrackedTarget(
@@ -436,6 +444,11 @@ public HashMap<String, Object> toHashMap() {
             ret.put("pose", SerializationUtils.transformToHashMap(bestCameraToTarget3d));
         }
         ret.put("fiducialId", getFiducialId());
+
+        ret.put("confidence", m_confidence);
+        ret.put("classId", m_classId);
+
+
         return ret;
     }