hand tracking

common/Utilities/FpsCounter.cs

@@ -0,0 +1,31 @@
+using System;
+using System.Collections.Generic;
+
+namespace Sst.Utilities;
+
+public class FpsCounter {
+  public TimeSpan WindowSize;
+
+  private Queue<DateTime> _times = new();
+
+  public FpsCounter(TimeSpan? windowSize = null) {
+    WindowSize = windowSize ?? TimeSpan.FromSeconds(0.2f);
+  }
+
+  public void OnFrame() {
+    RemoveOutOfWindowTimes();
+    _times.Enqueue(DateTime.Now);
+  }
+
+  public double Read() {
+    RemoveOutOfWindowTimes();
+    return (double)_times.Count / WindowSize.TotalSeconds;
+  }
+
+  private void RemoveOutOfWindowTimes() {
+    var windowStart = DateTime.Now - WindowSize;
+    while (_times.Count > 0 && _times.Peek() < windowStart) {
+      _times.Dequeue();
+    }
+  }
+}

projects/Bonelab/ColliderScope/Project.csproj

@@ -10,7 +10,7 @@
   </Target>
 
   <PropertyGroup>
-    <!-- <CopyIntoGameAfterBuild>true</CopyIntoGameAfterBuild> -->
+    <CopyIntoGameAfterBuild>true</CopyIntoGameAfterBuild>
 
     <ProjectGuid>{EAE1410F-B5CF-47D6-8764-2FCAEE822C9D}</ProjectGuid>
   </PropertyGroup>

projects/Bonelab/ColliderScope/src/Mod.cs

@@ -59,14 +59,18 @@ public override void OnInitializeMelon() {
         "Hides head colliders so that they do not obscure your vision");
     _prefOnlyResizeRigColliders = category.CreateEntry(
         "onlyResizeRigColliders", true, "Only resize rig colliders",
-        "Improves performance by watching for changes to collider size on only the rig");
+        "Improves performance by watching for changes to collider size on " +
+            "only the rig");
     // TODO: Change to frame time allocated for colliders
     _prefIterationsPerFrame = category.CreateEntry(
         "iterationsPerFrame", 8f, "Iterations per frame",
-        "Number of game objects to show colliders of per frame on load (higher loads faster but too high lags and crashes the game)");
+        "Number of game objects to show colliders of per frame on load " +
+            "(higher loads faster but too high lags and crashes the game)");
     _prefBackgroundIterationsPerFrame = category.CreateEntry(
         "backgroundIterationsPerFrame", 2f, "Background iterations per frame",
-        "Number of game objects to show colliders of per frame in the background (runs continuously to catch any objects added during play)");
+        "Number of game objects to show colliders of per frame in the " +
+            "background (runs continuously to catch any objects added during " +
+            "play)");
 
     LevelHooks.OnLoad += nextLevel => ResetState();
     LevelHooks.OnLevelStart += level => Visualize(false);
@@ -80,6 +84,7 @@ public override void OnUpdate() {
     if (LevelHooks.IsLoading)
       return;
 
+// TODO: Replace with a check for third person camera
 #if DEBUG
     if (LevelHooks.RigManager?.ControllerRig.rightController
             .GetThumbStickDown() ??

projects/Bonelab/HandTracking/Project.csproj

@@ -16,7 +16,15 @@
     <Compile Include="src/**/*.cs" />
     <Compile Include="../../../common/Utilities/Metadata.cs" />
     <Compile Include="../../../common/Utilities/Dbg.cs" />
+    <Compile Include="../../../common/Utilities/Geometry.cs" />
+    <Compile Include="../../../common/Utilities/FpsCounter.cs" />
     <Compile Include="../../../common/Bonelab/LevelHooks.cs" />
+
+    <Compile Include="../../../common/Bonelab/Levels.cs" />
+    <Compile Include="../../../common/Utilities/Colliders.cs" />
+    <Compile Include="../../../common/Utilities/Shaders.cs" />
+    <Compile Include="../../../common/Utilities/Geometry.cs" />
+    <Compile Include="../../../common/Utilities/Unity.cs" />
   </ItemGroup>
 
 </Project>

projects/Bonelab/HandTracking/README.md

@@ -1 +1,41 @@
 Adds support for hand tracking.
+
+## Installation
+
+- This mod is for the **patch 4 Quest standalone** version of the game
+- Install **Melon Loader 0.5.7** via Lemon Loader
+- Add the hand tracking permission to the game APK using Quest Patcher
+  - **IMPORTANT: This will RESET YOUR GAME and you will LOSE YOUR SAVE AND MODS so back them up first**
+  - Go to the "Tools & Options" tab -> click the "Change App" button -> select `com.StressLevelZero.BONELAB`
+  - Go to the "Patching" tab -> **select `None` for the mod loader** (you should have already installed Melon Loader)
+  - Click "Patching Options" -> **scroll to "Hand Tracking Type" -> select `V2`**
+  - Click the button to patch the game
+- Copy this mod to `/sdcard/Android/data/com.StressLevelZero.BONELAB/files/Mods/*.dll`
+- Make sure hand tracking is enabled in your Quest settings
+  - You should know by putting your controllers down while in the Quest menu and it switches to hands within a few seconds
+- Start the game
+
+## Usage
+
+- Click on UIs by pinching
+- Open the in-game menu by doing the standard Quest hand tracking menu gesture (left hand open, palm towards your face then touch the tips of your index finger and thumb)
+- Walk by moving your hands up and down alternately in a running motion
+- Grab by curling middle, ring and pinky fingers into a fist (same effect as pressing trigger and grip on controller)
+- Force pull objects by forming a fist with these three fingers then flicking your wrist (in any direction)
+- Held items like guns are triggered by curling the index finger
+
+## Tips
+
+Hand tracking's accuracy is _very_ limited. Don't expect everything to work perfectly. I did put in effort to make it less frustrating by taking into account the tracking confidence and assuming certain things when the hands are out of view, but it won't always do what you want if the headset can't see your hands/fingers. Here are some tips to help make your experience smoother:
+
+- When running, make sure the headset has a clear view of your hands by either:
+  - Holding your hands a bit higher and further forwards than normal (so they are not too close to the headset)
+  - Looking downwards
+- When running, instead of pointing your hands forwards in a fist, open them and have your palms facing you
+- Don't hold guns with two hands while aiming (headset gets confused and will make your in-game hands do weird things because your IRL hands are overlapping)
+- Don't hold guns too close to your face while aiming (hand will lose tracking if too close to headset)
+- While throwing things, climbing or any other action, try not to bring your hands too close to the headset or out of your eye sight
+
+## Fun facts
+
+SLZ has already added a bunch of hand tracking code. By just adding the hand tracking permission to the APK, `MarrowGame.xr.HandLeft/Right` will start tracking the hand position! Finger poses are not updated because the code is missing from the `OculusHandActionMap`, however they do actually have finger pose and gesture code in the generic `HandActionMap.ProcessesHand` method. I tried to lean on their work and get it working but there were too many gaps and it ended being easier to reimplement everything myself.

projects/Bonelab/HandTracking/build-and-start.sh

@@ -0,0 +1,22 @@
+set -eux
+cd "$(dirname $0)"
+
+dotnet build ./Project.csproj
+
+# TIP: Enable wireless mode from SideQuest
+adb connect 192.168.0.69
+
+FILE=HandTracking.P4.ML5.dll
+adb shell am force-stop com.StressLevelZero.BONELAB
+adb push "./bin/Debug/Patch4_MelonLoader0.5/$FILE" /sdcard/Download/
+adb shell mv "/sdcard/Download/$FILE" /sdcard/Android/data/com.StressLevelZero.BONELAB/files/mods/
+adb shell chmod 644 "/sdcard/Android/data/com.StressLevelZero.BONELAB/files/mods/$FILE"
+adb shell am start -n com.StressLevelZero.BONELAB/com.unity3d.player.UnityPlayerActivity
+
+# adb shell uiautomator dump /sdcard/ui_dump.xml && adb pull /sdcard/ui_dump.xml .
+
+# adb shell ls /sdcard/Android/data/com.StressLevelZero.BONELAB/files/melonloader
+# adb pull /sdcard/Android/data/com.StressLevelZero.BONELAB/files/melonloader/etc/managed/Assembly-CSharp.dll
+# adb pull /sdcard/Android/data/com.StressLevelZero.BONELAB/files/melonloader/etc/managed/SLZ.Marrow.dll
+
+# adb logcat -v time MelonLoader:D CRASH:D Mono:W mono:D mono-rt:D Zygote:D A64_HOOK:V DEBUG:D Binder:D AndroidRuntime:D "*:S"

projects/Bonelab/HandTracking/src/ForcePull.cs

@@ -0,0 +1,138 @@
+using System;
+using System.Collections.Generic;
+using MelonLoader;
+using UnityEngine;
+using SLZ.Interaction;
+using Sst.Utilities;
+
+namespace Sst.HandTracking;
+
+public class ForcePull {
+  private const float FLICK_MAX_DURATION_SECONDS = 0.25f;
+  private const float FLICK_MIN_ROTATION_DEGREES = 40f;
+
+  public HandTracker Tracker;
+
+  private LinkedList<StartingState> _startingStates = new();
+  private ForcePullGrip _grip;
+
+  public bool IsPulling() => _grip?._pullToHand == Tracker.GetPhysicalHand();
+
+  // Run after updating Tracker.IsGripping
+  public void Update() {
+    var hand = Tracker.GetPhysicalHand();
+    if (hand == null)
+      return;
+
+    var hoveringGrip =
+        hand.farHoveringReciever?.Host.TryCast<InteractableHost>()
+            ?.GetForcePullGrip();
+    if (hoveringGrip) {
+      var angle = (int)GetRotationDifference(
+          new Dictionary<ForcePullGrip, float>(), hoveringGrip);
+      Tracker.Log($"Force pull angle = {angle}");
+    }
+
+    //   LogSpam(
+    //       $"isFist={Tracker.IsGripping}, isfp={!!_grip},
+    //       curgo={!!hand.m_CurrentAttachedGO},
+    //       attrec={!!hand.AttachedReceiver},
+    //       farhov={!!hand.farHoveringReciever},
+    //       prim={hand.Controller._primaryInteractionButton},
+    //       sec={hand.Controller._secondaryInteractionButton},
+    //       grip={Tracker.ProxyController.GripButton}");
+
+    if (_grip?._pullToHand == hand) {
+      if (!Tracker.IsGripping) {
+        Tracker.ProxyController.GripButton = false;
+        _grip.CancelPull(hand);
+        _grip = null;
+        Tracker.Log("Cancelled force pull due to ungripping");
+      }
+      return;
+    }
+
+    if (_grip) {
+      if (hand.m_CurrentAttachedGO) {
+        Tracker.Log("Force pull complete");
+      } else {
+        Tracker.Log("Cancelling force pull due to hand not being set");
+      }
+      Tracker.ProxyController.GripButton = false;
+      _grip = null;
+    }
+
+    var isAlreadyHolding = hand.AttachedReceiver || hand.m_CurrentAttachedGO;
+    if (!Tracker.IsGripping || isAlreadyHolding)
+      return;
+
+    var earliest = Time.time - FLICK_MAX_DURATION_SECONDS;
+    while (_startingStates.Count > 0 &&
+           _startingStates.First.Value.time < earliest) {
+      _startingStates.RemoveFirst();
+    }
+
+    var rotationCache = new Dictionary<ForcePullGrip, float>();
+    var node = _startingStates.Last;
+    while (node != null) {
+      var handAngleDiff = Quaternion.Angle(node.Value.rotation,
+                                           Tracker.ProxyController.Rotation);
+      var angleFromObject =
+          GetRotationDifference(rotationCache, node.Value.grip);
+      if (handAngleDiff >= FLICK_MIN_ROTATION_DEGREES &&
+          angleFromObject > node.Value.angleFromObject) {
+        // Need to set these or the force pull will instantly cancel
+        hand.farHoveringReciever = node.Value.reciever;
+        hand.Controller._primaryInteractionButton = true;
+        hand.Controller._secondaryInteractionButton = true;
+
+        _grip = node.Value.grip;
+        _grip._pullToHand = hand;
+        Utils.ForcePullGrip_Pull.Call(_grip, hand);
+        Tracker.Log(
+            "Force pulling, handAngleDiff:", handAngleDiff.ToString("0.0f"),
+            "angleFromObject:", angleFromObject.ToString("0.0f"),
+            "fphand:", _grip._pullToHand,
+            "attached:", hand.AttachedReceiver?.name);
+        break;
+      }
+      node = node.Previous;
+    }
+
+    var hoveringForcePullGrip =
+        hand.farHoveringReciever?.Host.TryCast<InteractableHost>()
+            ?.GetForcePullGrip();
+    if (hoveringForcePullGrip) {
+      _startingStates.AddLast(new StartingState() {
+        time = Time.time,
+        grip = hoveringForcePullGrip,
+        reciever = hand.farHoveringReciever,
+        rotation = Tracker.ProxyController.Rotation,
+        angleFromObject =
+            GetRotationDifference(rotationCache, hoveringForcePullGrip),
+      });
+    }
+  }
+
+  private float GetRotationDifference(Dictionary<ForcePullGrip, float> cache,
+                                      ForcePullGrip grip) {
+    if (cache.TryGetValue(grip, out var cached))
+      return cached;
+    // TODO: Get IRL hand position/rotation in game world (not physics rig hand)
+    // TODO: Is this it? (verify on pc)
+    var controller = Utils.RigControllerOf(Tracker).transform;
+    var direction = grip.transform.position - controller.position;
+    var target = Quaternion.LookRotation(direction);
+    var diff = Quaternion.Angle(target, controller.rotation);
+    cache.Add(grip, diff);
+    return diff;
+  }
+
+  private struct StartingState {
+    public float time;
+    public ForcePullGrip grip;
+    public HandReciever reciever;
+    public Quaternion rotation;
+    public float angleFromObject;
+  }
+}

projects/Bonelab/HandTracking/src/HandState.cs

@@ -0,0 +1,103 @@
+using System;
+using MelonLoader;
+using UnityEngine;
+
+namespace Sst.HandTracking;
+
+public class HandState {
+  private static Vector3 FromFlippedXVector3f(OVRPlugin.Vector3f vector) =>
+      new Vector3(-vector.x, vector.y, vector.z);
+
+  private static Vector3 FromFlippedZVector3f(OVRPlugin.Vector3f vector) =>
+      new Vector3(vector.x, vector.y, -vector.z);
+
+  private static Quaternion FromFlippedXQuatf(OVRPlugin.Quatf quat) =>
+      new Quaternion(quat.x, -quat.y, -quat.z, quat.w);
+
+  private static Quaternion FromFlippedZQuatf(OVRPlugin.Quatf quat) =>
+      new Quaternion(-quat.x, -quat.y, quat.z, quat.w);
+
+  public bool IsLeft;
+  public Vector3 Position;
+  public Quaternion Rotation;
+  public float Scale;
+  public JointTransform[] Joints =
+      new JointTransform[(int)OVRPlugin.SkeletonConstants.MaxHandBones];
+  public OVRPlugin.TrackingConfidence HandConfidence;
+  public OVRPlugin.TrackingConfidence[] FingerConfidences;
+  public bool IsPinching;
+
+  private OVRPlugin.Hand _hand;
+  private OVRPlugin.SkeletonType _skeletonType;
+  private OVRInput.Controller _controller;
+  private OVRPlugin.HandState _state = new();
+  private OVRPlugin.Skeleton2 _skeleton = new();
+
+  public HandState(bool isLeft) {
+    IsLeft = isLeft;
+    _controller =
+        isLeft ? OVRInput.Controller.LHand : OVRInput.Controller.RHand;
+
+    _skeletonType = isLeft ? OVRPlugin.SkeletonType.HandLeft
+                           : OVRPlugin.SkeletonType.HandRight;
+    if (!OVRPlugin.GetSkeleton2(_skeletonType, _skeleton)) {
+      throw new Exception("Failed to get hand skeleton");
+    }
+
+    _hand = isLeft ? OVRPlugin.Hand.HandLeft : OVRPlugin.Hand.HandRight;
+    Update();
+  }
+
+  public void Update() {
+    if (!OVRPlugin.GetHandState(OVRPlugin.Step.Render, _hand, _state)) {
+      throw new Exception("Failed to get hand state");
+    }
+
+    Position = FromFlippedZVector3f(_state.RootPose.Position);
+    Rotation = FromFlippedZQuatf(_state.RootPose.Orientation);
+    Scale = _state.HandScale;
+
+    for (var i = 0; i < Joints.Length; i++) {
+      var localRot = FromFlippedXQuatf(_state.BoneRotations[i]);
+      var parentIdx = _skeleton.Bones[i].ParentBoneIndex;
+      var parentJoint =
+          OVRPlugin.IsValidBone((OVRPlugin.BoneId)parentIdx, _skeletonType)
+              // parentIdx will always be less than i so Joints[parentIdx] will
+              // already be updated
+              ? Joints[parentIdx]
+              : JointTransform.IDENTITY;
+      var handRot = parentJoint.HandRotation * localRot;
+      var localPos =
+          handRot * FromFlippedXVector3f(_skeleton.Bones[i].Pose.Position);
+      var handPos = parentJoint.HandPosition + localPos;
+
+      Joints[i] = new JointTransform() {
+        LocalPosition = localPos,
+        LocalRotation = localRot,
+        HandPosition = handPos,
+        HandRotation = handRot,
+      };
+    }
+
+    HandConfidence = _state.HandConfidence;
+    FingerConfidences = _state.FingerConfidences;
+
+    IsPinching = (_state.Pinches & OVRPlugin.HandFingerPinch.Index) != 0;
+  }
+
+  public bool IsActive() =>
+      // NOTE: Requires OVRInput.Update() to be called (game already does this)
+      OVRInput.IsControllerConnected(_controller);
+
+  public bool IsTracked() => (_state.Status &
+                              OVRPlugin.HandStatus.HandTracked) != 0;
+}
+
+public struct JointTransform {
+  public static JointTransform IDENTITY;
+
+  public Vector3 LocalPosition;
+  public Quaternion LocalRotation;
+  public Vector3 HandPosition;
+  public Quaternion HandRotation;
+}