Skip to content

Commit

Permalink
hand tracking progress
Browse files Browse the repository at this point in the history
  • Loading branch information
@jakzo
jakzo committed Jul 27, 2024
1 parent 510c61b commit c5681db
Show file tree
Hide file tree
Showing 8 changed files with 395 additions and 167 deletions.
1 change: 1 addition & 0 deletions projects/Bonelab/HandTracking/Project.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
<Compile Include="../../../common/Utilities/FpsCounter.cs" />
<Compile Include="../../../common/Bonelab/LevelHooks.cs" />
<Compile Include="../../../common/Bonelab/Levels.cs" />
<Compile Include="../../../common/Bonelab/Bonelab.cs" />
</ItemGroup>

</Project>
72 changes: 32 additions & 40 deletions projects/Bonelab/HandTracking/src/ForcePull.cs
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,16 @@
using System;
using System.Collections.Generic;
using MelonLoader;
using UnityEngine;
using SLZ.Interaction;
using Sst.Utilities;
using SLZ.Marrow.Utilities;

namespace Sst.HandTracking;

public class ForcePull {
private const float FLICK_MAX_DURATION_SECONDS = 0.25f;
private const float FLICK_MIN_ROTATION_DEGREES = 40f;
private const float FLICK_MIN_ROTATION_DEGREES = 50f;
private const float FLICK_MAX_DIST = 0.08f;
private const float FLICK_MAX_DIST_SQR = FLICK_MAX_DIST * FLICK_MAX_DIST;

public HandTracker Tracker;

Expand All @@ -24,24 +25,6 @@ public void Update() {
if (hand == null)
return;

var hoveringGrip =
hand.farHoveringReciever?.Host.TryCast<InteractableHost>()
?.GetForcePullGrip();
if (hoveringGrip) {
var angle = (int)GetRotationDifference(
new Dictionary<ForcePullGrip, float>(), hoveringGrip);
Tracker.Log($"Force pull angle = {angle}");
}

// LogSpam(
// $"isFist={Tracker.IsGripping}, isfp={!!_grip},
// curgo={!!hand.m_CurrentAttachedGO},
// attrec={!!hand.AttachedReceiver},
// farhov={!!hand.farHoveringReciever},
// prim={hand.Controller._primaryInteractionButton},
// sec={hand.Controller._secondaryInteractionButton},
// grip={Tracker.ProxyController.GripButton}");

if (_grip?._pullToHand == hand) {
if (!Tracker.IsGripping) {
Tracker.ProxyController.GripButton = false;
Expand Down Expand Up @@ -73,14 +56,28 @@ public void Update() {
}

var rotationCache = new Dictionary<ForcePullGrip, float>();
var rigController = Utils.RigControllerOf(Tracker)?.transform;
if (rigController == null)
return;

var rigRotAngles = rigController.rotation.eulerAngles;
var rigPosRotNoZ = new SimpleTransform() {
position = rigController.position,
rotation = Quaternion.Euler(rigRotAngles.x, rigRotAngles.y, 0f),
};
var localRot = rigController.localEulerAngles;
var localRotXY = Quaternion.Euler(localRot.x, localRot.y, 0f);

var node = _startingStates.Last;
while (node != null) {
var handAngleDiff = Quaternion.Angle(node.Value.rotation,
Tracker.ProxyController.Rotation);
var handRotDiff = Quaternion.Angle(node.Value.localRotXY, localRotXY);
var angleFromObject =
GetRotationDifference(rotationCache, node.Value.grip);
if (handAngleDiff >= FLICK_MIN_ROTATION_DEGREES &&
angleFromObject > node.Value.angleFromObject) {
GetRotationDifference(rotationCache, rigPosRotNoZ, node.Value.grip);
var distSqr =
(rigController.localPosition - node.Value.localPos).sqrMagnitude;
if (handRotDiff >= FLICK_MIN_ROTATION_DEGREES &&
angleFromObject > node.Value.angleFromObject &&
distSqr <= FLICK_MAX_DIST_SQR) {
// Need to set these or the force pull will instantly cancel
hand.farHoveringReciever = node.Value.reciever;
hand.Controller._primaryInteractionButton = true;
Expand All @@ -89,11 +86,6 @@ public void Update() {
_grip = node.Value.grip;
_grip._pullToHand = hand;
Utils.ForcePullGrip_Pull.Call(_grip, hand);
Tracker.Log(
"Force pulling, handAngleDiff:", handAngleDiff.ToString("0.0f"),
"angleFromObject:", angleFromObject.ToString("0.0f"),
"fphand:", _grip._pullToHand,
"attached:", hand.AttachedReceiver?.name);
break;
}
node = node.Previous;
Expand All @@ -107,23 +99,22 @@ public void Update() {
time = Time.time,
grip = hoveringForcePullGrip,
reciever = hand.farHoveringReciever,
rotation = Tracker.ProxyController.Rotation,
angleFromObject =
GetRotationDifference(rotationCache, hoveringForcePullGrip),
localPos = rigController.localPosition,
localRotXY = localRotXY,
angleFromObject = GetRotationDifference(rotationCache, rigPosRotNoZ,
hoveringForcePullGrip),
});
}
}

private float GetRotationDifference(Dictionary<ForcePullGrip, float> cache,
SimpleTransform rigPosRotNoZ,
ForcePullGrip grip) {
if (cache.TryGetValue(grip, out var cached))
return cached;
// TODO: Get IRL hand position/rotation in game world (not physics rig hand)
// TODO: Is this it? (verify on pc)
var controller = Utils.RigControllerOf(Tracker).transform;
var direction = grip.transform.position - controller.position;
var direction = grip.transform.position - rigPosRotNoZ.position;
var target = Quaternion.LookRotation(direction);
var diff = Quaternion.Angle(target, controller.rotation);
var diff = Quaternion.Angle(target, rigPosRotNoZ.rotation);
cache.Add(grip, diff);
return diff;
}
Expand All @@ -132,7 +123,8 @@ private struct StartingState {
public float time;
public ForcePullGrip grip;
public HandReciever reciever;
public Quaternion rotation;
public Vector3 localPos;
public Quaternion localRotXY;
public float angleFromObject;
}
}
86 changes: 46 additions & 40 deletions ...ts/Bonelab/HandTracking/src/Locomotion.cs → ...onelab/HandTracking/src/HandLocomotion.cs
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,15 @@
using System;
using System.Linq;
using MelonLoader;
using UnityEngine;

namespace Sst.HandTracking;

public class LocoState {
public abstract class Locomotion {
public Vector2? Axis;
public abstract void Init(HandTracker tracker);
public abstract void Update();
}

public class HandLocomotion : Locomotion {
private const float CONFIDENCE_BUILD_RATE = 4f;
private const float CONFIDENCE_DRAIN_RATE = 1f;
private const float VELOCITY_MIN = 1f;
Expand All @@ -19,13 +23,12 @@ public class LocoState {
private const float HEIGHT_MAX = 0.5f;
private const float HEIGHT_FACTOR = 1f / (HEIGHT_MAX - HEIGHT_MIN);

public Vector2? Axis;

private LocoHandState _left;
private LocoHandState _right;
private float _lastCorrespondence = 0f;
private float _confidence = 0f;

public void Init(HandTracker tracker) {
public override void Init(HandTracker tracker) {
var state = new LocoHandState() { Tracker = tracker };
if (tracker.Opts.isLeft) {
_left = state;
Expand All @@ -35,17 +38,17 @@ public void Init(HandTracker tracker) {
tracker.LocoState = this;
}

public void Update() {
// Apparently the game just swaps the MarrowGame.xr controllers when the
// left handed setting is enabled so the left tracker always has the
// locomotion stick
var locoTracker = _left.Tracker;
public override void Update() {
var locoTracker =
Utils.IsLocoControllerLeft() ? _left.Tracker : _right.Tracker;
if (locoTracker.IsControllerConnected() || _left == null ||
_right == null) {
Axis = null;
return;
}

var isConfident = _left.IsTrackedConfident && _right.IsTrackedConfident;

_left.Update();
_right.Update();

Expand All @@ -55,54 +58,55 @@ public void Update() {
: (_right, _left);
var scoreMaxVelocity =
(Mathf.Abs(stateMax.Velocity) - VELOCITY_MIN) * VELOCITY_FACTOR;
var scoreCorrespondence = Mathf.Clamp01(
(DIVERGENCE_MAX - Mathf.Abs(stateMax.Velocity + stateMin.Velocity)) *
DIVERGENCE_FACTOR);
var scoreCorrespondence =
isConfident
? Mathf.Clamp01((DIVERGENCE_MAX -
Mathf.Abs(stateMax.Velocity + stateMin.Velocity)) *
DIVERGENCE_FACTOR)
: _lastCorrespondence;
var scoreSwing = Mathf.Clamp01(
(Mathf.Max(_left.SwingSize, _right.SwingSize) - HEIGHT_MIN) *
HEIGHT_FACTOR);

// TODO: Allow one handed running after building confidence
_lastCorrespondence = scoreCorrespondence;

var confidenceChangeRate = scoreMaxVelocity * scoreCorrespondence *
scoreSwing * CONFIDENCE_BUILD_RATE -
CONFIDENCE_DRAIN_RATE;
_confidence = Mathf.Clamp(
_confidence + confidenceChangeRate * Time.deltaTime, 0f, 1.2f);

// Dbg.Log(string.Join(
// ", ",
// new[] {
// ("c", _confidence),
// ("sts", stateMax.SwingSize),
// ("stv", stateMax.Velocity),
// ("smv", scoreMaxVelocity),
// ("sc", scoreCorrespondence),
// ("ss", scoreSwing),
// }
// .Select(x => $"{x.Item1}={x.Item2.ToString(" 0.00;-0.00")}")));
if (isConfident) {
_confidence = Mathf.Clamp(
_confidence + confidenceChangeRate * Time.deltaTime, 0f, 1.2f);
}

Axis = new Vector2(0f, Mathf.Clamp01(_confidence));
}
}

public class LocoHandState {
public HandTracker Tracker;
public bool IsTrackedConfident;
public float Velocity;
public float SwingSize;

private bool _isTracked;
private float _predictedNextHeight;
public float _height;
private float _height;
private float _minHeight;
private float _maxHeight;
private float _lastTime;

// TODO: Continue running with low confidence hand states
public void Update() {
_isTracked =
Tracker.IsControllerConnected() || Tracker.HandState.IsActive();
if (!_isTracked || Time.deltaTime == 0f)
// TODO: Is there something else to say hand position is low confidence?
IsTrackedConfident = Tracker.IsControllerConnected() ||
Tracker.HandState.IsActive() &&
Tracker.HandState.HasState &&
Tracker.HandState.HandConfidence ==
OVRPlugin.TrackingConfidence.High;
if (!IsTrackedConfident || Time.deltaTime == 0f)
return;

var elapsed = Time.unscaledTime - _lastTime;
_lastTime = Time.unscaledTime;

var prevHeight = _height;
// TODO: Convert to game units (meters if player is 1.78m tall)
_height = (Tracker.HandState.IsActive() ? Tracker.ProxyController
Expand All @@ -112,12 +116,14 @@ public void Update() {
var prevVelocity = Velocity;
Velocity = (_height - prevHeight) / Time.deltaTime;

if (prevVelocity <= 0f && Velocity > 0f) {
_minHeight = _predictedNextHeight;
} else if (prevVelocity >= 0f && Velocity < 0f) {
_maxHeight = _predictedNextHeight;
if (elapsed < 0.2f) {
if (prevVelocity <= 0f && Velocity > 0f) {
_minHeight = _predictedNextHeight;
} else if (prevVelocity >= 0f && Velocity < 0f) {
_maxHeight = _predictedNextHeight;
}
SwingSize = _maxHeight - _minHeight;
}
SwingSize = _maxHeight - _minHeight;

_predictedNextHeight = _height + Velocity;
}
Expand Down
5 changes: 4 additions & 1 deletion projects/Bonelab/HandTracking/src/HandState.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ private static Quaternion FromFlippedZQuatf(OVRPlugin.Quatf quat) =>
public OVRPlugin.TrackingConfidence HandConfidence;
public OVRPlugin.TrackingConfidence[] FingerConfidences;
public bool IsPinching;
public bool HasState = false;

private OVRPlugin.Hand _hand;
private OVRPlugin.SkeletonType _skeletonType;
Expand All @@ -50,8 +51,10 @@ public HandState(bool isLeft) {

public void Update() {
if (!OVRPlugin.GetHandState(OVRPlugin.Step.Render, _hand, _state)) {
throw new Exception("Failed to get hand state");
HasState = false;
return;
}
HasState = true;

Position = FromFlippedZVector3f(_state.RootPose.Position);
Rotation = FromFlippedZQuatf(_state.RootPose.Orientation);
Expand Down
Loading

0 comments on commit c5681db

Please sign in to comment.