From 7c33d3313a0e436fe7d7f570367915bd1a2aebfa Mon Sep 17 00:00:00 2001 From: "Jonathan \"Geenz\" Goodman" Date: Tue, 22 Nov 2022 23:35:42 +0100 Subject: [PATCH 1/3] Refactor the vast majority of this Still lots to go, but eyes work. --- Interface/ALXR/ALXRModule.cs | 121 ++++++++++++++++++++++++-------- Interface/EyeDevice.cs | 26 ++++--- Interface/IQuestProModule.cs | 7 +- Interface/MouthDevice.cs | 2 +- Interface/QuestProMod.cs | 65 +++++++++++++---- Interface/QuestProModule.csproj | 6 +- 6 files changed, 171 insertions(+), 56 deletions(-) diff --git a/Interface/ALXR/ALXRModule.cs b/Interface/ALXR/ALXRModule.cs index 356c86b..9d74c63 100644 --- a/Interface/ALXR/ALXRModule.cs +++ b/Interface/ALXR/ALXRModule.cs @@ -4,6 +4,8 @@ using System.Net; using System.Net.Sockets; using System.Threading; +using System.Threading.Tasks; +using NeosModLoader; namespace QuestProModule.ALXR { @@ -25,34 +27,54 @@ public class ALXRModule : IQuestProModule private double pitch_L, yaw_L, pitch_R, yaw_R; // Eye rotations - public bool Initialize() + public async Task Initialize(string ipconfig) { - localAddr = QuestProMod.config.GetValue(QuestProMod.QuestProIP); - cancellationTokenSource = new CancellationTokenSource(); - ConnectToTCP(); // Will this block the main thread? + try + { + localAddr = IPAddress.Parse(ipconfig); + + cancellationTokenSource = new CancellationTokenSource(); - tcpThread = new Thread(Update); - tcpThread.Start(); + UniLog.Log("Attempting to connect to TCP socket."); + var connected = await ConnectToTCP(); // Will this block the main thread? + } catch (Exception e) + { + UniLog.Error(e.Message); + return false; + } + + if (connected) { + tcpThread = new Thread(Update); + tcpThread.Start(); + return true; + } - return true; + return false; } - private bool ConnectToTCP() + private async Task ConnectToTCP() { try { - localAddr = QuestProMod.config.GetValue(QuestProMod.QuestProIP); - client = new TcpClient(); UniLog.Log($"Trying to establish a Quest Pro connection at {localAddr}:{DEFAULT_PORT}..."); - client.Connect(localAddr, DEFAULT_PORT); - UniLog.Log("Connected to Quest Pro!"); + await client.ConnectAsync(localAddr, DEFAULT_PORT); - stream = client.GetStream(); - connected = true; + if (client.Connected) + { - return true; + UniLog.Log("Connected to Quest Pro!"); + + stream = client.GetStream(); + connected = true; + + return true; + } else + { + UniLog.Error("Couldn't connect!"); + return false; + } } catch (Exception e) { @@ -70,7 +92,7 @@ public void Update() // Attempt reconnection if needed if (!connected || stream == null) { - ConnectToTCP(); + ConnectToTCP().RunSynchronously(); } // If the connection was unsuccessful, wait a bit and try again @@ -276,41 +298,80 @@ public void Teardown() client.Dispose(); } - public void GetEyeExpressions(FBEye fbEye, in FrooxEngine.Eye frooxEye) + bool IsValid(float3 value) => IsValid(value.x) && IsValid(value.y) && IsValid(value.z); + + bool IsValid(float value) => !float.IsInfinity(value) && !float.IsNaN(value); + + public struct XrPosef + { + public float3 position; + public floatQ rotation; + } + + public struct XrEyeGazeFB + { + public bool isValid; + public XrPosef gazePose; + public float gazeConfidence; + } + + public XrEyeGazeFB GetEyeData(FBEye fbEye) + { + XrEyeGazeFB eyeRet = new XrEyeGazeFB(); + switch (fbEye) + { + case FBEye.Left: + eyeRet.gazePose.position = new float3(expressions[68], expressions[69], expressions[70]); + eyeRet.gazePose.rotation = new floatQ(-expressions[64], -expressions[66], -expressions[65], expressions[67]); + eyeRet.isValid = IsValid(eyeRet.gazePose.position); + return eyeRet; + case FBEye.Right: + eyeRet.gazePose.position = new float3(expressions[76], expressions[77], expressions[78]); + eyeRet.gazePose.rotation = new floatQ(-expressions[72], -expressions[74], -expressions[73], expressions[75]); + eyeRet.isValid = IsValid(eyeRet.gazePose.position); + return eyeRet; + default: + throw new Exception($"Invalid eye argument: {fbEye}"); + } + } + + public void GetEyeExpressions(FBEye fbEye, FrooxEngine.Eye frooxEye) { - frooxEye.IsDeviceActive = Engine.Current.InputInterface.VR_Active; - frooxEye.IsTracking = Engine.Current.InputInterface.VR_Active; - frooxEye.PupilDiameter = 0.0035f; + frooxEye.PupilDiameter = 0.004f; frooxEye.Frown = 0f; - + switch (fbEye) { case FBEye.Left: - frooxEye.RawPosition = new float3(expressions[68], expressions[69], expressions[70]); - frooxEye.RawRotation = new floatQ(expressions[64], expressions[65], expressions[66], expressions[67]); - frooxEye.Openness = expressions[FBExpression.Eyes_Closed_L]; + frooxEye.UpdateWithRotation(new floatQ(-expressions[64], -expressions[66], -expressions[65], expressions[67])); + frooxEye.RawPosition = new float3(expressions[68], expressions[70], expressions[69]); + frooxEye.Openness = MathX.Max(0, expressions[FBExpression.Eyes_Closed_L]); frooxEye.Squeeze = expressions[FBExpression.Lid_Tightener_L]; frooxEye.Widen = expressions[FBExpression.Upper_Lid_Raiser_L]; break; case FBEye.Right: - frooxEye.RawPosition = new float3(expressions[76], expressions[77], expressions[78]); - frooxEye.RawRotation = new floatQ(expressions[72], expressions[73], expressions[74], expressions[75]); - frooxEye.Openness = expressions[FBExpression.Eyes_Closed_R]; + frooxEye.UpdateWithRotation(new floatQ(-expressions[72], -expressions[74], -expressions[73], expressions[75])); + frooxEye.RawPosition = new float3(expressions[76], expressions[78], expressions[77]); + frooxEye.Openness = MathX.Max(0, expressions[FBExpression.Eyes_Closed_R]); frooxEye.Squeeze = expressions[FBExpression.Lid_Tightener_R]; frooxEye.Widen = expressions[FBExpression.Upper_Lid_Raiser_R]; break; case FBEye.Combined: + frooxEye.UpdateWithRotation(MathX.Slerp(new floatQ(expressions[64], expressions[65], expressions[66], expressions[67]), new floatQ(expressions[72], expressions[73], expressions[74], expressions[75]), 0.5f)); frooxEye.RawPosition = MathX.Average(new float3(expressions[68], expressions[69], expressions[70]), new float3(expressions[76], expressions[77], expressions[78])); - frooxEye.RawRotation = MathX.Slerp(new floatQ(expressions[64], expressions[65], expressions[66], expressions[67]), new floatQ(expressions[72], expressions[73], expressions[74], expressions[75]), 0.5f); // Compute the midpoint by slerping from one quaternion to the other - frooxEye.Openness = (expressions[FBExpression.Eyes_Closed_R] + expressions[FBExpression.Eyes_Closed_R]) / 2.0f; + frooxEye.Openness = MathX.Max(0, expressions[FBExpression.Eyes_Closed_R] + expressions[FBExpression.Eyes_Closed_R]) / 2.0f; frooxEye.Squeeze = (expressions[FBExpression.Lid_Tightener_R] + expressions[FBExpression.Lid_Tightener_R]) / 2.0f; frooxEye.Widen = (expressions[FBExpression.Upper_Lid_Raiser_R] + expressions[FBExpression.Upper_Lid_Raiser_R]) / 2.0f; break; } + + frooxEye.IsTracking = IsValid(frooxEye.RawPosition); + frooxEye.IsTracking = IsValid(frooxEye.Direction); + frooxEye.IsTracking = IsValid(frooxEye.Openness); } // TODO: Double check jaw movements and mappings - public void GetFacialExpressions(in FrooxEngine.Mouth mouth) + public void GetFacialExpressions(FrooxEngine.Mouth mouth) { mouth.IsDeviceActive = Engine.Current.InputInterface.VR_Active; mouth.IsTracking = Engine.Current.InputInterface.VR_Active; diff --git a/Interface/EyeDevice.cs b/Interface/EyeDevice.cs index d967128..1953b6d 100644 --- a/Interface/EyeDevice.cs +++ b/Interface/EyeDevice.cs @@ -1,5 +1,7 @@ using BaseX; using FrooxEngine; +using System; +using System.Numerics; using static QuestProModule.ALXR.ALXRModule; namespace QuestProModule @@ -7,11 +9,13 @@ namespace QuestProModule public class EyeDevice : IInputDriver { private Eyes _eyes; + InputInterface input; public int UpdateOrder => 100; - - public EyeDevice() + private float _openExponent = 1.0f; + public EyeDevice(float openExponent) { Engine.Current.OnShutdown += Teardown; + _openExponent = openExponent; } private void Teardown() @@ -31,17 +35,23 @@ public void CollectDeviceInfos(DataTreeList list) public void RegisterInputs(InputInterface inputInterface) { _eyes = new Eyes(inputInterface, "Quest Pro Eye Tracking"); + input = inputInterface; } public void UpdateInputs(float deltaTime) { - _eyes.IsEyeTrackingActive = Engine.Current.InputInterface.VR_Active; - _eyes.IsTracking = Engine.Current.InputInterface.VR_Active; + _eyes.IsEyeTrackingActive = input.VR_Active; + + _eyes.LeftEye.IsTracking = input.VR_Active; + + QuestProMod.qpm.GetEyeExpressions(FBEye.Left, _eyes.LeftEye); + QuestProMod.qpm.GetEyeExpressions(FBEye.Right, _eyes.RightEye); + QuestProMod.qpm.GetEyeExpressions(FBEye.Combined, _eyes.CombinedEye); + + _eyes.LeftEye.Openness = MathX.Pow(_eyes.LeftEye.Openness, _openExponent); + _eyes.RightEye.Openness = MathX.Pow(_eyes.RightEye.Openness, _openExponent); + _eyes.CombinedEye.Openness = MathX.Pow(_eyes.CombinedEye.Openness, _openExponent); - QuestProMod.qpm.GetEyeExpressions(FBEye.Left, in _eyes.LeftEye); - QuestProMod.qpm.GetEyeExpressions(FBEye.Right, in _eyes.RightEye); - QuestProMod.qpm.GetEyeExpressions(FBEye.Combined, in _eyes.CombinedEye); - _eyes.ComputeCombinedEyeParameters(); _eyes.ConvergenceDistance = 0f; _eyes.Timestamp += deltaTime; diff --git a/Interface/IQuestProModule.cs b/Interface/IQuestProModule.cs index 153b24a..82c14b6 100644 --- a/Interface/IQuestProModule.cs +++ b/Interface/IQuestProModule.cs @@ -1,8 +1,11 @@ -namespace QuestProModule +using System.Threading.Tasks; +using NeosModLoader; + +namespace QuestProModule { public interface IQuestProModule { - public bool Initialize(); + public Task Initialize(string ipaddress); public void Update(); diff --git a/Interface/MouthDevice.cs b/Interface/MouthDevice.cs index ca787cf..eb603de 100644 --- a/Interface/MouthDevice.cs +++ b/Interface/MouthDevice.cs @@ -34,7 +34,7 @@ public void RegisterInputs(InputInterface inputInterface) public void UpdateInputs(float deltaTime) { - QuestProMod.qpm.GetFacialExpressions(in mouth); + QuestProMod.qpm.GetFacialExpressions(mouth); } } } diff --git a/Interface/QuestProMod.cs b/Interface/QuestProMod.cs index 647312d..37f1155 100644 --- a/Interface/QuestProMod.cs +++ b/Interface/QuestProMod.cs @@ -3,34 +3,73 @@ using FrooxEngine; using System.Net; using QuestProModule.ALXR; +using BaseX; +using System; +using FrooxEngine.UIX; namespace QuestProModule { public class QuestProMod : NeosMod { - public static ModConfiguration config; - [AutoRegisterConfigKey] - public static ModConfigurationKey QuestProIP = new ModConfigurationKey("quest_pro_IP", "Quest Pro IP. This can be found in ALXR's settings, requires a restart to take effect", () => IPAddress.Parse("192.168.1.163")); + private readonly static ModConfigurationKey QuestProIP = new ModConfigurationKey("quest_pro_IP", "Quest Pro IP. This can be found in ALXR's settings, requires a restart to take effect", () => "127.0.0.1"); + + [AutoRegisterConfigKey] + private readonly static ModConfigurationKey EyeOpennessExponent = new ModConfigurationKey("quest_pro_eye_open_exponent", "Exponent to apply to eye openness. Useful for applying different curves for how open your eyes are.", () => 1.0f); public static ALXRModule qpm; + static ModConfiguration _config; + public override string Name => "QuestPro4Neos"; - public override string Author => "dfgHiatus"; + public override string Author => "dfgHiatus & Geenz"; public override string Version => "1.0.0"; public override string Link => "https://github.com/dfgHiatus/QuestPro4Neos"; public override void OnEngineInit() { - qpm = new ALXRModule(); - qpm.Initialize(); + _config = GetConfiguration(); - config = GetConfiguration(); - new Harmony("net.dfgHiatus.QuestPro4Neos").PatchAll(); - Engine.Current.InputInterface.RegisterInputDriver(new EyeDevice()); - Engine.Current.OnReady += () => - Engine.Current.InputInterface.RegisterInputDriver(new MouthDevice()); + var openness = default(float); - Engine.Current.OnShutdown += () => qpm.Teardown(); + if (!_config.TryGetValue(EyeOpennessExponent, out openness)) + { + openness = 1.0f; + + _config.Set(EyeOpennessExponent, openness); + } + + new Harmony("net.dfgHiatus.QuestPro4Neos").PatchAll(); } - } + + [HarmonyPatch(typeof(InputInterface), MethodType.Constructor)] + [HarmonyPatch(new Type[] { typeof(Engine) })] + public class InputInterfaceCtorPatch + { + public static void Postfix(InputInterface __instance) + { + try + { + qpm = new ALXRModule(); + + qpm.Initialize(_config.GetValue(QuestProIP)); + + __instance.RegisterInputDriver(new EyeDevice(_config.GetValue(EyeOpennessExponent))); + __instance.RegisterInputDriver(new MouthDevice()); + + Engine.Current.OnShutdown += () => qpm.Teardown(); + } + catch (Exception ex) + { + Warn("Module failed to initiallize."); + Warn(ex.ToString()); + } + } + } + + private void OnConfigurationChanged(ConfigurationChangedEvent @event) + { + if (@event.Label == "NeosModSettings variable change") return; + + } + } } diff --git a/Interface/QuestProModule.csproj b/Interface/QuestProModule.csproj index ae1c37f..f3ec6d4 100644 --- a/Interface/QuestProModule.csproj +++ b/Interface/QuestProModule.csproj @@ -67,9 +67,11 @@ - + + + - copy "$(TargetDir)\$(TargetFileName)" "G:\Steam\steamapps\common\NeosVR\nml_mods" + copy "$(TargetDir)\$(TargetFileName)" "C:\Program Files (x86)\Steam\steamapps\common\NeosVR\nml_mods" \ No newline at end of file From 626f2b2939ae7f628b4777bd6c1e7628d643313a Mon Sep 17 00:00:00 2001 From: "Jonathan \"Geenz\" Goodman" Date: Wed, 23 Nov 2022 00:42:12 +0100 Subject: [PATCH 2/3] Start refactoring ALXRModule into a data source. Mouth and Eye devices should be containing the logic for these things. --- Interface/ALXR/ALXRModule.cs | 34 +++++++++++--------- Interface/EyeDevice.cs | 55 +++++++++++++++++++++++++++------ Interface/MouthDevice.cs | 4 ++- Interface/QuestProMod.cs | 40 +++++++++++++++++------- Interface/QuestProModule.csproj | 3 -- 5 files changed, 96 insertions(+), 40 deletions(-) diff --git a/Interface/ALXR/ALXRModule.cs b/Interface/ALXR/ALXRModule.cs index 9d74c63..a008621 100644 --- a/Interface/ALXR/ALXRModule.cs +++ b/Interface/ALXR/ALXRModule.cs @@ -302,33 +302,37 @@ public void Teardown() bool IsValid(float value) => !float.IsInfinity(value) && !float.IsNaN(value); - public struct XrPosef + public struct EyeGazeData { + public bool isValid; public float3 position; public floatQ rotation; - } - - public struct XrEyeGazeFB - { - public bool isValid; - public XrPosef gazePose; + public float open; + public float squeeze; + public float wide; public float gazeConfidence; } - public XrEyeGazeFB GetEyeData(FBEye fbEye) + public EyeGazeData GetEyeData(FBEye fbEye) { - XrEyeGazeFB eyeRet = new XrEyeGazeFB(); + EyeGazeData eyeRet = new EyeGazeData(); switch (fbEye) { case FBEye.Left: - eyeRet.gazePose.position = new float3(expressions[68], expressions[69], expressions[70]); - eyeRet.gazePose.rotation = new floatQ(-expressions[64], -expressions[66], -expressions[65], expressions[67]); - eyeRet.isValid = IsValid(eyeRet.gazePose.position); + eyeRet.position = new float3(expressions[68], expressions[70], expressions[69]); + eyeRet.rotation = new floatQ(-expressions[64], -expressions[66], -expressions[65], expressions[67]); + eyeRet.open = MathX.Max(0, expressions[FBExpression.Eyes_Closed_L]); + eyeRet.squeeze = expressions[FBExpression.Lid_Tightener_L]; + eyeRet.wide = expressions[FBExpression.Upper_Lid_Raiser_L]; + eyeRet.isValid = IsValid(eyeRet.position); return eyeRet; case FBEye.Right: - eyeRet.gazePose.position = new float3(expressions[76], expressions[77], expressions[78]); - eyeRet.gazePose.rotation = new floatQ(-expressions[72], -expressions[74], -expressions[73], expressions[75]); - eyeRet.isValid = IsValid(eyeRet.gazePose.position); + eyeRet.position = new float3(expressions[76], expressions[78], expressions[77]); + eyeRet.rotation = new floatQ(-expressions[72], -expressions[74], -expressions[73], expressions[75]); + eyeRet.open = MathX.Max(0, expressions[FBExpression.Eyes_Closed_R]); + eyeRet.squeeze = expressions[FBExpression.Lid_Tightener_R]; + eyeRet.wide = expressions[FBExpression.Upper_Lid_Raiser_R]; + eyeRet.isValid = IsValid(eyeRet.position); return eyeRet; default: throw new Exception($"Invalid eye argument: {fbEye}"); diff --git a/Interface/EyeDevice.cs b/Interface/EyeDevice.cs index 1953b6d..54d2257 100644 --- a/Interface/EyeDevice.cs +++ b/Interface/EyeDevice.cs @@ -1,5 +1,6 @@ using BaseX; using FrooxEngine; +using QuestProModule.ALXR; using System; using System.Numerics; using static QuestProModule.ALXR.ALXRModule; @@ -11,11 +12,10 @@ public class EyeDevice : IInputDriver private Eyes _eyes; InputInterface input; public int UpdateOrder => 100; - private float _openExponent = 1.0f; - public EyeDevice(float openExponent) + + public EyeDevice() { Engine.Current.OnShutdown += Teardown; - _openExponent = openExponent; } private void Teardown() @@ -44,13 +44,50 @@ public void UpdateInputs(float deltaTime) _eyes.LeftEye.IsTracking = input.VR_Active; - QuestProMod.qpm.GetEyeExpressions(FBEye.Left, _eyes.LeftEye); - QuestProMod.qpm.GetEyeExpressions(FBEye.Right, _eyes.RightEye); - QuestProMod.qpm.GetEyeExpressions(FBEye.Combined, _eyes.CombinedEye); + var leftEyeData = QuestProMod.qpm.GetEyeData(FBEye.Left); + var rightEyeData = QuestProMod.qpm.GetEyeData(FBEye.Right); + + _eyes.LeftEye.IsTracking = leftEyeData.isValid; + _eyes.LeftEye.RawPosition = leftEyeData.position; + _eyes.LeftEye.UpdateWithRotation(leftEyeData.rotation); + _eyes.LeftEye.PupilDiameter = 0.004f; + _eyes.LeftEye.Widen = leftEyeData.wide; + _eyes.LeftEye.Squeeze = leftEyeData.squeeze; + _eyes.LeftEye.Openness = leftEyeData.open; + + _eyes.RightEye.IsTracking = rightEyeData.isValid; + _eyes.RightEye.RawPosition = rightEyeData.position; + _eyes.RightEye.UpdateWithRotation(rightEyeData.rotation); + _eyes.RightEye.PupilDiameter = 0.004f; + _eyes.RightEye.Widen = rightEyeData.wide; + _eyes.RightEye.Squeeze = rightEyeData.squeeze; + _eyes.RightEye.Openness = rightEyeData.open; + + if (_eyes.LeftEye.IsTracking || _eyes.RightEye.IsTracking && (!_eyes.LeftEye.IsTracking || !_eyes.RightEye.IsTracking)) + { + if (_eyes.LeftEye.IsTracking) + { + _eyes.CombinedEye.RawPosition = _eyes.LeftEye.RawPosition; + _eyes.CombinedEye.UpdateWithRotation(_eyes.LeftEye.RawRotation); + } else + { + _eyes.CombinedEye.RawPosition = _eyes.RightEye.RawPosition; + _eyes.CombinedEye.UpdateWithRotation(_eyes.RightEye.RawRotation); + } + _eyes.CombinedEye.IsTracking = true; + } + else + { + _eyes.CombinedEye.IsTracking = false; + } + + _eyes.CombinedEye.IsTracking = _eyes.LeftEye.IsTracking || _eyes.RightEye.IsTracking; + _eyes.CombinedEye.RawPosition = (_eyes.LeftEye.RawPosition + _eyes.RightEye.RawPosition) * 0.5f; + _eyes.CombinedEye.UpdateWithRotation(MathX.Slerp(_eyes.LeftEye.RawRotation, _eyes.RightEye.RawRotation, 0.5f)); + _eyes.CombinedEye.PupilDiameter = 0.004f; - _eyes.LeftEye.Openness = MathX.Pow(_eyes.LeftEye.Openness, _openExponent); - _eyes.RightEye.Openness = MathX.Pow(_eyes.RightEye.Openness, _openExponent); - _eyes.CombinedEye.Openness = MathX.Pow(_eyes.CombinedEye.Openness, _openExponent); + _eyes.LeftEye.Openness = MathX.Pow(_eyes.LeftEye.Openness, QuestProMod.EyeOpenExponent); + _eyes.RightEye.Openness = MathX.Pow(_eyes.RightEye.Openness, QuestProMod.EyeOpenExponent); _eyes.ComputeCombinedEyeParameters(); _eyes.ConvergenceDistance = 0f; diff --git a/Interface/MouthDevice.cs b/Interface/MouthDevice.cs index eb603de..5d15ff6 100644 --- a/Interface/MouthDevice.cs +++ b/Interface/MouthDevice.cs @@ -7,6 +7,7 @@ public class MouthDevice : IInputDriver { private Mouth mouth; public int UpdateOrder => 100; + private InputInterface _input; public MouthDevice() { @@ -22,7 +23,7 @@ public void CollectDeviceInfos(DataTreeList list) { var mouthDataTreeDictionary = new DataTreeDictionary(); mouthDataTreeDictionary.Add("Name", "Quest Pro Face Tracking"); - mouthDataTreeDictionary.Add("Type", "Face Tracking"); + mouthDataTreeDictionary.Add("Type", "Lip Tracking"); mouthDataTreeDictionary.Add("Model", "Quest Pro"); list.Add(mouthDataTreeDictionary); } @@ -30,6 +31,7 @@ public void CollectDeviceInfos(DataTreeList list) public void RegisterInputs(InputInterface inputInterface) { mouth = new Mouth(inputInterface, "Quest Pro Mouth Tracking"); + _input = inputInterface; } public void UpdateInputs(float deltaTime) diff --git a/Interface/QuestProMod.cs b/Interface/QuestProMod.cs index 37f1155..fd374b1 100644 --- a/Interface/QuestProMod.cs +++ b/Interface/QuestProMod.cs @@ -15,13 +15,19 @@ public class QuestProMod : NeosMod private readonly static ModConfigurationKey QuestProIP = new ModConfigurationKey("quest_pro_IP", "Quest Pro IP. This can be found in ALXR's settings, requires a restart to take effect", () => "127.0.0.1"); [AutoRegisterConfigKey] - private readonly static ModConfigurationKey EyeOpennessExponent = new ModConfigurationKey("quest_pro_eye_open_exponent", "Exponent to apply to eye openness. Useful for applying different curves for how open your eyes are.", () => 1.0f); + private readonly static ModConfigurationKey EyeOpennessExponent = new ModConfigurationKey("quest_pro_eye_open_exponent", "Exponent to apply to eye openness. Can be updated at runtime. Useful for applying different curves for how open your eyes are.", () => 1.0f); - public static ALXRModule qpm; + [AutoRegisterConfigKey] + private readonly static ModConfigurationKey EyeWideMultiplier = new ModConfigurationKey("quest_pro_eye_wide_multiplier", "Multiplier to apply to eye wideness. Can be updated at runtime. Useful for multiplying the amount your eyes can widen by.", () => 1.0f); + + public static ALXRModule qpm; static ModConfiguration _config; - public override string Name => "QuestPro4Neos"; + public static float EyeOpenExponent = 1.0f; + public static float EyeWideMult = 1.0f; + + public override string Name => "QuestPro4Neos"; public override string Author => "dfgHiatus & Geenz"; public override string Version => "1.0.0"; public override string Link => "https://github.com/dfgHiatus/QuestPro4Neos"; @@ -29,14 +35,7 @@ public override void OnEngineInit() { _config = GetConfiguration(); - var openness = default(float); - - if (!_config.TryGetValue(EyeOpennessExponent, out openness)) - { - openness = 1.0f; - - _config.Set(EyeOpennessExponent, openness); - } + _config.OnThisConfigurationChanged += OnConfigurationChanged; new Harmony("net.dfgHiatus.QuestPro4Neos").PatchAll(); } @@ -53,7 +52,7 @@ public static void Postfix(InputInterface __instance) qpm.Initialize(_config.GetValue(QuestProIP)); - __instance.RegisterInputDriver(new EyeDevice(_config.GetValue(EyeOpennessExponent))); + __instance.RegisterInputDriver(new EyeDevice()); __instance.RegisterInputDriver(new MouthDevice()); Engine.Current.OnShutdown += () => qpm.Teardown(); @@ -69,7 +68,24 @@ public static void Postfix(InputInterface __instance) private void OnConfigurationChanged(ConfigurationChangedEvent @event) { if (@event.Label == "NeosModSettings variable change") return; + UniLog.Log($"Var changed! {@event.Label}"); + if (@event.Key == EyeOpennessExponent) + { + float openExp = 1.0f; + if (@event.Config.TryGetValue(EyeOpennessExponent, out openExp)) + { + EyeOpenExponent = openExp; + } + } + if (@event.Key == EyeOpennessExponent) + { + float wideMult = 1.0f; + if (@event.Config.TryGetValue(EyeWideMultiplier, out wideMult)) + { + EyeWideMult = wideMult; + } + } } } } diff --git a/Interface/QuestProModule.csproj b/Interface/QuestProModule.csproj index f3ec6d4..196ca6b 100644 --- a/Interface/QuestProModule.csproj +++ b/Interface/QuestProModule.csproj @@ -67,9 +67,6 @@ - - - copy "$(TargetDir)\$(TargetFileName)" "C:\Program Files (x86)\Steam\steamapps\common\NeosVR\nml_mods" From 5eab87d583b1ea03400677958e6eba2e167646d1 Mon Sep 17 00:00:00 2001 From: dfgHiatus <51272212+dfgHiatus@users.noreply.github.com> Date: Mon, 28 Nov 2022 08:53:27 -0600 Subject: [PATCH 3/3] Merge Geenz, added more const int values for FBExpression --- Interface/ALXR/ALXRModule.cs | 57 ++++++++++++++++++------------------ Interface/FBExpression.cs | 19 +++++++++++- 2 files changed, 47 insertions(+), 29 deletions(-) diff --git a/Interface/ALXR/ALXRModule.cs b/Interface/ALXR/ALXRModule.cs index a008621..fc348d9 100644 --- a/Interface/ALXR/ALXRModule.cs +++ b/Interface/ALXR/ALXRModule.cs @@ -5,7 +5,6 @@ using System.Net.Sockets; using System.Threading; using System.Threading.Tasks; -using NeosModLoader; namespace QuestProModule.ALXR { @@ -36,14 +35,16 @@ public async Task Initialize(string ipconfig) cancellationTokenSource = new CancellationTokenSource(); UniLog.Log("Attempting to connect to TCP socket."); - var connected = await ConnectToTCP(); // Will this block the main thread? - } catch (Exception e) + var connected = await ConnectToTCP(); // This should not block the main thread anymore...? + } + catch (Exception e) { UniLog.Error(e.Message); return false; } - if (connected) { + if (connected) + { tcpThread = new Thread(Update); tcpThread.Start(); return true; @@ -63,14 +64,14 @@ private async Task ConnectToTCP() if (client.Connected) { - UniLog.Log("Connected to Quest Pro!"); stream = client.GetStream(); connected = true; return true; - } else + } + else { UniLog.Error("Couldn't connect!"); return false; @@ -104,7 +105,7 @@ public void Update() if (!stream.CanRead) { - UniLog.Warning("Can't read from network stream just yet! Trying again..."); + UniLog.Warning("Can't read from the Quest Pro network stream just yet! Trying again..."); return; } @@ -135,6 +136,7 @@ public void Update() // We receive information from the stream as a byte array 63*4 bytes long, since floats are 32 bits long and we have 63 expressions. // We then need to convert these bytes to a floats. I've opted to use Buffer.BlockCopy instead of BitConverter.ToSingle, since it's faster. + // Future note: Testing this against Array.Copy() doesn't seem to show any difference in performance, so I'll stick to this Buffer.BlockCopy(rawExpressions, 0, expressions, 0, NATURAL_EXPRESSIONS_COUNT * 4 + (8 * 2 * 4)); // Preprocess our expressions per Meta's Documentation @@ -145,7 +147,6 @@ public void Update() UniLog.Error(e.Message); Thread.Sleep(1000); } - } } @@ -153,10 +154,10 @@ private void PrepareUpdate() { // Eye Expressions - double q_x = expressions[64]; - double q_y = expressions[65]; - double q_z = expressions[66]; - double q_w = expressions[67]; + double q_x = expressions[FBExpression.LeftRot_x]; + double q_y = expressions[FBExpression.LeftRot_y]; + double q_z = expressions[FBExpression.LeftRot_z]; + double q_w = expressions[FBExpression.LeftRot_w]; double yaw = Math.Atan2(2.0 * (q_y * q_z + q_w * q_x), q_w * q_w - q_x * q_x - q_y * q_y + q_z * q_z); double pitch = Math.Asin(-2.0 * (q_x * q_z - q_w * q_y)); @@ -167,10 +168,10 @@ private void PrepareUpdate() pitch_L = 180.0 / Math.PI * pitch; yaw_L = 180.0 / Math.PI * yaw; - q_x = expressions[72]; - q_y = expressions[73]; - q_z = expressions[74]; - q_w = expressions[75]; + q_x = expressions[FBExpression.RightRot_x]; + q_y = expressions[FBExpression.RightRot_y]; + q_z = expressions[FBExpression.RightRot_z]; + q_w = expressions[FBExpression.RightRot_w]; yaw = Math.Atan2(2.0 * (q_y * q_z + q_w * q_x), q_w * q_w - q_x * q_x - q_y * q_y + q_z * q_z); pitch = Math.Asin(-2.0 * (q_x * q_z - q_w * q_y)); @@ -319,16 +320,16 @@ public EyeGazeData GetEyeData(FBEye fbEye) switch (fbEye) { case FBEye.Left: - eyeRet.position = new float3(expressions[68], expressions[70], expressions[69]); - eyeRet.rotation = new floatQ(-expressions[64], -expressions[66], -expressions[65], expressions[67]); + eyeRet.position = new float3(expressions[FBExpression.LeftPos_x], expressions[FBExpression.LeftPos_y], expressions[FBExpression.LeftPos_z]); + eyeRet.rotation = new floatQ(-expressions[FBExpression.LeftRot_x], -expressions[FBExpression.LeftRot_z], -expressions[FBExpression.LeftRot_y], expressions[FBExpression.LeftRot_w]); eyeRet.open = MathX.Max(0, expressions[FBExpression.Eyes_Closed_L]); eyeRet.squeeze = expressions[FBExpression.Lid_Tightener_L]; eyeRet.wide = expressions[FBExpression.Upper_Lid_Raiser_L]; eyeRet.isValid = IsValid(eyeRet.position); return eyeRet; case FBEye.Right: - eyeRet.position = new float3(expressions[76], expressions[78], expressions[77]); - eyeRet.rotation = new floatQ(-expressions[72], -expressions[74], -expressions[73], expressions[75]); + eyeRet.position = new float3(expressions[FBExpression.RightPos_x], expressions[FBExpression.RightPos_y], expressions[FBExpression.RightPos_z]); + eyeRet.rotation = new floatQ(-expressions[FBExpression.RightRot_x], -expressions[FBExpression.RightRot_z], -expressions[FBExpression.RightRot_y], expressions[FBExpression.RightRot_w]); eyeRet.open = MathX.Max(0, expressions[FBExpression.Eyes_Closed_R]); eyeRet.squeeze = expressions[FBExpression.Lid_Tightener_R]; eyeRet.wide = expressions[FBExpression.Upper_Lid_Raiser_R]; @@ -347,22 +348,22 @@ public void GetEyeExpressions(FBEye fbEye, FrooxEngine.Eye frooxEye) switch (fbEye) { case FBEye.Left: - frooxEye.UpdateWithRotation(new floatQ(-expressions[64], -expressions[66], -expressions[65], expressions[67])); - frooxEye.RawPosition = new float3(expressions[68], expressions[70], expressions[69]); + frooxEye.UpdateWithRotation(new floatQ(-expressions[FBExpression.LeftRot_x], -expressions[FBExpression.LeftRot_z], -expressions[FBExpression.LeftRot_y], expressions[FBExpression.LeftRot_w])); + frooxEye.RawPosition = new float3(expressions[FBExpression.LeftPos_x], expressions[FBExpression.LeftPos_y], expressions[FBExpression.LeftPos_z]); frooxEye.Openness = MathX.Max(0, expressions[FBExpression.Eyes_Closed_L]); frooxEye.Squeeze = expressions[FBExpression.Lid_Tightener_L]; frooxEye.Widen = expressions[FBExpression.Upper_Lid_Raiser_L]; break; case FBEye.Right: - frooxEye.UpdateWithRotation(new floatQ(-expressions[72], -expressions[74], -expressions[73], expressions[75])); - frooxEye.RawPosition = new float3(expressions[76], expressions[78], expressions[77]); + frooxEye.UpdateWithRotation(new floatQ(-expressions[FBExpression.RightRot_x], -expressions[FBExpression.RightRot_z], -expressions[FBExpression.RightRot_y], expressions[FBExpression.RightRot_w])); + frooxEye.RawPosition = new float3(expressions[FBExpression.RightPos_x], expressions[FBExpression.RightPos_y], expressions[FBExpression.RightPos_z]); frooxEye.Openness = MathX.Max(0, expressions[FBExpression.Eyes_Closed_R]); frooxEye.Squeeze = expressions[FBExpression.Lid_Tightener_R]; frooxEye.Widen = expressions[FBExpression.Upper_Lid_Raiser_R]; break; case FBEye.Combined: - frooxEye.UpdateWithRotation(MathX.Slerp(new floatQ(expressions[64], expressions[65], expressions[66], expressions[67]), new floatQ(expressions[72], expressions[73], expressions[74], expressions[75]), 0.5f)); - frooxEye.RawPosition = MathX.Average(new float3(expressions[68], expressions[69], expressions[70]), new float3(expressions[76], expressions[77], expressions[78])); + frooxEye.UpdateWithRotation(MathX.Slerp(new floatQ(expressions[FBExpression.LeftRot_x], expressions[FBExpression.LeftRot_y], expressions[FBExpression.LeftRot_z], expressions[FBExpression.LeftRot_w]), new floatQ(expressions[FBExpression.RightRot_x], expressions[FBExpression.RightRot_y], expressions[FBExpression.RightRot_z], expressions[FBExpression.RightRot_w]), 0.5f)); + frooxEye.RawPosition = MathX.Average(new float3(expressions[FBExpression.LeftPos_x], expressions[FBExpression.LeftPos_z], expressions[FBExpression.LeftPos_y]), new float3(expressions[FBExpression.RightPos_x], expressions[FBExpression.RightPos_z], expressions[FBExpression.RightPos_y])); frooxEye.Openness = MathX.Max(0, expressions[FBExpression.Eyes_Closed_R] + expressions[FBExpression.Eyes_Closed_R]) / 2.0f; frooxEye.Squeeze = (expressions[FBExpression.Lid_Tightener_R] + expressions[FBExpression.Lid_Tightener_R]) / 2.0f; frooxEye.Widen = (expressions[FBExpression.Upper_Lid_Raiser_R] + expressions[FBExpression.Upper_Lid_Raiser_R]) / 2.0f; @@ -397,8 +398,8 @@ public void GetFacialExpressions(FrooxEngine.Mouth mouth) mouth.LipUpperHorizontal = stretch; mouth.LipLowerHorizontal = stretch; - mouth.MouthLeftSmileFrown = Math.Min(1, expressions[FBExpression.Lip_Corner_Puller_L] * 1.2f) - Math.Min(1, (expressions[FBExpression.Lip_Corner_Depressor_L] + expressions[FBExpression.Lip_Stretcher_L]) * 0.75f);//Math.Min(1, (expressions[FBExpression.Lip_Corner_Depressor_L]) * 1.5f);; - mouth.MouthRightSmileFrown = Math.Min(1, expressions[FBExpression.Lip_Corner_Puller_R] * 1.2f) - Math.Min(1, (expressions[FBExpression.Lip_Corner_Depressor_R] + expressions[FBExpression.Lip_Stretcher_R]) * 0.75f);//Math.Min(1, (expressions[FBExpression.Lip_Corner_Depressor_R]) * 1.5f);; + mouth.MouthLeftSmileFrown = Math.Min(1, expressions[FBExpression.Lip_Corner_Puller_L] * 1.2f) - Math.Min(1, (expressions[FBExpression.Lip_Corner_Depressor_L] + expressions[FBExpression.Lip_Stretcher_L]) * SRANIPAL_NORMALIZER);//Math.Min(1, (expressions[FBExpression.Lip_Corner_Depressor_L]) * 1.5f);; + mouth.MouthRightSmileFrown = Math.Min(1, expressions[FBExpression.Lip_Corner_Puller_R] * 1.2f) - Math.Min(1, (expressions[FBExpression.Lip_Corner_Depressor_R] + expressions[FBExpression.Lip_Stretcher_R]) * SRANIPAL_NORMALIZER);//Math.Min(1, (expressions[FBExpression.Lip_Corner_Depressor_R]) * 1.5f);; mouth.MouthPout = (expressions[FBExpression.Lip_Pucker_L] + expressions[FBExpression.Lip_Pucker_R]) / 3; diff --git a/Interface/FBExpression.cs b/Interface/FBExpression.cs index e0ed02d..da0819f 100644 --- a/Interface/FBExpression.cs +++ b/Interface/FBExpression.cs @@ -1,6 +1,5 @@ namespace QuestProModule { - // TODO Get position for eye and face data public static class FBExpression { public const int Brow_Lowerer_L = 0; @@ -67,5 +66,23 @@ public static class FBExpression public const int Upper_Lip_Raiser_L = 61; public const int Upper_Lip_Raiser_R = 62; public const int Max = 63; + + // Above are the natural expressions tracked by the Quest Pro + // Below is the eye tracking information + public const int LeftRot_x = 64; + public const int LeftRot_y = 65; + public const int LeftRot_z = 66; + public const int LeftRot_w = 67; + public const int LeftPos_x = 68; + public const int LeftPos_y = 70; // Flipped, need to convert RHS to LHS + public const int LeftPos_z = 69; + // public const int 71 is unused + public const int RightRot_x = 72; + public const int RightRot_y = 73; + public const int RightRot_z = 74; + public const int RightRot_w = 75; + public const int RightPos_x = 76; + public const int RightPos_y = 78; // Flipped, need to convert RHS to LHS + public const int RightPos_z = 77; } }