Merge pull request #195 from inworld-ai/yjRegen2

Add Sentis viseme generating support

Assets/Inworld/Editor/DependencyImporter.cs

@@ -28,7 +28,11 @@ public class DependencyImporter : AssetPostprocessor
         const string k_DependencyPackage = "com.inworld.unity.core";
         const string k_InworldAssetsPath = "Assets/Inworld/Inworld.Assets";
         const string k_ExtraPackagePath = "Assets/Inworld/InworldExtraAssets.unitypackage";
-
+
+        static string[] k_DependentOfficialPackages = 
+        {
+            "com.unity.sentis"
+        };
         static DependencyImporter()
         {
             AssetDatabase.importPackageCompleted += _ =>
@@ -46,7 +50,13 @@ public static async void InstallDependencies()
                     return;
             } 
             Debug.Log("Import Dependency Packages...");
-            await _AddPackage();
+            await _AddPackage(); 
+            #if UNITY_2022_3_OR_NEWER
+            foreach (string official in k_DependentOfficialPackages)
+            {
+                await _AddUnityPackage(official);
+            }
+            #endif
         }
 
         static string _GetTgzFileName()
@@ -55,7 +65,7 @@ static string _GetTgzFileName()
             string[] tgzFiles = Directory.GetFiles(searchDirectory, "*.tgz", SearchOption.TopDirectoryOnly);
             return tgzFiles.Length > 0 ? $"file:{tgzFiles[0]}" : "";
         }
-        static async Task _AddPackage()
+        static async Task _AddUnityPackage(string package, string detail = "")
         {
             ListRequest listRequest = Client.List();
 
@@ -68,29 +78,62 @@ static async Task _AddPackage()
                 Debug.LogError(listRequest.Error.ToString());
                 return;
             }
-            if (listRequest.Result.Any(x => x.name == k_DependencyPackage))
-            {
-                Debug.Log($"{k_DependencyPackage} Found.");
-                return;
-            }
-            string tgzFile = _GetTgzFileName();
-            if (string.IsNullOrEmpty(tgzFile))
+            if (listRequest.Result.Any(x => x.name == package))
             {
-                Debug.LogError("Cannot Find core package.");
+                Debug.Log($"{package} Found.");
                 return;
             }
-            AddRequest addRequest = Client.Add(tgzFile);
+            string pkgToLoad = string.IsNullOrEmpty(detail) ? package : detail;
+            AddRequest addRequest = Client.Add(pkgToLoad);
             while (!addRequest.IsCompleted)
             {
                 await Task.Yield();
             }
-
             if (addRequest.Status != StatusCode.Success)
             {
-                Debug.LogError($"Failed to add {tgzFile}.");
+                Debug.LogError($"Failed to add {package}.");
                 return;
             }
-            Debug.Log($"Import {k_DependencyPackage} Completed");
+            Debug.Log($"Import {package} Completed");
+        }
+
+        static async Task _AddPackage()
+        {
+            await _AddUnityPackage(k_DependencyPackage, _GetTgzFileName());
+            // ListRequest listRequest = Client.List();
+            //
+            // while (!listRequest.IsCompleted)
+            // {
+            //     await Task.Yield();
+            // }
+            // if (listRequest.Status != StatusCode.Success)
+            // {
+            //     Debug.LogError(listRequest.Error.ToString());
+            //     return;
+            // }
+            // if (listRequest.Result.Any(x => x.name == k_DependencyPackage))
+            // {
+            //     Debug.Log($"{k_DependencyPackage} Found.");
+            //     return;
+            // }
+            // string tgzFile = _GetTgzFileName();
+            // if (string.IsNullOrEmpty(tgzFile))
+            // {
+            //     Debug.LogError("Cannot Find core package.");
+            //     return;
+            // }
+            // AddRequest addRequest = Client.Add(tgzFile);
+            // while (!addRequest.IsCompleted)
+            // {
+            //     await Task.Yield();
+            // }
+            //
+            // if (addRequest.Status != StatusCode.Success)
+            // {
+            //     Debug.LogError($"Failed to add {tgzFile}.");
+            //     return;
+            // }
+            // Debug.Log($"Import {k_DependencyPackage} Completed");
             if (!Directory.Exists(k_InworldAssetsPath) && File.Exists(k_ExtraPackagePath))
                 AssetDatabase.ImportPackage(k_ExtraPackagePath, false);
             if (!File.Exists("Assets/TextMesh Pro/Resources/TMP Settings.asset"))

Assets/Inworld/Editor/UnityPackageExporter.cs

@@ -81,7 +81,7 @@ public static void ExportExtraAssets()
             {
                 $"{k_FullPackagePath}/Inworld.Assets", 
                 $"{k_FullPackagePath}/Inworld.Editor",
-                $"{k_FullPackagePath}/Inworld.NDK",
+                $"{k_FullPackagePath}/Inworld.Native",
                 $"{k_FullPackagePath}/Inworld.Samples.Innequin",
                 $"{k_FullPackagePath}/Inworld.Samples.RPM"
             }; 

Assets/Inworld/Inworld.Assets/Animations/Facial/InworldLipsync.asset

@@ -15,111 +15,187 @@ MonoBehaviour:
   p2vMap:
   - phoneme: <INTERSPERSE_CHARACTER>
     visemeIndex: -1
+    tensorIndex: -1
   - phoneme: ' '
     visemeIndex: -1
+    tensorIndex: -1
+  - phoneme: sil
+    visemeIndex: -1
+    tensorIndex: 0
   - phoneme: a
     visemeIndex: 10
+    tensorIndex: 1
   - phoneme: "\xE6"
     visemeIndex: 10
+    tensorIndex: 2
   - phoneme: "\u0251"
     visemeIndex: 10
+    tensorIndex: 1
   - phoneme: b
     visemeIndex: 1
+    tensorIndex: 17
   - phoneme: d
     visemeIndex: 4
+    tensorIndex: 19
   - phoneme: "\xF0"
     visemeIndex: 3
+    tensorIndex: 20
+  - phoneme: "d\u0292"
+    visemeIndex: 3
+    tensorIndex: 24
   - phoneme: e
     visemeIndex: 11
+    tensorIndex: 8
   - phoneme: "\u0259"
     visemeIndex: 10
+    tensorIndex: 3
   - phoneme: "\u025B"
     visemeIndex: 11
+    tensorIndex: 8
   - phoneme: "\u025A"
     visemeIndex: 10
+    tensorIndex: 9
+  - phoneme: "a\u028A"
+    visemeIndex: 10
+    tensorIndex: 6
+  - phoneme: "a\u026A"
+    visemeIndex: 10
+    tensorIndex: 7
+  - phoneme: "e\u026A"
+    visemeIndex: 11
+    tensorIndex: 10
   - phoneme: "\u025D"
     visemeIndex: 11
+    tensorIndex: 9
   - phoneme: f
     visemeIndex: 2
+    tensorIndex: 21
   - phoneme: "\u0261"
     visemeIndex: 5
+    tensorIndex: 22
   - phoneme: h
     visemeIndex: -1
+    tensorIndex: 23
   - phoneme: i
     visemeIndex: 12
+    tensorIndex: 12
   - phoneme: "\u026A"
     visemeIndex: 12
+    tensorIndex: 11
   - phoneme: j
     visemeIndex: 12
+    tensorIndex: 38
   - phoneme: k
     visemeIndex: 5
+    tensorIndex: 25
   - phoneme: l
     visemeIndex: 8
+    tensorIndex: 26
   - phoneme: m
     visemeIndex: 1
+    tensorIndex: 27
   - phoneme: n
     visemeIndex: 8
+    tensorIndex: 28
   - phoneme: "\u014B"
     visemeIndex: 8
+    tensorIndex: 29
   - phoneme: o
     visemeIndex: 13
+    tensorIndex: 5
   - phoneme: "\u0254"
     visemeIndex: 13
+    tensorIndex: 5
   - phoneme: "\u0254\u026A"
     visemeIndex: 13
+    tensorIndex: 14
+  - phoneme: "o\u028A"
+    visemeIndex: 14
+    tensorIndex: 13
   - phoneme: p
     visemeIndex: 1
+    tensorIndex: 30
+  - phoneme: r
+    visemeIndex: 9
+    tensorIndex: 31
   - phoneme: "\u0279"
     visemeIndex: 9
+    tensorIndex: 31
   - phoneme: s
     visemeIndex: 7
+    tensorIndex: 32
   - phoneme: "\u0283"
     visemeIndex: 6
+    tensorIndex: 33
   - phoneme: t
     visemeIndex: 4
+    tensorIndex: 34
   - phoneme: "t\u0283"
     visemeIndex: 6
+    tensorIndex: 18
   - phoneme: u
     visemeIndex: 14
+    tensorIndex: 16
   - phoneme: "\u028A"
     visemeIndex: 14
+    tensorIndex: 15
   - phoneme: v
     visemeIndex: 2
+    tensorIndex: 36
   - phoneme: "\u028C"
     visemeIndex: 10
+    tensorIndex: 4
   - phoneme: w
     visemeIndex: 14
+    tensorIndex: 37
   - phoneme: z
     visemeIndex: 7
+    tensorIndex: 39
   - phoneme: "\u0292"
     visemeIndex: 6
+    tensorIndex: 40
   - phoneme: "\u03B8"
     visemeIndex: 3
+    tensorIndex: 35
   - phoneme: "\u0250"
     visemeIndex: 11
+    tensorIndex: 9
   - phoneme: "\u1D7B"
     visemeIndex: 12
+    tensorIndex: 12
   - phoneme: "\u027E"
     visemeIndex: 4
+    tensorIndex: 31
   - phoneme: "\u025C"
     visemeIndex: 11
+    tensorIndex: 9
   - phoneme: "\u0294"
-    visemeIndex: 4
+    visemeIndex: 8
+    tensorIndex: 29
   - phoneme: x
     visemeIndex: -1
+    tensorIndex: 23
   - phoneme: '-'
     visemeIndex: -1
+    tensorIndex: -1
   - phoneme: ','
     visemeIndex: -1
+    tensorIndex: -1
   - phoneme: ;
     visemeIndex: -1
+    tensorIndex: -1
   - phoneme: ':'
     visemeIndex: -1
+    tensorIndex: -1
   - phoneme: .
     visemeIndex: 0
+    tensorIndex: 0
   - phoneme: '?'
     visemeIndex: -1
+    tensorIndex: 0
   - phoneme: '!'
     visemeIndex: -1
+    tensorIndex: 0
   - phoneme: "\u02C8"
     visemeIndex: 0
+    tensorIndex: -1

Assets/Inworld/Inworld.Assets/Scripts/FeedbackCanvas.cs

@@ -64,6 +64,13 @@ public void Submit()
             m_Feedback.comment = m_InputField.text;
             m_InputField.text = "";
             InworldController.Client.SendFeedbackAsync(m_InteractionID, m_CorrelationID, m_Feedback);
+            if (!m_Feedback.isLike)
+            {
+                InworldController.CharacterHandler.CurrentCharacter.CancelResponse();
+                // TODO(Yan): Replace bubble to contain more info.
+                InworldController.Client.SendRegenerateEvent(InworldController.CharacterHandler.CurrentCharacter.ID,m_InteractionID); 
+            }
+
             m_Result.SetActive(true);
         }
 

Assets/Inworld/Inworld.Assets/Scripts/LipsyncMap.cs

@@ -6,6 +6,7 @@
 *************************************************************************************************/
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using UnityEngine;
 
 namespace Inworld.Assets
@@ -15,10 +16,12 @@ public class PhonemeToViseme
     {
         public string phoneme;
         public int visemeIndex;
+        public int tensorIndex;
     }
     public class LipsyncMap : ScriptableObject
     {
         public List<PhonemeToViseme> p2vMap;
+        public int TensorIndexOf(string targetPhoneme) => p2vMap.FirstOrDefault(p => p.phoneme == targetPhoneme)?.tensorIndex ?? -1;
     }
 
 }

Assets/Inworld/Inworld.NDK/Scripts/AEC/InworldAECAudioCapture.cs → Assets/Inworld/Inworld.Native/AEC/InworldAECAudioCapture.cs

@@ -42,14 +42,6 @@ public class InworldAECAudioCapture : AudioCapture
                                    || Application.platform == RuntimePlatform.OSXEditor
                                    || Application.platform == RuntimePlatform.OSXPlayer;
 
-        void Update()
-        {
-            m_IsAudioDebugging = Input.GetKey(m_DumpAudioHotKey);
-            if (!m_IsAudioDebugging)
-            {
-                _DumpAudioFiles();
-            }
-        }
         protected override void OnDestroy()
         {
             base.OnDestroy();
@@ -58,6 +50,15 @@ protected override void OnDestroy()
             AECInterop.WebRtcAec3_Free(m_AECHandle);
             m_AECHandle = IntPtr.Zero;
         }
+        protected override void HandlePTT()
+        {
+            base.HandlePTT();
+            m_IsAudioDebugging = Input.GetKey(m_DumpAudioHotKey);
+            if (!m_IsAudioDebugging)
+            {
+                _DumpAudioFiles();
+            }
+        }
         // YAN: Currently if you'd like to use AEC. The Audio Setting for output has to be 16000 sample rate, and mono.
         //      We'll add resampling features in the next update.
         protected override void Init()
@@ -71,7 +72,7 @@ protected override void Init()
             }
             else
                 m_SamplingMode = MicSampleMode.TURN_BASED;
-            m_LastSampleMode = m_SamplingMode;
+            m_InitSampleMode = m_SamplingMode;
             base.Init();
         }
         void _DumpAudioFiles()

Assets/Inworld/Inworld.NDK/InworldAI.NDK.asmdef → Assets/Inworld/Inworld.Native/InworldAI.NDK.asmdef

@@ -1,5 +1,5 @@
 {
-    "name": "InworldAI.NDK",
+    "name": "InworldAI.Native",
     "rootNamespace": "",
     "references": [
         "GUID:d967223c9a0ec784a869892a4c0859aa",