Audio for Unity

This was quite the convoluted process and did not get implemented quite as expected, but it I eventually got to a solution which I think is good.

- I originally looked at using the [Native Audio Plugin SDK](https://docs.unity3d.com/Manual/AudioMixerNativeAudioPlugin.html) but that was wildly incompatible with a graphics native plugin 🫠
- Then I found the [AudioClip API](https://docs.unity3d.com/ScriptReference/AudioClip.Create.html) which has a PCMReaderCallback! It worked but it had a ton of latency (like you'd hear things half a second delayed).
  - I decreased the sample frame size, no change.
  - I lowered the DSP buffer size in Unity, no change.
  - Finally found [this post](https://forum.unity.com/threads/audioclip-pcmreadercallback-has-insane-latency.1022065/) that confirmed what I was seeing.
- That led me to the OnAudioFilterRead callback, which finally respects the DSP buffer size setting. Now there's no audible delay! Hover events with audio work great!

What's cool about this is that you can inject a different AudioEngine per artboard (nested artboards inherit it) or you can share one. Separating them means that the AudioSource (if you use one instead of an AudioListener on a camera) will have spatial audio! So you'll hear the Audio from the artboard fade away as you walk away from whatever GameObject you attach the script to.

Diffs=
61f553d6d Audio for Unity (#6606)

Co-authored-by: Luigi Rosso <[email protected]>

.rive_head

@@ -1 +1 @@
-562fc5c51f6ea36b510c8403418e80569079a042
+61f553d6d079a00c264c8668c07762aee46b9720

dependencies/premake5_miniaudio.lua

@@ -1,3 +1,2 @@
 local dependency = require('dependency')
--- miniaudio = dependency.github('rive-app/miniaudio', 'rive')
 miniaudio = dependency.github('rive-app/miniaudio', 'rive_changes')

dependencies/premake5_miniaudio_v2.lua

@@ -1,3 +1,3 @@
 dofile('rive_build_config.lua')
 local dependency = require('dependency')
-miniaudio = dependency.github('rive-app/miniaudio', 'rive')
+miniaudio = dependency.github('rive-app/miniaudio', 'rive_changes')

include/rive/artboard.hpp

@@ -11,6 +11,7 @@
 #include "rive/shapes/shape_paint_container.hpp"
 #include "rive/text/text_value_run.hpp"
 #include "rive/event.hpp"
+#include "rive/audio/audio_engine.hpp"
 
 #include <queue>
 #include <vector>
@@ -57,6 +58,10 @@ class Artboard : public ArtboardBase, public CoreContext, public ShapePaintConta
     bool m_IsInstance = false;
     bool m_FrameOrigin = true;
 
+#ifdef EXTERNAL_RIVE_AUDIO_ENGINE
+    rcp<AudioEngine> m_audioEngine;
+#endif
+
     void sortDependencies();
     void sortDrawOrder();
 
@@ -251,6 +256,11 @@ class Artboard : public ArtboardBase, public CoreContext, public ShapePaintConta
     void frameOrigin(bool value);
 
     StatusCode import(ImportStack& importStack) override;
+
+#ifdef EXTERNAL_RIVE_AUDIO_ENGINE
+    rcp<AudioEngine> audioEngine() const;
+    void audioEngine(rcp<AudioEngine> audioEngine);
+#endif
 };
 
 class ArtboardInstance : public Artboard

include/rive/audio/audio_engine.hpp

@@ -45,6 +45,7 @@ class AudioEngine : public RefCnt<AudioEngine>
 
 #ifdef EXTERNAL_RIVE_AUDIO_ENGINE
     bool readAudioFrames(float* frames, uint64_t numFrames, uint64_t* framesRead = nullptr);
+    bool sumAudioFrames(float* frames, uint64_t numFrames);
 #endif
 
 private:
@@ -57,6 +58,10 @@ class AudioEngine : public RefCnt<AudioEngine>
     void completeSound(rcp<AudioSound> sound);
     void purgeCompletedSounds();
     static void SoundCompleted(void* pUserData, ma_sound* pSound);
+
+#ifdef EXTERNAL_RIVE_AUDIO_ENGINE
+    std::vector<float> m_readFrames;
+#endif
 };
 } // namespace rive
 

src/artboard.cpp

@@ -846,3 +846,19 @@ std::unique_ptr<Scene> ArtboardInstance::defaultScene()
     }
     return scene;
 }
+
+#ifdef EXTERNAL_RIVE_AUDIO_ENGINE
+rcp<AudioEngine> Artboard::audioEngine() const { return m_audioEngine; }
+void Artboard::audioEngine(rcp<AudioEngine> audioEngine)
+{
+    m_audioEngine = audioEngine;
+    for (auto nestedArtboard : m_NestedArtboards)
+    {
+        auto artboard = nestedArtboard->artboard();
+        if (artboard != nullptr)
+        {
+            artboard->audioEngine(audioEngine);
+        }
+    }
+}
+#endif

src/audio/audio_engine.cpp

@@ -1,4 +1,5 @@
 #ifdef WITH_RIVE_AUDIO
+#include "rive/math/simd.hpp"
 #ifdef __APPLE__
 #include <TargetConditionals.h>
 #if TARGET_IPHONE_SIMULATOR || TARGET_OS_MACCATALYST || TARGET_OS_IPHONE
@@ -172,6 +173,45 @@ bool AudioEngine::readAudioFrames(float* frames, uint64_t numFrames, uint64_t* f
                                      (ma_uint64)numFrames,
                                      (ma_uint64*)framesRead) == MA_SUCCESS;
 }
+bool AudioEngine::sumAudioFrames(float* frames, uint64_t numFrames)
+{
+    size_t numChannels = (size_t)channels();
+    size_t count = (size_t)numFrames * numChannels;
+
+    if (m_readFrames.size() < count)
+    {
+        m_readFrames.resize(count);
+    }
+    ma_uint64 framesRead = 0;
+    if (ma_engine_read_pcm_frames(m_engine,
+                                  (void*)m_readFrames.data(),
+                                  (ma_uint64)numFrames,
+                                  &framesRead) != MA_SUCCESS)
+    {
+        return false;
+    }
+
+    count = framesRead * numChannels;
+
+    const size_t alignedCount = count - count % 4;
+    float* src = m_readFrames.data();
+    float* dst = frames;
+    float* srcEnd = src + alignedCount;
+
+    while (src != srcEnd)
+    {
+        float4 sum = simd::load4f(src) + simd::load4f(dst);
+        simd::store(dst, sum);
+
+        src += 4;
+        dst += 4;
+    }
+    for (size_t i = alignedCount; i < count; i++)
+    {
+        frames[i] += m_readFrames[i];
+    }
+    return true;
+}
 #endif
 
 #endif

src/audio_event.cpp

@@ -2,6 +2,7 @@
 #include "rive/assets/audio_asset.hpp"
 #include "rive/audio/audio_engine.hpp"
 #include "rive/audio/audio_sound.hpp"
+#include "rive/artboard.hpp"
 
 using namespace rive;
 
@@ -20,7 +21,13 @@ void AudioEvent::trigger(const CallbackData& value)
     {
         return;
     }
-    auto engine = AudioEngine::RuntimeEngine();
+
+    auto engine =
+#ifdef EXTERNAL_RIVE_AUDIO_ENGINE
+        artboard()->audioEngine() != nullptr ? artboard()->audioEngine() :
+#endif
+                                             AudioEngine::RuntimeEngine();
+
     engine->play(audioSource, engine->timeInFrames(), 0, 0);
 #endif
 }