Audio engine

Adds an audio engine abstraction (implemented with miniaudio) in Rive. We can selectively replace it with other abstractions later if we find miniaudio isn't well suited everywhere but I'm confident it will be based on flexibility with getting it working in the recorder.

Adds audio support in:

- packages/rive_common
  - WASM: rive_audio_wasm.dart
  - FFI: rive_audio_ffi.dart
- packages/runtime
- packages/recorder

Subsequent PR will add support in:

- packages/rive_unity
  - This is getting meaty enough...
- packages/rive_flutter
  - This requires publishing rive_common so I want to make sure this is reviewed and accepted before publishing
  - I'd also prefer to merge layout constraints into rive_common before this lands.

Editor features:
- Updated preview window:
<img width="248" alt="CleanShot 2024-01-15 at 14 44 31@2x" src="https://github.com/rive-app/rive/assets/454182/a9588be6-8370-4e22-ab32-af1e9ed71183">

- Preview waveforms in the timeline:
<img width="651" alt="CleanShot 2024-01-15 at 14 44 53@2x" src="https://github.com/rive-app/rive/assets/454182/2710667f-838f-483d-9647-e2bcb9e0237a">

- Subsequent PR will also use threads in web editor build (currently uses a time/frame based decoder to offload ui). I can't do that until I can verify the Shared Memory access is granted once rive_common lands in pub.dev and then we can push to UAT.

Diffs=
73bf11db3 Audio engine (#6454)

Co-authored-by: Luigi Rosso <[email protected]>

.rive_head

@@ -1 +1 @@
-b098ad23a15cc7b11f144e407374c83d5db0fee0
+73bf11db39230e7a6d302da9a27414d35bcbee99

build.sh

@@ -32,8 +32,8 @@ if [ "$OPTION" = 'help' ]; then
 else
     build() {
         echo "Building Rive for platform=$platform option=$OPTION"
-        echo premake5 gmake2 --with_rive_text "$1"
-        PREMAKE="premake5 gmake2 --with_rive_text $1"
+        echo premake5 gmake2 --with_rive_text --with_rive_audio=system "$1"
+        PREMAKE="premake5 gmake2 --with_rive_text --with_rive_audio=system $1"
         eval "$PREMAKE"
         if [ "$OPTION" = "clean" ]; then
             make clean

build/premake5.lua

@@ -12,9 +12,19 @@ do
     defines {'WITH_RIVE_TEXT'}
 end
 filter {}
+filter {'options:with_rive_audio=system'}
+do
+    defines {'WITH_RIVE_AUDIO'}
+end
+filter {'options:with_rive_audio=external'}
+do
+    defines {'WITH_RIVE_AUDIO', 'EXTERNAL_RIVE_AUDIO_ENGINE', 'MA_NO_DEVICE_IO'}
+end
+filter {}
 
 dofile(path.join(path.getabsolute('../dependencies/'), 'premake5_harfbuzz.lua'))
 dofile(path.join(path.getabsolute('../dependencies/'), 'premake5_sheenbidi.lua'))
+dofile(path.join(path.getabsolute('../dependencies/'), 'premake5_miniaudio.lua'))
 
 project 'rive'
 do
@@ -26,13 +36,14 @@ do
     includedirs {
         '../include',
         harfbuzz .. '/src',
-        sheenbidi .. '/Headers'
+        sheenbidi .. '/Headers',
+        miniaudio
     }
 
     files {'../src/**.cpp'}
 
     flags {
-        'FatalCompileWarnings',
+        'FatalCompileWarnings'
     }
 
     filter {'system:macosx'}
@@ -43,12 +54,27 @@ do
         }
     end
 
-    filter {'toolset:not msc'}
+    -- filter {'toolset:not msc', 'files:../src/audio/audio_engine.cpp'}
+    filter {'system:not windows', 'files:../src/audio/audio_engine.cpp'}
     do
         buildoptions {
-            '-Wimplicit-int-conversion',
+            '-Wno-implicit-int-conversion'
         }
     end
+
+    filter {'system:windows', 'files:../src/audio/audio_engine.cpp'}
+    do
+        -- Too many warnings from miniaudio.h
+        removeflags {'FatalCompileWarnings'}
+    end
+
+    -- filter 'files:../src/audio/audio_engine.cpp'
+    -- do
+    --     buildoptions {
+    --         '-Wno-implicit-int-conversion'
+    --     }
+    -- end
+
     filter {'system:macosx', 'options:variant=runtime'}
     do
         buildoptions {
@@ -64,7 +90,8 @@ do
 
     filter {'system:ios'}
     do
-        buildoptions {'-flto=full'}
+        buildoptions {'-flto=full', '-Wno-implicit-int-conversion'}
+        files {'../src/audio/audio_engine.m'}
     end
 
     filter 'system:windows'
@@ -76,8 +103,7 @@ do
     filter {'system:ios', 'options:variant=system'}
     do
         buildoptions {
-            '-mios-version-min=13.0 -fembed-bitcode -arch arm64 -isysroot ' ..
-                (os.getenv('IOS_SYSROOT') or '')
+            '-mios-version-min=13.0 -fembed-bitcode -arch arm64 -isysroot ' .. (os.getenv('IOS_SYSROOT') or '')
         }
     end
 
@@ -121,9 +147,9 @@ do
         objdir '%{cfg.system}/arm64/obj/%{cfg.buildcfg}'
     end
 
-    filter "system:emscripten"
+    filter 'system:emscripten'
     do
-        buildoptions {"-pthread"}
+        buildoptions {'-pthread'}
     end
 
     filter 'configurations:debug'
@@ -161,3 +187,20 @@ newoption {
     trigger = 'with_rive_text',
     description = 'Compiles in text features.'
 }
+
+newoption {
+    trigger = 'with_rive_audio',
+    value = 'disabled',
+    description = 'The audio mode to use.',
+    allowed = {
+        {
+            'disabled'
+        },
+        {
+            'system'
+        },
+        {
+            'external'
+        }
+    }
+}

dependencies/premake5_miniaudio.lua

@@ -0,0 +1,2 @@
+local dependency = require 'dependency'
+miniaudio = dependency.github('rive-app/miniaudio', 'rive')

dev/defs/assets/audio_asset.json

@@ -0,0 +1,8 @@
+{
+    "name": "AudioAsset",
+    "key": {
+        "int": 406,
+        "string": "audioasset"
+    },
+    "extends": "assets/file_asset.json"
+}

dev/defs/audio_event.json

@@ -0,0 +1,21 @@
+{
+    "name": "AudioEvent",
+    "key": {
+        "int": 407,
+        "string": "audioevent"
+    },
+    "extends": "event.json",
+    "properties": {
+        "assetId": {
+            "type": "Id",
+            "typeRuntime": "uint",
+            "initialValue": "Core.missingId",
+            "initialValueRuntime": "-1",
+            "key": {
+                "int": 408,
+                "string": "assetid"
+            },
+            "description": "Audio asset to play when event fires"
+        }
+    }
+}

dev/test/premake5.lua

@@ -18,6 +18,7 @@ configurations {'debug'}
 
 dofile(path.join(path.getabsolute('../../dependencies/'), 'premake5_harfbuzz.lua'))
 dofile(path.join(path.getabsolute('../../dependencies/'), 'premake5_sheenbidi.lua'))
+dofile(path.join(path.getabsolute('../../dependencies/'), 'premake5_miniaudio.lua'))
 
 project('tests')
 do
@@ -28,13 +29,14 @@ do
     objdir 'build/obj/%{cfg.buildcfg}'
     flags {'FatalWarnings'}
     buildoptions {'-Wall', '-fno-exceptions', '-fno-rtti'}
-    exceptionhandling "On"
+    exceptionhandling 'On'
 
     includedirs {
         './include',
         '../../include',
         harfbuzz .. '/src',
-        sheenbidi .. '/Headers'
+        sheenbidi .. '/Headers',
+        miniaudio
     }
     links {
         'rive_harfbuzz',
@@ -47,14 +49,48 @@ do
         '../../utils/**.cpp' -- no_op utils
     }
 
-    defines {'TESTING', 'ENABLE_QUERY_FLAT_VERTICES', 'WITH_RIVE_TOOLS', 'WITH_RIVE_TEXT'}
+    defines {'TESTING', 'ENABLE_QUERY_FLAT_VERTICES', 'WITH_RIVE_TOOLS', 'WITH_RIVE_TEXT', 'WITH_RIVE_AUDIO'}
+
+    filter {'system:windows', 'files:../../src/audio/audio_engine.cpp'}
+    do
+        -- Too many warnings from miniaudio.h
+        removeflags {'FatalCompileWarnings'}
+        removebuildoptions {'-Wall'}
+    end
+
+    filter {'system:windows', 'toolset:clang'}
+    do
+        -- Too many warnings from miniaudio.h
+        buildoptions {
+            '-Wno-nonportable-system-include-path',
+            '-Wno-zero-as-null-pointer-constant',
+            '-Wno-missing-prototypes',
+            '-Wno-cast-qual',
+            '-Wno-format-nonliteral',
+            '-Wno-cast-align',
+            '-Wno-covered-switch-default',
+            '-Wno-comma',
+            '-Wno-tautological-type-limit-compare',
+            '-Wno-extra-semi-stmt',
+            '-Wno-tautological-constant-out-of-range-compare',
+            '-Wno-implicit-fallthrough',
+            '-Wno-implicit-int-conversion',
+            '-Wno-undef'
+        }
+    end
 
     filter 'configurations:debug'
     do
         defines {'DEBUG'}
         symbols 'On'
     end
 
+    filter 'system:linux'
+    do
+        defines {'EXTERNAL_RIVE_AUDIO_ENGINE'}
+        links {'dl', 'pthread'}
+    end
+
     filter 'system:windows'
     do
         removebuildoptions {

include/rive/animation/linear_animation_instance.hpp

@@ -2,12 +2,12 @@
 #define _RIVE_LINEAR_ANIMATION_INSTANCE_HPP_
 
 #include "rive/artboard.hpp"
+#include "rive/core/field_types/core_callback_type.hpp"
 #include "rive/scene.hpp"
 
 namespace rive
 {
 class LinearAnimation;
-class KeyedCallbackReporter;
 
 class LinearAnimationInstance : public Scene
 {

include/rive/animation/state_machine_instance.hpp

@@ -5,7 +5,7 @@
 #include <stddef.h>
 #include <vector>
 #include "rive/animation/linear_animation_instance.hpp"
-#include "rive/animation/keyed_callback_reporter.hpp"
+#include "rive/core/field_types/core_callback_type.hpp"
 #include "rive/listener_type.hpp"
 #include "rive/scene.hpp"
 
@@ -37,7 +37,7 @@ class EventReport
     float m_secondsDelay;
 };
 
-class StateMachineInstance : public Scene, public KeyedCallbackReporter
+class StateMachineInstance : public Scene
 {
     friend class SMIInput;
     friend class KeyedProperty;
@@ -110,20 +110,14 @@ class StateMachineInstance : public Scene, public KeyedCallbackReporter
     NestedArtboard* parentNestedArtboard() { return m_parentNestedArtboard; }
 
     /// Tracks an event that reported, will be cleared at the end of the next advance.
-    void reportEvent(Event* event, float secondsDelay = 0.0f);
+    void reportEvent(Event* event, float secondsDelay = 0.0f) override;
 
     /// Gets the number of events that reported since the last advance.
     std::size_t reportedEventCount() const;
 
     /// Gets a reported event at an index < reportedEventCount().
     const EventReport reportedEventAt(std::size_t index) const;
 
-    /// Report which time based events have elapsed on a timeline within this
-    /// state machine.
-    void reportKeyedCallback(uint32_t objectId,
-                             uint32_t propertyKey,
-                             float elapsedSeconds) override;
-
 private:
     std::vector<EventReport> m_reportedEvents;
     const StateMachine* m_machine;

include/rive/assets/audio_asset.hpp

@@ -0,0 +1,29 @@
+#ifndef _RIVE_AUDIO_ASSET_HPP_
+#define _RIVE_AUDIO_ASSET_HPP_
+#include "rive/generated/assets/audio_asset_base.hpp"
+#include "rive/audio/audio_source.hpp"
+
+namespace rive
+{
+class AudioAsset : public AudioAssetBase
+{
+public:
+    AudioAsset();
+    ~AudioAsset() override;
+    bool decode(SimpleArray<uint8_t>&, Factory*) override;
+    std::string fileExtension() const override;
+
+#ifdef WITH_RIVE_AUDIO
+#ifdef TESTING
+    bool hasAudioSource() { return m_audioSource != nullptr; }
+#endif
+
+    rcp<AudioSource> audioSource() { return m_audioSource; }
+
+private:
+    rcp<AudioSource> m_audioSource;
+#endif
+};
+} // namespace rive
+
+#endif

include/rive/assets/file_asset.hpp

@@ -3,6 +3,7 @@
 #include "rive/assets/file_asset_referencer.hpp"
 #include "rive/generated/assets/file_asset_base.hpp"
 #include "rive/span.hpp"
+#include "rive/simple_array.hpp"
 #include <string>
 
 namespace rive
@@ -20,7 +21,7 @@ class FileAsset : public FileAssetBase
 
     void decodeCdnUuid(Span<const uint8_t> value) override;
     void copyCdnUuid(const FileAssetBase& object) override;
-    virtual bool decode(Span<const uint8_t>, Factory*) = 0;
+    virtual bool decode(SimpleArray<uint8_t>&, Factory*) = 0;
     virtual std::string fileExtension() const = 0;
     StatusCode import(ImportStack& importStack) override;
     const std::vector<FileAssetReferencer*> fileAssetReferencers()

include/rive/assets/file_asset_contents.hpp

@@ -2,19 +2,20 @@
 #define _RIVE_FILE_ASSET_CONTENTS_HPP_
 #include "rive/generated/assets/file_asset_contents_base.hpp"
 #include <cstdint>
+#include "rive/simple_array.hpp"
 
 namespace rive
 {
 class FileAssetContents : public FileAssetContentsBase
 {
-private:
-    std::vector<uint8_t> m_Bytes;
-
 public:
-    Span<const uint8_t> bytes() const;
+    SimpleArray<uint8_t>& bytes();
     StatusCode import(ImportStack& importStack) override;
     void decodeBytes(Span<const uint8_t> value) override;
     void copyBytes(const FileAssetContentsBase& object) override;
+
+private:
+    SimpleArray<uint8_t> m_bytes;
 };
 } // namespace rive
 

include/rive/assets/font_asset.hpp

@@ -9,7 +9,7 @@ namespace rive
 class FontAsset : public FontAssetBase
 {
 public:
-    bool decode(Span<const uint8_t>, Factory*) override;
+    bool decode(SimpleArray<uint8_t>&, Factory*) override;
     std::string fileExtension() const override;
     const rcp<Font> font() const { return m_font; }
     void font(rcp<Font> font);

include/rive/assets/image_asset.hpp

@@ -3,6 +3,7 @@
 
 #include "rive/generated/assets/image_asset_base.hpp"
 #include "rive/renderer.hpp"
+#include "rive/simple_array.hpp"
 #include <string>
 
 namespace rive
@@ -19,7 +20,7 @@ class ImageAsset : public ImageAssetBase
 #ifdef TESTING
     std::size_t decodedByteSize = 0;
 #endif
-    bool decode(Span<const uint8_t>, Factory*) override;
+    bool decode(SimpleArray<uint8_t>&, Factory*) override;
     std::string fileExtension() const override;
     RenderImage* renderImage() const { return m_RenderImage.get(); }
     void renderImage(rcp<RenderImage> renderImage);