Add the ability to access FFT data from the previous frame

examples/fft_delta.glsl

@@ -0,0 +1,40 @@
+// Configure builtin uniforms
+// These macros are optional, but improve the user experience
+#pragma shaderfilter set main__mix__description Main Mix/Track
+#pragma shaderfilter set main__channel__description Main Channel
+#pragma shaderfilter set main__dampening_factor_attack 0.0
+#pragma shaderfilter set main__dampening_factor_release 0.0001
+uniform texture2d builtin_texture_fft_main;
+uniform texture2d builtin_texture_fft_main_previous;
+
+float remap(float x, vec2 from, vec2 to) {
+    float normalized = (x - from[0]) / (from[1] - from[0]);
+    return normalized * (to[1] - to[0]) + to[0];
+}
+
+float remap_amplitude(float fft_amplitude) {
+    float fft_db = 20.0 * log(fft_amplitude / 0.5) / log(10.0);
+
+    return remap(fft_db, vec2(-50, -0), vec2(0, 1));
+}
+
+bool below_db(vec2 uv, float fft_amplitude) {
+    return 1.0 - uv.y < remap_amplitude(fft_amplitude);
+}
+
+vec4 render(vec2 uv) {
+    vec3 color = image.Sample(builtin_texture_sampler, uv).rgb;
+    float fft_frequency = uv.x;
+    float fft_amplitude = builtin_texture_fft_main.Sample(builtin_texture_sampler, vec2(fft_frequency, 0.5)).r;
+    float fft_amplitude_previous = builtin_texture_fft_main_previous.Sample(builtin_texture_sampler, vec2(fft_frequency, 0.5)).r;
+    float value = float(below_db(uv, fft_amplitude));
+    float value_previous = float(below_db(uv, fft_amplitude_previous));
+
+    float difference = value - value_previous;
+    float rising = float(difference > 0);
+    float falling = float(difference < 0);
+
+    vec4 fft_color = vec4(falling, rising, 0.0, abs(difference));
+
+    return vec4(mix(color, fft_color.rgb, fft_color.a), 1.0);
+}

src/effect.rs

@@ -175,7 +175,18 @@ impl<T: EffectParamType> EffectParam<T> {
         }
     }
 
+    pub fn stage_value_custom<'a>(&mut self, value: GraphicsContextDependentDisabled<<<T as EffectParamType>::ShaderParamType as ShaderParamType>::RustType>, graphics_context: &'a GraphicsContext) {
+        if let Some(previous) = self.staged_value.replace(value) {
+            previous.enable(graphics_context);
+        }
+    }
+
+    pub fn take_staged_value(&mut self) -> Option<GraphicsContextDependentDisabled<<<T as EffectParamType>::ShaderParamType as ShaderParamType>::RustType>> {
+        self.staged_value.take()
+    }
+
     /// Assigns the staged value to the effect current filter.
+    /// Keeps the staged value around.
     pub fn assign_value<'a>(&mut self, context: &'a FilterContext) {
         let staged_value = self.staged_value.as_ref()
             .expect("Tried to assign a value before staging it.")
@@ -188,6 +199,12 @@ impl<T: EffectParamType> EffectParam<T> {
         );
     }
 
+    pub fn assign_value_if_staged<'a>(&mut self, context: &'a FilterContext) {
+        if self.staged_value.is_some() {
+            self.assign_value(context);
+        }
+    }
+
     fn enable_and_drop(self, graphics_context: &GraphicsContext) {
         self.param.enable(graphics_context);
         if let Some(staged_value) = self.staged_value {
@@ -986,6 +1003,7 @@ where T: LoadedValueTypePropertyDescriptor,
 
 pub struct EffectParamCustomFFT {
     pub effect_param: EffectParamTexture,
+    pub effect_param_previous: Option<EffectParamTexture>,
     pub audio_fft: Arc<GlobalStateAudioFFT>,
     pub property_mix: LoadedValueTypeProperty<LoadedValueTypePropertyDescriptorI32>,
     pub property_channel: LoadedValueTypeProperty<LoadedValueTypePropertyDescriptorI32>,
@@ -996,6 +1014,7 @@ pub struct EffectParamCustomFFT {
 impl EffectParamCustomFFT {
     fn new<'a>(
         param: GraphicsContextDependentEnabled<'a, GraphicsEffectParamTyped<ShaderParamTypeTexture>>,
+        param_previous: Option<GraphicsContextDependentEnabled<'a, GraphicsEffectParamTyped<ShaderParamTypeTexture>>>,
         identifier: &str,
         settings: &mut SettingsContext,
         preprocess_result: &PreprocessResult,
@@ -1076,6 +1095,7 @@ impl EffectParamCustomFFT {
 
         Ok(Self {
             effect_param: EffectParam::new(param.disable()),
+            effect_param_previous: param_previous.map(|param_previous| EffectParam::new(param_previous.disable())),
             audio_fft: GLOBAL_STATE.request_audio_fft(&audio_fft_descriptor),
             property_mix,
             property_channel,
@@ -1115,14 +1135,26 @@ impl EffectParamCustomFFT {
     }
 
     fn stage_value<'a>(&mut self, graphics_context: &'a GraphicsContext) {
+        if let Some(effect_param_previous) = self.effect_param_previous.as_mut() {
+            if let Some(previous_texture_fft) = self.effect_param.take_staged_value() {
+                effect_param_previous.stage_value_custom(previous_texture_fft, graphics_context);
+            }
+        }
+
         self.effect_param.stage_value(graphics_context);
     }
 
     fn assign_value<'a>(&mut self, graphics_context: &'a FilterContext) {
-        self.effect_param.assign_value(graphics_context);
+        if let Some(effect_param_previous) = self.effect_param_previous.as_mut() {
+            effect_param_previous.assign_value_if_staged(graphics_context);
+        }
+        self.effect_param.assign_value_if_staged(graphics_context);
     }
 
     fn enable_and_drop(self, graphics_context: &GraphicsContext) {
+        if let Some(effect_param_previous) = self.effect_param_previous {
+            effect_param_previous.enable_and_drop(graphics_context);
+        }
         self.effect_param.enable_and_drop(graphics_context);
     }
 }
@@ -1181,50 +1213,86 @@ impl EffectParamsCustom {
     pub fn add_param<'a>(
         &mut self,
         param: GraphicsContextDependentEnabled<'a, GraphicsEffectParam>,
+        param_name: &str,
+        settings: &mut SettingsContext,
+        preprocess_result: &PreprocessResult,
+    ) -> Result<(), Cow<'static, str>> {
+        use ShaderParamTypeKind::*;
+
+        Ok(match param.param_type() {
+            Unknown => throw!("Cannot add an effect param of unknown type. Make sure to use HLSL type names for uniform variables."),
+            Bool  => self.params_bool.push(EffectParamCustomBool::new(param.downcast().unwrap(), &param_name, settings, preprocess_result)?),
+            Float => self.params_float.push(EffectParamCustomFloat::new(param.downcast().unwrap(), &param_name, settings, preprocess_result)?),
+            Int   => self.params_int.push(EffectParamCustomInt::new(param.downcast().unwrap(), &param_name, settings, preprocess_result)?),
+            Vec4  => self.params_vec4.push(EffectParamCustomColor::new(param.downcast().unwrap(), &param_name, settings, preprocess_result)?),
+            Vec2 | Vec3 | IVec2 | IVec3 | IVec4 | Mat4 => {
+                throw!("Multi-component types as effect params are not yet supported.");
+            },
+            String => throw!("Strings as effect params are not yet supported."),
+            Texture => throw!("Textures as effect params are not yet supported."),
+        })
+    }
+
+    pub fn add_params<'a>(
+        &mut self,
+        mut params: HashMap<String, GraphicsContextDependentEnabled<'a, GraphicsEffectParam>>,
         settings: &mut SettingsContext,
         preprocess_result: &PreprocessResult,
     ) -> Result<(), Cow<'static, str>> {
         use ShaderParamTypeKind::*;
 
-        let param_name = param.name().to_string();
         let result: Result<(), Cow<'static, str>> = try {
             {
                 let pattern_builtin_texture_fft = Regex::new(r"^builtin_texture_fft_(?P<field>\w+)$").unwrap();
-
-                if let Some(captures) = pattern_builtin_texture_fft.captures(&param_name) {
+                let pattern_field_previous = Regex::new(r"^.*_previous$").unwrap();
+                let param_names = params.keys().cloned().collect::<Vec<_>>();
+
+                for param_name in &param_names {
+                    let captures = if let Some(captures) = pattern_builtin_texture_fft.captures(&param_name) {
+                        captures
+                    } else {
+                        continue;
+                    };
                     let field_name = captures.name("field").unwrap().as_str();
 
+                    if pattern_field_previous.is_match(&field_name) {
+                        continue;
+                    }
+
+                    let param = params.remove(param_name).unwrap();
+                    let param_previous = params.remove(&format!("{}_previous", param_name));
+
                     if param.param_type() != Texture {
                         throw!(format!("Builtin field `{}` must be of type `{}`", field_name, "texture2d"));
                     }
 
+                    if let Some(ref param_previous) = param_previous.as_ref() {
+                        if param_previous.param_type() != Texture {
+                            throw!(format!("Builtin field `{}` must be of type `{}`", field_name, "texture2d"));
+                        }
+                    }
+
                     self.params_fft.push(EffectParamCustomFFT::new(
-                        param.downcast().unwrap(),
-                        field_name,
-                        settings,
-                        preprocess_result,
+                            param.downcast().unwrap(),
+                            param_previous.map(|param_previous| param_previous.downcast().unwrap()),
+                            field_name,
+                            settings,
+                            preprocess_result,
                     )?);
-                    return Ok(());
                 }
             }
-
-            match param.param_type() {
-                Unknown => throw!("Cannot add an effect param of unknown type. Make sure to use HLSL type names for uniform variables."),
-                Bool  => self.params_bool.push(EffectParamCustomBool::new(param.downcast().unwrap(), &param_name, settings, preprocess_result)?),
-                Float => self.params_float.push(EffectParamCustomFloat::new(param.downcast().unwrap(), &param_name, settings, preprocess_result)?),
-                Int   => self.params_int.push(EffectParamCustomInt::new(param.downcast().unwrap(), &param_name, settings, preprocess_result)?),
-                Vec4  => self.params_vec4.push(EffectParamCustomColor::new(param.downcast().unwrap(), &param_name, settings, preprocess_result)?),
-                Vec2 | Vec3 | IVec2 | IVec3 | IVec4 | Mat4 => {
-                    throw!("Multi-component types as effect params are not yet supported.");
-                },
-                String => throw!("Strings as effect params are not yet supported."),
-                Texture => throw!("Textures as effect params are not yet supported."),
-            }
         };
 
-        result.map_err(|err| {
-            Cow::Owned(format!("An error occurred while binding effect uniform variable `{}`: {}", param_name, err))
-        })
+        result?;
+
+        for (param_name, param) in params {
+            self.add_param(param, &param_name, settings, preprocess_result)
+                .map_err(|err| {
+                    Cow::Owned(format!("An error occurred while binding effect uniform variable `{}`: {}", param_name, err))
+                })?;
+        }
+
+        Ok(())
     }
 }
 

src/lib.rs

@@ -643,13 +643,17 @@ impl UpdateSource<Data> for ScrollFocusFilter {
                 custom: Default::default(),
             };
 
-            let custom_params_iter = effect.params_iter().filter(|item| {
-                !builtin_param_names.contains(&item.name())
-            });
+            let custom_params = effect.params_iter()
+                .filter(|item| {
+                    !builtin_param_names.contains(&item.name())
+                })
+                .map(|param| {
+                    (param.name().to_string(), param)
+                })
+                // FIXME: Iterating over a hashmap does not guarantee an order
+                .collect::<HashMap<_, _>>();
 
-            for custom_param in custom_params_iter {
-                params.custom.add_param(custom_param, settings, &preprocess_result)?;
-            }
+            params.custom.add_params(custom_params, settings, &preprocess_result)?;
 
             // Drop old effect before the new one is created.
             if let Some(old_effect) = data.effect.take() {