Merge pull request #59 from nomalab/complete_loudness_results

Complete loudness results

.github/workflows/Dockerfile

@@ -0,0 +1,38 @@
+FROM ubuntu:22.04
+
+RUN apt-get update
+
+RUN apt-get install -y autoconf \
+                       automake \
+                       build-essential \
+                       clang \
+                       cmake \
+                       git-core \
+                       libfreetype6-dev \
+                       libgnutls28-dev \
+                       libsdl2-dev \
+                       libssl-dev \
+                       libtool \
+                       libva-dev \
+                       libvdpau-dev \
+                       libunistring-dev \
+                       libxcb1-dev \
+                       libxcb-shm0-dev \
+                       libxcb-xfixes0-dev \
+                       meson \
+                       ninja-build \
+                       pkg-config \
+                       texinfo \
+                       wget \
+                       yasm \
+                       zlib1g-dev
+
+# ffmpeg 5.0.1 libav => codec59.18.100 device59.4.100 filter8.24.100 format59.16.100 util57.17.100
+RUN wget -q http://ffmpeg.org/releases/ffmpeg-5.0.1.tar.gz && \
+    tar xf ffmpeg-5.0.1.tar.gz && \
+    cd ffmpeg-5.0.1 && \
+    PATH="/bin:$PATH" PKG_CONFIG_PATH="/usr/lib/pkgconfig" \
+    ./configure --pkg-config-flags="--static" --extra-cflags="-I/usr/include" --extra-ldflags="-L/usr/lib" --extra-libs="-lpthread -lm" --ld="g++" --bindir="/bin" \
+    --enable-gpl --enable-gnutls --enable-libfreetype --enable-nonfree && \
+    PATH="/bin:$PATH" make -j32 && \
+    make install

.github/workflows/main.yml

@@ -16,18 +16,22 @@ jobs:
   build_and_test:
     # The type of runner that the job will run on
     runs-on: ubuntu-22.04
+    container: nomalab/ffmpeg:5.0.1 # image made from Dockerfile in directory
 
     continue-on-error: ${{ (matrix.rust == 'beta') || (matrix.rust == 'nightly') }}
 
     strategy:
       fail-fast: false
       matrix:
         rust: [
-          1.69.0,
-          1.70.0,
           1.71.0,
           1.72.0,
           1.73.0,
+          1.74.0,
+          1.75.0,
+          1.76.0,
+          1.77.0,
+          1.78.0,
           stable,
           beta,
           nightly
@@ -37,18 +41,6 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Install libs
-        run: >-
-          sudo apt-get update &&
-          sudo apt-get install libasound2-dev libavcodec-dev
-          libavformat-dev libavutil-dev libavdevice-dev libavfilter-dev
-          libpostproc-dev libswscale-dev -y
-
-      - name: Setup FFmpeg
-        uses: Iamshankhadeep/[email protected]
-        with:
-          version: "5.0"
-
       - name: Install Rust
         uses: actions-rs/toolchain@v1
         with:
@@ -81,17 +73,11 @@ jobs:
 
   clippy:
     runs-on: ubuntu-22.04
+    container: nomalab/ffmpeg:5.0.1
 
     steps:
       - uses: actions/checkout@v3
 
-      - name: Install libs
-        run: >-
-          sudo apt-get update &&
-          sudo apt-get install libasound2-dev libavcodec-dev
-          libavformat-dev libavutil-dev libavdevice-dev libavfilter-dev
-          libpostproc-dev libswscale-dev -y
-
       - name: Install Rust with clippy
         uses: actions-rs/toolchain@v1
         with:
@@ -105,22 +91,13 @@ jobs:
 
   tarpaulin:
     runs-on: ubuntu-22.04
+    container:
+      image: nomalab/ffmpeg:5.0.1
+      options: --security-opt seccomp=unconfined
 
     steps:
       - uses: actions/checkout@v3
 
-      - name: Install libs
-        run: >-
-          sudo apt-get update &&
-          sudo apt-get install libasound2-dev libavcodec-dev
-          libavformat-dev libavutil-dev libavdevice-dev libavfilter-dev
-          libpostproc-dev libswscale-dev -y
-
-      - name: Setup FFmpeg
-        uses: Iamshankhadeep/[email protected]
-        with:
-          version: "5.0"
-
       - name: Install Rust
         uses: actions-rs/toolchain@v1
         with:

examples/deep_probe.rs

@@ -69,7 +69,7 @@ fn main() {
     };
     let spot_check = CheckParameterValue {
       min: None,
-      max: Some(3),
+      max: Some(5),
       num: None,
       den: None,
       th: None,

src/probe/deep.rs

@@ -84,10 +84,18 @@ pub struct OcrResult {
   pub word_confidence: String,
 }
 
+#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)]
+pub struct MinMax {
+  pub min: f64,
+  pub max: f64,
+}
+
 #[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)]
 pub struct LoudnessResult {
   pub integrated: f64,
   pub range: f64,
+  pub momentary: MinMax,
+  pub short_term: MinMax,
   pub true_peaks: Vec<f64>,
 }
 
@@ -130,7 +138,7 @@ pub struct StreamProbeResult {
   #[serde(skip_serializing_if = "Option::is_none")]
   pub detected_ocr: Option<Vec<OcrResult>>,
   #[serde(skip_serializing_if = "Option::is_none")]
-  pub detected_loudness: Option<Vec<LoudnessResult>>,
+  pub detected_loudness: Option<LoudnessResult>,
   #[serde(skip_serializing_if = "Option::is_none")]
   pub detected_dualmono: Option<Vec<DualMonoResult>>,
   #[serde(skip_serializing_if = "Option::is_none")]
@@ -194,6 +202,7 @@ pub struct VideoDetails {
   pub metadata_width: i32,
   pub metadata_height: i32,
   pub aspect_ratio: Rational,
+  pub sample_rate: i32,
 }
 
 #[derive(Default)]
@@ -303,8 +312,8 @@ impl StreamProbeResult {
       color_primaries: None,
       color_trc: None,
       color_matrix: None,
-      min_packet_size: std::i32::MAX,
-      max_packet_size: std::i32::MIN,
+      min_packet_size: i32::MAX,
+      max_packet_size: i32::MIN,
       detected_silence: None,
       silent_stream: None,
       detected_black: None,
@@ -360,6 +369,7 @@ impl VideoDetails {
       metadata_width: 0,
       metadata_height: 0,
       aspect_ratio: Rational::new(1, 1),
+      sample_rate: 0,
     }
   }
 }
@@ -430,6 +440,11 @@ impl DeepProbe {
             deep_orders.video_details.aspect_ratio = stream.get_picture_aspect_ratio();
           }
         }
+        if context.get_stream_type(stream_index as isize) == AVMediaType::AVMEDIA_TYPE_AUDIO {
+          if let Ok(stream) = Stream::new(context.get_stream(stream_index as isize)) {
+            deep_orders.video_details.sample_rate = stream.get_sample_rate();
+          }
+        }
       }
     }
 
@@ -624,6 +639,7 @@ impl DeepProbe {
               &mut deep_orders.streams,
               deep_orders.audio_indexes.clone(),
               params,
+              deep_orders.video_details.clone(),
             )
           }
         }

src/probe/loudness_detect.rs

@@ -4,7 +4,9 @@ use crate::order::{
   output::Output, output_kind::OutputKind, stream::Stream, Filter, Order, OutputResult::Entry,
   ParameterValue,
 };
-use crate::probe::deep::{CheckName, CheckParameterValue, LoudnessResult, StreamProbeResult};
+use crate::probe::deep::{
+  CheckName, CheckParameterValue, LoudnessResult, MinMax, StreamProbeResult, VideoDetails,
+};
 use ffmpeg_sys_next::log10;
 use std::collections::{BTreeMap, HashMap};
 
@@ -27,11 +29,12 @@ pub fn create_graph<S: ::std::hash::BuildHasher>(
   let mut outputs = vec![];
   let mut filters = vec![];
 
-  let metadata_param = ParameterValue::Bool(true);
+  let true_param = ParameterValue::Bool(true);
   let peak_param = ParameterValue::String("true".to_string());
   let mut loudnessdetect_params: HashMap<String, ParameterValue> = HashMap::new();
-  loudnessdetect_params.insert("metadata".to_string(), metadata_param);
+  loudnessdetect_params.insert("metadata".to_string(), true_param.clone());
   loudnessdetect_params.insert("peak".to_string(), peak_param);
+  loudnessdetect_params.insert("dualmono".to_string(), true_param);
 
   match params.get("pairing_list") {
     Some(pairing_list) => {
@@ -40,7 +43,12 @@ pub fn create_graph<S: ::std::hash::BuildHasher>(
           let mut amerge_params: HashMap<String, ParameterValue> = HashMap::new();
           let mut amerge_input = vec![];
           let mut input_streams_vec = vec![];
-          let mut lavfi_keys = vec!["lavfi.r128.I".to_string(), "lavfi.r128.LRA".to_string()];
+          let mut lavfi_keys = vec![
+            "lavfi.r128.I".to_string(),
+            "lavfi.r128.LRA".to_string(),
+            "lavfi.r128.S".to_string(),
+            "lavfi.r128.M".to_string(),
+          ];
           let output_label = format!("output_label_{iter:?}");
 
           for track in pair {
@@ -124,9 +132,22 @@ pub fn detect_loudness<S: ::std::hash::BuildHasher>(
   streams: &mut [StreamProbeResult],
   audio_indexes: Vec<u32>,
   params: HashMap<String, CheckParameterValue, S>,
+  video_details: VideoDetails,
 ) {
-  for index in audio_indexes {
-    streams[index as usize].detected_loudness = Some(vec![]);
+  for index in audio_indexes.clone() {
+    streams[index as usize].detected_loudness = Some(LoudnessResult {
+      range: -99.9,
+      integrated: -99.9,
+      true_peaks: vec![],
+      momentary: MinMax {
+        min: 99.9,
+        max: -99.9,
+      },
+      short_term: MinMax {
+        min: 99.9,
+        max: -99.9,
+      },
+    });
   }
   let results = output_results.get(&CheckName::Loudness).unwrap();
   info!("END OF LOUDNESS PROCESS");
@@ -144,24 +165,37 @@ pub fn detect_loudness<S: ::std::hash::BuildHasher>(
           .detected_loudness
           .as_mut()
           .unwrap();
-        let mut loudness = LoudnessResult {
-          range: -99.9,
-          integrated: -99.9,
-          true_peaks: vec![],
-        };
         let mut channel_start = 0;
         let mut channel_end = 0;
+        let mut pts_time: f32 = 0.0;
+        if let Some(pts) = entry_map.get("pts") {
+          pts_time = pts.parse::<f32>().unwrap() / video_details.sample_rate as f32;
+        }
 
         if let Some(value) = entry_map.get("lavfi.r128.I") {
-          let x = value.parse::<f64>().unwrap();
-          loudness.integrated = (x * 100.0).round() / 100.0;
-          if loudness.integrated == -70.0 {
-            loudness.integrated = -99.0;
+          let i = value.parse::<f64>().unwrap();
+          detected_loudness.integrated = (i * 100.0).round() / 100.0;
+          if detected_loudness.integrated == -70.0 {
+            detected_loudness.integrated = -99.0;
           }
         }
         if let Some(value) = entry_map.get("lavfi.r128.LRA") {
-          let y = value.parse::<f64>().unwrap();
-          loudness.range = (y * 100.0).round() / 100.0;
+          let lra = value.parse::<f64>().unwrap();
+          detected_loudness.range = (lra * 100.0).round() / 100.0;
+        }
+        if let Some(value) = entry_map.get("lavfi.r128.S") {
+          if pts_time >= 2.9 {
+            let s = (value.parse::<f64>().unwrap() * 100.0).round() / 100.0;
+            detected_loudness.short_term.min = f64::min(detected_loudness.short_term.min, s);
+            detected_loudness.short_term.max = f64::max(detected_loudness.short_term.max, s);
+          }
+        }
+        if let Some(value) = entry_map.get("lavfi.r128.M") {
+          if pts_time >= 0.3 {
+            let m = (value.parse::<f64>().unwrap() * 100.0).round() / 100.0;
+            detected_loudness.momentary.min = f64::min(detected_loudness.momentary.min, m);
+            detected_loudness.momentary.max = f64::max(detected_loudness.momentary.max, m);
+          }
         }
 
         match params.get("pairing_list") {
@@ -184,22 +218,23 @@ pub fn detect_loudness<S: ::std::hash::BuildHasher>(
           }
           None => warn!("No input message for the loudness analysis (audio qualification)"),
         }
+        let mut tpks = vec![];
         for i in channel_start..channel_end {
           let str_tpk_key = format!("lavfi.r128.true_peaks_ch{i}");
           if let Some(value) = entry_map.get(&str_tpk_key) {
             let energy = value.parse::<f64>().unwrap();
             unsafe {
               let mut tpk = 20.0 * log10(energy);
               tpk = (tpk * 100.0).round() / 100.0;
-              if tpk == std::f64::NEG_INFINITY {
+              if tpk == f64::NEG_INFINITY {
                 tpk = -99.00;
               }
-              loudness.true_peaks.push(tpk);
+              tpks.push(tpk)
             }
           }
         }
-        detected_loudness.drain(..);
-        detected_loudness.push(loudness);
+        detected_loudness.true_peaks.drain(..);
+        detected_loudness.true_peaks = tpks;
       }
     }
   }