Merge pull request #409 from claritychallenge/cad2-fix-evaluation

Fix Task1 evaluation

recipes/cad2/task1/baseline/evaluate.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import hashlib
 import json
 import logging
 from pathlib import Path
@@ -25,6 +26,13 @@
 logger = logging.getLogger(__name__)
 
 
+def set_song_seed(song: str) -> None:
+ """Set a seed that is unique for the given song"""
+ song_encoded = hashlib.md5(song.encode("utf-8")).hexdigest()
+ song_md5 = int(song_encoded, 16) % (10**8)
+ np.random.seed(song_md5)
+
+
 def make_scene_listener_list(scenes_listeners: dict, small_test: bool = False) -> list:
  """Make the list of scene-listener pairing to process
 
@@ -58,7 +66,7 @@ def compute_intelligibility(
  save_intermediate: bool = False,
  path_intermediate: str | Path | None = None,
  equiv_0db_spl: float = 100,
-) -> tuple[float, float]:
+) -> tuple[float, float, dict]:
  """
  Compute the Intelligibility score for the enhanced signal
  using the Whisper model.
@@ -79,6 +87,9 @@ def compute_intelligibility(
  Returns:
  The intelligibility score for the left and right channels
  """
+
+ lyrics = {}
+
  if path_intermediate is None:
  path_intermediate = Path.cwd()
  if isinstance(path_intermediate, str):
@@ -90,6 +101,7 @@ def compute_intelligibility(
  )
 
  reference = segment_metadata["text"]
+ lyrics["reference"] = reference
 
  # Compute left ear
  ear.set_audiogram(listener.audiogram_left)
@@ -101,8 +113,10 @@ def compute_intelligibility(
  44100,
  sample_rate,
  )
- hipothesis = scorer.transcribe(left_path, fp16=False)["text"]
- left_results = compute_measures(reference, hipothesis)
+ hypothesis = scorer.transcribe(left_path.as_posix(), fp16=False)["text"]
+ lyrics["hypothesis_left"] = hypothesis
+
+ left_results = compute_measures(reference, hypothesis)
 
  # Compute right ear
  ear.set_audiogram(listener.audiogram_right)
@@ -114,8 +128,10 @@ def compute_intelligibility(
  44100,
  sample_rate,
  )
- hipothesis = scorer.transcribe(right_path, fp16=False)["text"]
- right_results = compute_measures(reference, hipothesis)
+ hypothesis = scorer.transcribe(right_path.as_posix(), fp16=False)["text"]
+ lyrics["hypothesis_right"] = hypothesis
+
+ right_results = compute_measures(reference, hypothesis)
 
  # Compute the average score for both ears
  total_words = (
@@ -136,7 +152,11 @@ def compute_intelligibility(
  Path(left_path).unlink()
  Path(right_path).unlink()
 
- return left_results["hits"] / total_words, right_results["hits"] / total_words
+ return (
+ left_results["hits"] / total_words,
+ right_results["hits"] / total_words,
+ lyrics,
+ )
 
 
 def compute_quality(
@@ -203,14 +223,14 @@ def load_reference_signal(
 
 
 def normalise_luft(
- signal: np.ndarray, sample_rate: float, target_luft=-40
+ signal: np.ndarray, sample_rate: float, target_luft: float = -40.0
 ) -> np.ndarray:
  """
  Normalise the signal to a target loudness level.
  Args:
  signal: input signal to normalise
  sample_rate: sample rate of the signal
- target_luft: target loudness level in LUFS
+ target_luft: target loudness level in LUFS.
 
  Returns:
  np.ndarray: normalised signal
@@ -254,6 +274,9 @@ def run_compute_scores(config: DictConfig) -> None:
  "scene",
  "song",
  "listener",
+ "lyrics",
+ "hypothesis_left",
+ "hypothesis_right",
  "haaqi_left",
  "haaqi_right",
  "haaqi_avg",
@@ -302,6 +325,10 @@ def run_compute_scores(config: DictConfig) -> None:
 
  scene_id, listener_id = scene_listener_ids
 
+ # Set the random seed for the scene
+ if config.evaluate.set_random_seed:
+ set_song_seed(scene_id)
+
  # Load scene details
  scene = scenes[scene_id]
  listener = listener_dict[listener_id]
@@ -363,7 +390,7 @@ def run_compute_scores(config: DictConfig) -> None:
 
  # Compute the HAAQI and Whisper scores
  haaqi_scores = compute_quality(reference, enhanced_signal, listener, config)
- whisper_scores = compute_intelligibility(
+ whisper_left, whisper_right, lyrics_text = compute_intelligibility(
  enhanced_signal=enhanced_signal,
  segment_metadata=songs[scene["segment_id"]],
  scorer=intelligibility_scorer,
@@ -375,20 +402,24 @@ def run_compute_scores(config: DictConfig) -> None:
  equiv_0db_spl=config.evaluate.equiv_0db_spl,
  )
 
+ max_whisper = np.max([whisper_left, whisper_right])
+ mean_haaqi = np.mean(haaqi_scores)
  results_file.add_result(
  {
  "scene": scene_id,
  "song": songs[scene["segment_id"]]["track_name"],
  "listener": listener_id,
+ "lyrics": lyrics_text["reference"],
+ "hypothesis_left": lyrics_text["hypothesis_left"],
+ "hypothesis_right": lyrics_text["hypothesis_right"],
  "haaqi_left": haaqi_scores[0],
  "haaqi_right": haaqi_scores[1],
- "haaqi_avg": np.mean(haaqi_scores),
- "whisper_left": whisper_scores[0],
- "whisper_rigth": whisper_scores[1],
- "whisper_be": np.max(whisper_scores),
+ "haaqi_avg": mean_haaqi,
+ "whisper_left": whisper_left,
+ "whisper_rigth": whisper_right,
+ "whisper_be": max_whisper,
  "alpha": alpha,
- "score": alpha * np.max(whisper_scores)
- + (1 - alpha) * np.mean(haaqi_scores),
+ "score": alpha * max_whisper + (1 - alpha) * mean_haaqi,
  }
  )
 

recipes/cad2/task2/baseline/config.yaml

@@ -1,7 +1,5 @@
-# Zenodo download path: path to the zenodo download folder.
-# root: root path of the dataset. This path will contain the audio and metadata folders
 path:
- root: /media/gerardoroadabike/Extreme SSD1/Challenges/CAD2/cadenza_data/cad2/task2
+ root: ?? # Set to the root of the dataset
  metadata_dir: ${path.root}/metadata
  music_dir: ${path.root}/audio
  gains_file: ${path.metadata_dir}/gains.json