From 11447025b2af8f9f1df2a1b6290489073b623822 Mon Sep 17 00:00:00 2001
From: Stefan Kahl <kahst@hrz.tu-chemnitz.de>
Date: Tue, 31 Dec 2019 13:08:50 +0100
Subject: [PATCH] Spectrograms with wavesurfer

---
 README.md         |  1 +
 css/style.css     | 10 +++++++-
 index.html        | 24 +++++++++++++++++-
 js/birdnet.js     | 62 ++++++++++++++++++++++++++++++++++++++++++++++-
 js/ui.js          | 36 ++++++++++++++++++++++++++-
 main.js           |  3 +++
 package-lock.json |  5 ++++
 package.json      |  3 ++-
 8 files changed, 139 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index fd98c2a6..e7c0c5d5 100644
--- a/README.md
+++ b/README.md
@@ -75,4 +75,5 @@ This app also needs some additional packages that we have to install.
 npm install audio-loader
 npm install audio-resampler
 npm install array-normalize
+npm install wavesurfer.js
 ```
diff --git a/css/style.css b/css/style.css
index fd101e65..c4f3a6c5 100644
--- a/css/style.css
+++ b/css/style.css
@@ -1,3 +1,11 @@
+html, body {
+    height: 100%;
+}
+
 .border-10 {
     border-width:10px !important;
-}
\ No newline at end of file
+}
+
+.h-33 {height: 33%;}
+.h-40 {height: 40%;}
+.h-85 {height: 85%;}
\ No newline at end of file
diff --git a/index.html b/index.html
index 2b36465a..59081b5c 100644
--- a/index.html
+++ b/index.html
@@ -12,7 +12,8 @@
   <script>require('popper.js');</script>
   <script>require('bootstrap');</script>
   
-  <!-- Additional JavaScript -->
+  <!-- Additional JavaScript & CSS -->
+  <link rel="stylesheet" href="css/style.css" />
   <script src="js/birdnet.js"></script>  
   <script src="js/ui.js"></script>
 
@@ -44,4 +45,25 @@
       </ul>
     </div>
   </nav>
+
+  <!-- Load audio file hint -->
+  <div class="d-flex h-85 justify-content-center align-items-center" id="loadFileHint">
+    <p class="lead justify-content-center" id="loadFileHintText">Load an audio file for analysis by clicking</br>on <i>File > Open audio file</i> in the top menu.</p>
+    <div class="d-none spinner-border text-secondary" role="status" id="loadFileHintSpinner"></div>
+  </div>
+
+  <!-- Waveform view -->
+  <div class="" id="waveformContainer"></div>
+
+  <!-- Spectrogram view -->
+  <div class="d-none h-33" id="specContainer"></div>
+
+  <!-- Controls view -->
+  <div class="container-fluid bg-dark text-white d-none" id="controlsWrapper">
+    <div class="row p-2">
+      <div class="col-3">
+        <button type="button" class="btn btn-primary" onclick="WAVESURFER.playPause();">Play/Pause</button>
+      </div>
+    </div>
+  </div>
 </body>
\ No newline at end of file
diff --git a/js/birdnet.js b/js/birdnet.js
index e8d18853..7fd6bf27 100644
--- a/js/birdnet.js
+++ b/js/birdnet.js
@@ -3,6 +3,8 @@ const tf = require('@tensorflow/tfjs');
 const load = require('audio-loader')
 const resampler = require('audio-resampler');
 const normalize = require('array-normalize')
+const WaveSurfer = require('wavesurfer.js');
+const SpectrogramPlugin = require('wavesurfer.js/dist/plugin/wavesurfer.spectrogram.min.js');
 
 const CONFIG = {
 
@@ -11,6 +13,8 @@ const CONFIG = {
 }
 
 var AUDIO_DATA = [];
+var WAVESURFER = null;
+var CURRENT_ADUIO_BUFFER = null;
 
  /////////////////////////  DO AFTER LOAD ////////////////////////////
  window.onload = function () {
@@ -21,6 +25,12 @@ var AUDIO_DATA = [];
 
 function loadAudioFile(filePath) {
 
+    // Hide load hint and show spinnner
+    hideAll();
+    showElement('loadFileHint');
+    showElement('loadFileHintSpinner');
+
+
     // load one file
     load(filePath).then(function (buffer) {
 
@@ -33,10 +43,60 @@ function loadAudioFile(filePath) {
             // Normalize audio data
             AUDIO_DATA = normalize(AUDIO_DATA)
 
-            console.log(AUDIO_DATA);
+            //console.log(AUDIO_DATA);  
+
+            //Hide center div when done
+            hideElement('loadFileHint');
+            
+            // Draw and show spectrogram
+            drawSpectrogram(buffer);            
 
         });
 
     });
+    
+}
+
+function drawSpectrogram(audioBuffer) {
+
+    // Set global buffer
+    CURRENT_ADUIO_BUFFER = audioBuffer;
+
+    // Show waveform container
+    showElement('waveformContainer');
+
+    // Setup waveform and spec views
+    var options = {
+        container: '#waveformContainer',
+        plugins: [
+            SpectrogramPlugin.create({
+                container: '#specContainer',
+                fftSamples: 1024,
+                pixelRatio: 1,
+                labels: false,
+                name: 'specCanvas'
+            })
+        ]
+    };
+
+    // Create wavesurfer object
+    WAVESURFER = WaveSurfer.create(options);
+
+    // Load audio file
+    WAVESURFER.loadDecodedBuffer(CURRENT_ADUIO_BUFFER);
+
+    // Hide waveform view for now
+    hideElement('waveformContainer');
+    showElement('specContainer');
+
+    // Resize canvas of spec and labels
+    $('canvas').each(function() {
+        $( this ).height($('#specContainer').height());
+    });
+
+    // Show controls
+    showElement('controlsWrapper');
+
 }
 
+
diff --git a/js/ui.js b/js/ui.js
index bcf19893..6435a18d 100644
--- a/js/ui.js
+++ b/js/ui.js
@@ -19,4 +19,38 @@ function exitApplication() {
 
     remote.getCurrentWindow().close()
 
-}
\ No newline at end of file
+}
+
+function showElement(id) {
+
+    $('#' + id).removeClass('d-none');
+    $('#' + id).addClass('d-flex');
+
+}
+
+function hideElement(id) {
+    
+    $('#' + id).removeClass('d-flex');
+    $('#' + id).addClass('d-none');
+
+}
+
+function hideAll() {
+
+    // File hint div
+    hideElement('loadFileHint');
+    hideElement('loadFileHintText');
+    hideElement('loadFileHintSpinner');
+
+    // Waveform and spec
+    hideElement('waveformContainer');
+    hideElement('specContainer');
+
+    // Controls    
+    hideElement('controlsWrapper');
+
+}
+
+
+// Event listener
+//window.addEventListener('resize', drawSpectrogram(CURRENT_ADUIO_BUFFER), false);
\ No newline at end of file
diff --git a/main.js b/main.js
index 0d57160b..265bdcec 100644
--- a/main.js
+++ b/main.js
@@ -15,6 +15,9 @@ function createWindow () {
   // Set icon
   win.setIcon(__dirname + '/img/icon/icon.png');
 
+  // Always maximize
+  win.maximize()
+
   // Hide nav bar
   win.setMenuBarVisibility(false);
 
diff --git a/package-lock.json b/package-lock.json
index c6418b94..f098c058 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1504,6 +1504,11 @@
         "inline-worker": "^0.1.0"
       }
     },
+    "wavesurfer.js": {
+      "version": "3.3.0",
+      "resolved": "https://registry.npmjs.org/wavesurfer.js/-/wavesurfer.js-3.3.0.tgz",
+      "integrity": "sha512-Lfa5c2uN3ykVTxjkves+hcCpT+aePIr6aTKDgHXvIXIZo/ypC9KNd6kpclygnhEMCZo8VZA2ST/rq8UrBdyteQ=="
+    },
     "webaudioloader": {
       "version": "1.0.4",
       "resolved": "https://registry.npmjs.org/webaudioloader/-/webaudioloader-1.0.4.tgz",
diff --git a/package.json b/package.json
index 1bcfa5ad..691acc1f 100644
--- a/package.json
+++ b/package.json
@@ -32,6 +32,7 @@
     "audio-resampler": "^1.0.1",
     "bootstrap": "^4.4.1",
     "jquery": "^3.4.1",
-    "popper.js": "^1.16.0"
+    "popper.js": "^1.16.0",
+    "wavesurfer.js": "^3.3.0"
   }
 }