Working with DenseNet model, spectrogram bugs to iron out

Mattk70 · Jan 12, 2022 · c6dd874 · c6dd874
1 parent f90c027
commit c6dd874
Show file tree

Hide file tree

Showing 6 changed files with 87 additions and 2,231 deletions.
diff --git a/index.html b/index.html
@@ -4,7 +4,7 @@
   <!-- Required meta tags -->
   <meta charset="utf-8">
   <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
-  <meta http-equiv="Content-Security-Policy" content="script-src 'self' 'unsafe-inline';" /> 
+  <meta http-equiv="Content-Security-Policy" content="script-src 'self'  'unsafe-inline';" />
 
   <!-- Bootstrap & CSS-->
   <link rel="stylesheet" href="css/bootstrap.min.css" />
@@ -15,7 +15,7 @@
   <!-- Additional JavaScript & CSS -->
   <link rel="stylesheet" href="css/style.css" />
   <script src="js/labels.js"></script>  
-  <script src="js/birdnet.js"></script>  
+  <script src="js/chirpity.js"></script>
   <script src="js/ui.js"></script>
 
   <!-- Modified wavesurfer drawer -->
@@ -26,14 +26,14 @@
   <!--script src="js/wavesurfer.timeline.min.js"></script-->
 
   <!-- App title -->
-  <title>BirdNET Sound Analysis</title>
+  <title>Chirpity Sound Analysis</title>
 
 </head>
 <body>
 
   <!-- Navigation bar -->
   <nav class="navbar navbar-expand-sm navbar-dark bg-dark fixed-top">
-    <a class="navbar-brand" href="#">BirdNET</a>
+    <a class="navbar-brand" href="#">Chirpity</a>
     <button class="navbar-toggler" type="button" data-toggle="collapse" data-target="#navbarSupportedContent" aria-controls="navbarSupportedContent" aria-expanded="false" aria-label="Toggle navigation">
       <span class="navbar-toggler-icon"></span>
     </button>  
@@ -124,6 +124,6 @@
 
   <!-- Footer -->
   <div class="d-flex fixed-bottom p-2 bg-primary text-white justify-content-center" id="footer">
-    <small>BirdNET | Center for Conservation Bioacoustics - Cornell Lab of Ornithology | Chair Mediainformatics - Chemnitz University of Technology | <span id="year"></span></small>
+    <small>Chirpity | <span id="year"></span></small>
   </div>
 </body>
diff --git a/js/birdnet.js → js/chirpity.js b/js/birdnet.js → js/chirpity.js
@@ -5,13 +5,13 @@ const resampler = require('audio-resampler');
 const normalize = require('array-normalize')
 const colormap = require('colormap')
 
-const MODEL_JSON = 'model/model.json'
+const MODEL_JSON = '../Chirpity/JS_model/model.json'
 const CONFIG = {
 
     sampleRate: 48000,
     specLength: 3,
     sigmoid: 1.0,
-    minConfidence: 0.15,
+    minConfidence: 0.2,
 
 }
 
@@ -138,28 +138,73 @@ async function loadModel() {
     // Load model
     if (MODEL == null) {
         console.log('Loading model...');
-        MODEL = await tf.loadLayersModel(MODEL_JSON);
+        MODEL = await tf.loadGraphModel(MODEL_JSON);
         //CONFIG.labels = MODEL.getLayer('SIGMOID').config.labels;
         CONFIG.labels = LABELS;
         console.log('...done loading model!');
     }
 
 }
 
+function normalize_and_fix_shape(spec) {
+    spec = spec.slice(253, 256);
+    // Normalize to 0-255
+    spec = tf.div(
+       tf.sub(
+          spec,
+          tf.min(spec)
+       ),
+       tf.sub(
+          tf.max(spec),
+          tf.min(spec)
+       )
+    );
+    spec = spec.mul(255);
+    return spec
+}
+
 async function predict(audioData, model) {
 
     const audioTensor = tf.tensor1d(audioData)
     RESULTS = [];
 
     // Slice and expand
-    var cunkLength = CONFIG.sampleRate * CONFIG.specLength;
-    for (var i = 0; i < audioTensor.shape[0] - cunkLength; i += CONFIG.sampleRate) {
+    var chunkLength = CONFIG.sampleRate * CONFIG.specLength;
+    for (var i = 0; i < audioTensor.shape[0] - chunkLength; i += CONFIG.sampleRate) {
+
+        if (i + chunkLength > audioTensor.shape[0]) i = audioTensor.shape[0] - chunkLength;
+        const chunkTensor = audioTensor.slice(i, chunkLength).expandDims(0);
+        // For now, let's work with hard coded values to avoid strange errors when reading the config
+        // const spec_shape = [257, 384];
+        const frame_length = 1024;
+        const frame_step = 373;
 
-        if (i + cunkLength > audioTensor.shape[0]) i = audioTensor.shape[0] - cunkLength;
-        const chunkTensor = audioTensor.slice(i, cunkLength).expandDims(0);
+        // Perform STFT
+        let spec = tf.signal.stft(chunkTensor.squeeze(),
+                              frame_length,
+                              frame_step,
+                              )
+
+        // Cast from complex to float
+        spec = tf.cast(spec, 'float32');
 
+        // Swap axes to fit output shape
+        spec = tf.transpose(spec);
+        spec = tf.abs(spec);
+        // Fix Spectrogram shape
+        spec = normalize_and_fix_shape(spec);
+        // Add channel axis
+        spec = tf.expandDims(spec, -1);
+
+        // Add batch axis
+        spec = tf.expandDims(spec, 0);
+
+        //const verbose = true
+        //spec.print(verbose)
+
+        // console.log(spec.dataSync())
         // Make prediction
-        const prediction = model.predict(chunkTensor);
+        const prediction = model.predict(spec);
 
         // Get label
         const index = prediction.argMax(1).dataSync()[0];
@@ -170,7 +215,7 @@ async function predict(audioData, model) {
         if (score >= CONFIG.minConfidence) {
             RESULTS.push({
 
-                timestamp: timestampFromSeconds(i / CONFIG.sampleRate) + ' - ' + timestampFromSeconds((i + cunkLength) / CONFIG.sampleRate),
+                timestamp: timestampFromSeconds(i / CONFIG.sampleRate) + ' - ' + timestampFromSeconds((i + chunkLength) / CONFIG.sampleRate),
                 sname: CONFIG.labels[index].split('_')[0],
                 cname: CONFIG.labels[index].split('_')[1],
                 score: score
@@ -210,15 +255,15 @@ function loadAudioFile(filePath) {
             hideElement('loadFileHint');
 
             // Draw and show spectrogram
-            drawSpectrogram(buffer);    
+            drawSpectrogram(buffer);
 
             // Show results
             showResults();
 
         });
 
     });
-    
+
 }
 
 function drawSpectrogram(audioBuffer) {
@@ -243,6 +288,7 @@ function drawSpectrogram(audioBuffer) {
         responsive: true,
         height: 512,
         fftSamples: 1024,
+        windowFunc: 'hamming',
         minPxPerSec: 50,
         colorMap: colormap({
                 colormap: 'inferno',
@@ -252,7 +298,6 @@ function drawSpectrogram(audioBuffer) {
         }),
         hideScrollbar: false,
         visualization: 'spectrogram',
-        labels: true,
         plugins: []
     };
 

diff --git a/js/labels.js b/js/labels.js
diff --git a/main.js b/main.js
@@ -8,24 +8,26 @@ function createWindow () {
     width: 1280,
     height: 768,
     webPreferences: {
-      nodeIntegration: true
+      nodeIntegration: true,
+      contextIsolation: false,
+      enableRemoteModule: true,
     }
   })
 
   // Set icon
   win.setIcon(__dirname + '/img/icon/icon.png');
 
   // Always maximize
-  win.maximize()
+  // win.maximize()
 
   // Hide nav bar
   win.setMenuBarVisibility(false);
 
   // and load the index.html of the app.
   win.loadFile('index.html')
 
-  // Open the DevTools.
-  //win.webContents.openDevTools()
+  // Open the DevTools. Comment out for release
+  win.webContents.openDevTools()
 
   // Emitted when the window is closed.
   win.on('closed', () => {