Skip to content

Commit

Permalink
Spectrogram interactions
Browse files Browse the repository at this point in the history
  • Loading branch information
kahst committed Jan 2, 2020
1 parent a199ab6 commit 894142d
Show file tree
Hide file tree
Showing 10 changed files with 1,030 additions and 41 deletions.
1 change: 0 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,5 @@ This app also needs some additional packages that we have to install.
npm install audio-loader
npm install audio-resampler
npm install array-normalize
npm install wavesurfer.js
npm install colormap
```
30 changes: 29 additions & 1 deletion css/style.css
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,32 @@ html, body {

.h-33 {height: 33%;}
.h-40 {height: 40%;}
.h-85 {height: 85%;}
.h-85 {height: 85%;}

@font-face {
font-family: 'Material Icons';
font-style: normal;
font-weight: 400;
src: local('Material Icons'),
local('MaterialIcons-Regular'),
url(../fonts/MaterialIcons-Regular.woff2) format('woff2')
}

.material-icons {
font-family: 'Material Icons';
font-weight: normal;
font-style: normal;
font-size: 24px; /* Preferred icon size */
display: inline-block;
line-height: 1;
text-transform: none;
letter-spacing: normal;
word-wrap: normal;
white-space: nowrap;
direction: ltr;
vertical-align: middle;
padding-bottom: 0px;

/* Support for all WebKit browsers. */
-webkit-font-smoothing: antialiased;
}
Binary file added fonts/MaterialIcons-Regular.woff2
Binary file not shown.
26 changes: 21 additions & 5 deletions index.html
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,11 @@
<script src="js/birdnet.js"></script>
<script src="js/ui.js"></script>

<!-- Modified wavesurfer drawer -->
<script src="js/wavesurfer.min.js"></script>
<script src="js/wavesurfer.spectrogram.min.js"></script>
<script src="js/wavesurfer.drawer.extended.js"></script>

<!-- App title -->
<title>BirdNET Sound Analysis</title>

Expand Down Expand Up @@ -67,13 +72,24 @@
</div>

<!-- Spectrogram view -->
<div class="d-none h-33" id="specContainer" onclick="WAVESURFER.playPause();"></div>

<!-- Waveform view -->
<div class="" id="waveformContainer"></div>
<div class="" id="specContainer"></div>

<!-- Controls view -->
<!--div class="container-fluid bg-dark text-white d-none" style="height: 5px;" id="controlsWrapper"-->
<div class="container-fluid p-2 bg-dark text-white d-none" id="controlsWrapper">

<div class="btn-group btn-group" role="group">
<button class="btn btn-primary " onclick="WAVESURFER.play()">
<i class="material-icons">play_arrow</i><!--span>Play</span-->
</button>
<button class="btn btn-primary " onclick="WAVESURFER.pause()">
<i class="material-icons">pause</i><!--span>Pause</span-->
</button>
<button class="btn btn-primary " onclick="zoomSpecIn();">
<i class="material-icons">zoom_in</i>
</button>
<button class="btn btn-primary " onclick="zoomSpecOut();">
<i class="material-icons">zoom_out</i>
</button>
</div>
</div>
</body>
74 changes: 43 additions & 31 deletions js/birdnet.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,6 @@ const tf = require('@tensorflow/tfjs');
const load = require('audio-loader')
const resampler = require('audio-resampler');
const normalize = require('array-normalize')
const WaveSurfer = require('wavesurfer.js');
const SpectrogramPlugin = require('wavesurfer.js/dist/plugin/wavesurfer.spectrogram.min.js');
const colormap = require('colormap')

const MODEL_JSON = 'model/model.json'
Expand All @@ -20,6 +18,7 @@ let MODEL = null;
var AUDIO_DATA = [];
var WAVESURFER = null;
var CURRENT_ADUIO_BUFFER = null;
var WS_ZOOM = 0;

///////////////////////// Build SimpleSpecLayer /////////////////////////
class SimpleSpecLayer extends tf.layers.Layer {
Expand Down Expand Up @@ -54,6 +53,9 @@ class SimpleSpecLayer extends tf.layers.Layer {
// Convert magnitudes using nonlinearity
spec = tf.pow(spec, tf.div(1.0, tf.add(1.0, tf.exp(this.mag_scale.read()))))

// Normalize values between 0 and 1
spec = tf.div(tf.sub(spec, tf.min(spec)), tf.max(spec));

// Swap axes to fit output shape
spec = tf.transpose(spec)

Expand Down Expand Up @@ -191,8 +193,6 @@ function loadAudioFile(filePath) {
// Normalize audio data
AUDIO_DATA = normalize(AUDIO_DATA)

//console.log(AUDIO_DATA);

// Predict
//predict(AUDIO_DATA, MODEL);

Expand All @@ -214,53 +214,65 @@ function drawSpectrogram(audioBuffer) {
CURRENT_ADUIO_BUFFER = audioBuffer;

// Show waveform container
showElement('waveformContainer', false, true);
showElement('specContainer', false, true);

// Setup waveform and spec views
var options = {
container: '#waveformContainer',
container: '#specContainer',
backgroundColor: '#363a40',
height: 50,
responsive: true,
waveColor: '#fff',
cursorColor: '#fff',
progressColor: '#4b79fa',
mediaControls: true,
plugins: [
SpectrogramPlugin.create({
container: '#specContainer',
fftSamples: 1024,
pixelRatio: 1,
labels: false,
colorMap: colormap({
colormap: 'viridis',
nshades: 256,
format: 'float'
})
})
]
cursorWidth: 2,
normalize: true,
fillParent: true,
responsive: true,
height: 512,
fftSamples: 1024,
minPxPerSec: 50,
colorMap: colormap({
colormap: 'viridis',
nshades: 256,
format: 'rgb',
alpha: 1
}),
hideScrollbar: false,
visualization: 'spectrogram',
plugins: []
};

// Create wavesurfer object
WAVESURFER = WaveSurfer.create(options);

// Load audio file
WAVESURFER.loadDecodedBuffer(CURRENT_ADUIO_BUFFER);

WS_ZOOM = WAVESURFER.params.minPxPerSec;

// Hide waveform view for now
//hideElement('waveformContainer');
showElement('specContainer');

// Resize canvas of spec and labels
$('#specContainer canvas').each(function() {
$( this ).height($('#specContainer').height());
});
$('spectrogram').each(function() {
$( this ).height($('#specContainer').height());
$('#specContainer wave, canvas').each(function() {
$( this ).height(400);
});

// Show controls
//showElement('controlsWrapper');
showElement('controlsWrapper');

}

function zoomSpecIn() {

console.log(WAVESURFER.params.minPxPerSec);
console.log(WS_ZOOM);
WS_ZOOM += 25;
WAVESURFER.zoom(WS_ZOOM);
//WAVESURFER.drawBuffer();
}

function zoomSpecOut() {

WS_ZOOM -= 25;
WAVESURFER.zoom(WS_ZOOM);

}

Expand Down
175 changes: 175 additions & 0 deletions js/wavesurfer.drawer.extended.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,175 @@
/*! wavesurfer.js 1.1.1 (Mon, 04 Apr 2016 09:49:47 GMT)
* https://github.com/katspaugh/wavesurfer.js
* @license CC-BY-3.0 */

'use strict';

/**
* Purpose:
* Add methods getFrequencyRGB, getFrequencies, resample, drawSpectrogram
* to WaveSurfer.Drawer.Canvas. These methods are modified versions from the the
* spectrogram plugin (https://github.com/katspaugh/wavesurfer.js/blob/master/plugin/wavesurfer.spectrogram.js)
* to allow the wavesurfer drawer to draw a spectrogram representation when this.params.visualization is
* set to "spectrogram"
* Dependencies:
* WaveSurfer (lib/wavesurfer.min.js & lib/wavesurfer.spectrogram.min.js)
*/
WaveSurfer.util.extend(WaveSurfer.Drawer.Canvas, {

// Takes in integer 0-255 and maps it to rgb string
getFrequencyRGB: function(colorValue) {
if (this.params.colorMap) {
// If the wavesurfer has a specified colour map
var rgb = this.params.colorMap[colorValue];
return 'rgb(' + rgb[0] + ',' + rgb[1] + ',' + rgb[2] + ')';
} else {
// If not just use gray scale
return 'rgb(' + colorValue + ',' + colorValue + ',' + colorValue + ')';
}

},

getFrequencies: function(buffer) {
var fftSamples = this.params.fftSamples || 512;
var channelOne = Array.prototype.slice.call(buffer.getChannelData(0));
var bufferLength = buffer.length;
var sampleRate = buffer.sampleRate;
var frequencies = [];

if (! buffer) {
this.fireEvent('error', 'Web Audio buffer is not available');
return;
}

var noverlap = this.params.noverlap;
if (! noverlap) {
var uniqueSamplesPerPx = buffer.length / this.width;
noverlap = Math.max(0, Math.round(fftSamples - uniqueSamplesPerPx));
}

var fft = new WaveSurfer.FFT(fftSamples, sampleRate);

var maxSlicesCount = Math.floor(bufferLength/ (fftSamples - noverlap));

var currentOffset = 0;

while (currentOffset + fftSamples < channelOne.length) {
var segment = channelOne.slice(currentOffset, currentOffset + fftSamples);
var spectrum = fft.calculateSpectrum(segment);
var length = fftSamples / 2 + 1;
var array = new Uint8Array(length);
for (var j = 0; j < length; j++) {
array[j] = Math.max(-255, Math.log10(spectrum[j])*45);
}
frequencies.push(array);
currentOffset += (fftSamples - noverlap);
}

return frequencies;
},

resample: function(oldMatrix) {
var columnsNumber = this.width;
var newMatrix = [];

var oldPiece = 1 / oldMatrix.length;
var newPiece = 1 / columnsNumber;

for (var i = 0; i < columnsNumber; i++) {
var column = new Array(oldMatrix[0].length);

for (var j = 0; j < oldMatrix.length; j++) {
var oldStart = j * oldPiece;
var oldEnd = oldStart + oldPiece;
var newStart = i * newPiece;
var newEnd = newStart + newPiece;

var overlap = (oldEnd <= newStart || newEnd <= oldStart) ?
0 :
Math.min(Math.max(oldEnd, newStart), Math.max(newEnd, oldStart)) -
Math.max(Math.min(oldEnd, newStart), Math.min(newEnd, oldStart));

if (overlap > 0) {
for (var k = 0; k < oldMatrix[0].length; k++) {
if (column[k] == null) {
column[k] = 0;
}
column[k] += (overlap / newPiece) * oldMatrix[j][k];
}
}
}

var intColumn = new Uint8Array(oldMatrix[0].length);

for (var k = 0; k < oldMatrix[0].length; k++) {
intColumn[k] = column[k];
}

newMatrix.push(intColumn);
}

return newMatrix;
},

drawSpectrogram: function (buffer) {
var pixelRatio = this.params.pixelRatio;
var length = buffer.duration;
var height = (this.params.fftSamples / 2) * pixelRatio;
var frequenciesData = this.getFrequencies(buffer);

var pixels = this.resample(frequenciesData);

var heightFactor = pixelRatio;

for (var i = 0; i < pixels.length; i++) {
for (var j = 0; j < pixels[i].length; j++) {
this.waveCc.fillStyle = this.getFrequencyRGB(pixels[i][j]);
this.waveCc.fillRect(i, height - j * heightFactor, 1, heightFactor);
}
}
}
});

/**
* Override the method WaveSurfer.drawBuffer to pass in the this.backend.buffer to
* WaveSurfer.Drawer.drawPeaks since the buffer is needed to draw the spectrogram
*/
WaveSurfer.util.extend(WaveSurfer, {
drawBuffer: function () {
var nominalWidth = Math.round(
this.getDuration() * this.params.minPxPerSec * this.params.pixelRatio
);
var parentWidth = this.drawer.getWidth();
var width = nominalWidth;

// Fill container
if (this.params.fillParent && (!this.params.scrollParent || nominalWidth < parentWidth)) {
width = parentWidth;
}

var peaks = this.backend.getPeaks(width);
this.drawer.drawPeaks(peaks, width, this.backend.buffer);
this.fireEvent('redraw', peaks, width);
},
});

/**
* Override the methods WaveSurfer.Drawer.drawPeaks to support invisible and
* spectrogram representations
*/
WaveSurfer.util.extend(WaveSurfer.Drawer, {
drawPeaks: function (peaks, length, buffer) {
this.resetScroll();
this.setWidth(length);
var visualization = this.params.visualization;
if (visualization === 'invisible') {
//draw nothing
} else if (visualization === 'spectrogram' && buffer) {
this.drawSpectrogram(buffer);
} else {
this.params.barWidth ?
this.drawBars(peaks) :
this.drawWave(peaks);
}
}
});
Loading

0 comments on commit 894142d

Please sign in to comment.