From 8d8b6fda642cea5cb060a9261d0678efa0f80e9e Mon Sep 17 00:00:00 2001 From: Mattk70 Date: Sat, 20 Apr 2024 18:59:22 +0100 Subject: [PATCH] New custom colormaps, more work on deteched buffers UI: Added 3-tone custom colormap feature Allowed setting of window function Changed the daefault window function to Hann Worker: Added file parameter to setupCtx, to aid debugging processPredictQueue no longer async - Emit a warning if no header found in a file during prediction - call updatefilesbeingprocessed when concatenatedbuffer has no length when null chunk received - DO NOT call STREAM.end() In fetchAudioBuffer, don't look for headers each chunk. It should always be at the beginning *Set timeout in fluent-ffmpeg to 2000, increased backlog limit to 200* Bump version. --- Help/settings.html | 14 +++++- index.html | 48 +++++++++++++++----- js/ui.js | 106 +++++++++++++++++++++++++++++++++++++-------- js/worker.js | 73 +++++++++++++++---------------- 4 files changed, 170 insertions(+), 71 deletions(-) diff --git a/Help/settings.html b/Help/settings.html index 17e85618..b255282b 100644 --- a/Help/settings.html +++ b/Help/settings.html @@ -164,7 +164,7 @@ -
Audio Export
+
Audio Export
Format and Bitrate @@ -192,7 +192,17 @@ Colourmap - Choose the colour theme for the spectrogram display + Choose a colour theme for the spectrogram display, or create your own. If you select 'custom', you will have the option to set the colours for peak, mid and quiet sounds according + to personal preference. You can also adjust the mid-point position: with a value of 0 or 1, the Spectrogram will be two-tone. Values in between will blend the three colours. + If you set the Mid colour the same as one of the others, you will be able to adjust the contrast in the spectrogram using Mid Position adjustments. +

In combination with audio filter adjustments, a custom colormap allows you to enhance the contrast / visibility of calls using the colours of your choice.

+ + + + Window Function + A variety of windowing functinos are available for the spectrogram display. Each has slightly different characteristics, so changing the window function may also enhance the + appearance of the calls in the display. + Timeline diff --git a/index.html b/index.html index 68a4e8e6..1efa68f0 100644 --- a/index.html +++ b/index.html @@ -368,6 +368,7 @@
Audio Export:
+
+
+
+ +
+ +
+ + +
+ +
+ +
+ 0.5 +
+
+
+
+ +
diff --git a/js/ui.js b/js/ui.js index 37f3b227..22492157 100644 --- a/js/ui.js +++ b/js/ui.js @@ -82,6 +82,28 @@ const SunCalc = window.module.SunCalc; const uuidv4 = window.module.uuidv4; const os = window.module.os; +function hexToRgb(hex) { + // Remove the '#' character if present + hex = hex.replace(/^#/, ''); + + // Parse the hex string into individual RGB components + var r = parseInt(hex.substring(0, 2), 16); + var g = parseInt(hex.substring(2, 4), 16); + var b = parseInt(hex.substring(4, 6), 16); + + // Return the RGB components as an array + return [r, g, b]; +} +function createColormap(){ + const map = config.colormap === 'custom' ? [ + {index: 0, rgb: hexToRgb(config.customColormap.quiet)}, + {index: config.customColormap.threshold, rgb: hexToRgb(config.customColormap.mid)}, + {index: 1, rgb: hexToRgb(config.customColormap.loud)} + ] : config.colormap; + return colormap({ colormap: map, nshades:256, format: 'float' }); +} + + let worker; /// Set up communication channel between UI and worker window @@ -332,7 +354,7 @@ const initWavesurfer = ({ waveColor: 'rgba(109,41,164,0)', progressColor: 'rgba(109,41,16,0)', // but keep the playhead - cursorColor: '#fff', + cursorColor: config.colormap === 'custom' ? config.customColormap.loud : '#fff', cursorWidth: 2, skipLength: 0.1, partialRender: true, @@ -1210,15 +1232,17 @@ const footerHeight = document.getElementById('footer').offsetHeight; const navHeight = document.getElementById('navPadding').offsetHeight; function adjustSpecDims(redraw, fftSamples) { //Contentwrapper starts below navbar (66px) and ends above footer (30px). Hence - 96 - DOM.contentWrapperElement.style.height = (bodyElement.clientHeight - footerHeight - navHeight) + 'px'; - const contentHeight = DOM.contentWrapperElement.offsetHeight; + const contentWrapper = document.getElementById('contentWrapper'); + contentWrapper.style.height = (bodyElement.clientHeight - footerHeight - navHeight) + 'px'; + const contentHeight = contentWrapper.offsetHeight; // + 2 for padding const formOffset = document.getElementById('exploreWrapper').offsetHeight; let specOffset; - if (!DOM.spectrogramWrapper.classList.contains('d-none')) { + const spectrogramWrapper = document.getElementById('spectrogramWrapper') + if (!spectrogramWrapper.classList.contains('d-none')) { // Expand up to 512px unless fullscreen const controlsHeight = document.getElementById('controlsWrapper').offsetHeight; - const timelineHeight = document.getElementById('timeline').offsetHeight; + const timelineHeight = 22 ; //document.getElementById('timeline').offsetHeight; // This is unset when there is no wavesurfer, so hard-coding const specHeight = config.fullscreen ? contentHeight - timelineHeight - formOffset - controlsHeight : Math.min(contentHeight * 0.4, 512); if (currentFile) { // give the wrapper space for the transport controls and element padding/margins @@ -1239,12 +1263,13 @@ function adjustSpecDims(redraw, fftSamples) { document.querySelector('.spec-labels').style.width = '55px'; } if (wavesurfer && redraw) { - specOffset = DOM.spectrogramWrapper.offsetHeight; + specOffset = spectrogramWrapper.offsetHeight; } } else { specOffset = 0 } - DOM.resultTableElement.style.height = (contentHeight - specOffset - formOffset) + 'px'; + const resultTableElement = document.getElementById('resultTableContainer'); + resultTableElement.style.height = (contentHeight - specOffset - formOffset) + 'px'; } @@ -1444,6 +1469,7 @@ window.onload = async () => { lastUpdateCheck: 0, UUID: uuidv4(), colormap: 'inferno', + customColormap: {'loud': "#00f5d8", 'mid': "#000000", 'quiet': "#000000", 'threshold': 0.5, 'windowFn': 'hann'}, timeOfDay: true, list: 'birds', customListFile: {birdnet: '', chirpity: ''}, @@ -1539,8 +1565,15 @@ window.onload = async () => { DOM.timelineSetting.value = config.timeOfDay ? 'timeOfDay' : 'timecode'; // Spectrogram colour DOM.colourmap.value = config.colormap; - // Nocmig mode state - console.log('nocmig mode is ' + config.detect.nocmig); + // Window function & colormap + document.getElementById('window-function').value = config.customColormap.windowFn; + config.colormap === 'custom' && document.getElementById('colormap-fieldset').classList.remove('d-none'); + document.getElementById('color-threshold').textContent = config.customColormap.threshold; + document.getElementById('loud-color').value = config.customColormap.loud; + document.getElementById('mid-color').value = config.customColormap.mid; + document.getElementById('quiet-color').value = config.customColormap.quiet; + document.getElementById('color-threshold-slider').value = config.customColormap.threshold; + // Audio preferences: DOM.gain.value = config.audio.gain; DOM.gainAdjustment.textContent = config.audio.gain + 'dB'; @@ -2275,13 +2308,15 @@ function onChartData(args) { height = fftSamples / 2 } if (wavesurfer.spectrogram) wavesurfer.destroyPlugin('spectrogram'); + // set colormap + const colors = createColormap() ; wavesurfer.addPlugin(WaveSurfer.spectrogram.create({ //deferInit: false, wavesurfer: wavesurfer, container: "#spectrogram", scrollParent: false, fillParent: true, - windowFunc: 'hamming', + windowFunc: config.customColormap.windowFn, frequencyMin: 0, frequencyMax: 11_950, normalize: false, @@ -2289,9 +2324,7 @@ function onChartData(args) { labels: true, height: height, fftSamples: fftSamples, - colorMap: colormap({ - colormap: config.colormap, nshades: 256, format: 'float' - }), + colorMap: colors })).initPlugin('spectrogram') updateElementCache(); } @@ -2597,7 +2630,7 @@ function onChartData(args) { }, Minus: function (e) { if (e.shiftKey) { - if (wavesurfer.spectrogram.fftSamples <= 2048) { + if (wavesurfer.spectrogram.fftSamples <= 4096) { wavesurfer.spectrogram.fftSamples *= 2; const position = clamp(wavesurfer.getCurrentTime() / windowLength, 0, 1); postBufferUpdate({ begin: bufferBegin, position: position, region: getRegion(), goToRegion: false }) @@ -2609,7 +2642,7 @@ function onChartData(args) { }, NumpadSubtract: function (e) { if (e.shiftKey) { - if (wavesurfer.spectrogram.fftSamples <= 2048) { + if (wavesurfer.spectrogram.fftSamples <= 4096) { wavesurfer.spectrogram.fftSamples *= 2; const position = clamp(wavesurfer.getCurrentTime() / windowLength, 0, 1); postBufferUpdate({ begin: bufferBegin, position: position, region: getRegion(), goToRegion: false }) @@ -4042,6 +4075,12 @@ function onChartData(args) { SNRSlider.addEventListener('input', () => { SNRThreshold.textContent = SNRSlider.value; }); + + const colorMapThreshhold = document.getElementById('color-threshold'); + const colorMapSlider = document.getElementById('color-threshold-slider'); + colorMapSlider.addEventListener('input', () => { + colorMapThreshhold.textContent = colorMapSlider.value; + }); const handleHPchange = () => { config.filters.highPassFrequency = HPSlider.valueAsNumber; @@ -4360,11 +4399,42 @@ DOM.gain.addEventListener('input', () => { } case 'colourmap': { config.colormap = element.value; + const colorMapFieldset = document.getElementById('colormap-fieldset') + if (config.colormap === 'custom'){ + colorMapFieldset.classList.remove('d-none') + } else { + colorMapFieldset.classList.add('d-none') + } + if (wavesurfer && currentFile) { + + // refresh caches + updateElementCache() + const fftSamples = wavesurfer.spectrogram.fftSamples; + wavesurfer.destroy(); + wavesurfer = undefined; + adjustSpecDims(true, fftSamples) + } + break; + } + case 'window-function': + case 'loud-color': + case 'mid-color': + case 'quiet-color': + case 'color-threshold-slider': { + const windowFn = document.getElementById('window-function').value; + const loud = document.getElementById('loud-color').value; + const mid = document.getElementById('mid-color').value; + const quiet = document.getElementById('quiet-color').value; + const threshold = document.getElementById('color-threshold-slider').valueAsNumber; + document.getElementById('color-threshold').textContent = threshold; + config.customColormap = {'loud': loud, 'mid': mid, 'quiet': quiet, 'threshold': threshold, 'windowFn': windowFn}; if (wavesurfer && currentFile) { - initSpectrogram(); // refresh caches updateElementCache() - adjustSpecDims(true) + const fftSamples = wavesurfer.spectrogram.fftSamples; + wavesurfer.destroy(); + wavesurfer = undefined; + adjustSpecDims(true, fftSamples) } break; } @@ -5201,7 +5271,7 @@ function showCompareSpec() { //deferInit: false, wavesurfer: ws, container: "#" + specContainer, - windowFunc: 'hamming', + windowFunc: 'hann', frequencyMin: 0, frequencyMax: 11_950, hideScrollbar: false, diff --git a/js/worker.js b/js/worker.js index 2cfd3c80..0d763ae1 100644 --- a/js/worker.js +++ b/js/worker.js @@ -1126,7 +1126,7 @@ const setMetadata = async ({ file, proxy = file, source_file = file }) => { }).catch(error => console.warn(error.message)) } -function setupCtx(audio, rate, destination) { +function setupCtx(audio, rate, destination, file) { rate ??= sampleRate; // Deal with detached arraybuffer issue const useFilters = (STATE.filters.sendToModel && STATE.filters.active) || destination === 'UI'; @@ -1175,7 +1175,7 @@ function setupCtx(audio, rate, destination) { offlineSource.start(); return offlineCtx; }) - .catch(error => console.warn(error)); + .catch(error => console.warn(error, file)); }; @@ -1184,7 +1184,7 @@ function checkBacklog(stream) { const backlog = sumObjectValues(predictionsRequested) - sumObjectValues(predictionsReceived); DEBUG && console.log('backlog:', backlog); - if (backlog > 100) { + if (backlog > 200) { // If queued value is above 100, wait and check again setTimeout(() => { checkBacklog(stream) @@ -1288,34 +1288,30 @@ const getWavePredictBuffers = async ({ }) } -async function processPredictQueue(){ - return new Promise((resolve, reject) => { - const [audio, file, end, chunkStart] = predictQueue.shift(); // Dequeue chunk - setupCtx(audio, undefined, 'model').then(offlineCtx => { - let worker; - if (offlineCtx) { - offlineCtx.startRendering().then((resampled) => { - const myArray = resampled.getChannelData(0); - workerInstance = ++workerInstance >= NUM_WORKERS ? 0 : workerInstance; - worker = workerInstance; - feedChunksToModel(myArray, chunkStart, file, end, worker); - return resolve('done'); - }).catch((error) => { - console.error(`PredictBuffer rendering failed: ${error}, file ${file}`); - updateFilesBeingProcessed(file); - return reject(error) - }); - } else { - console.log('Short chunk', audio.length, 'padding'); - let chunkLength = STATE.model === 'birdnet' ? 144_000 : 72_000; +function processPredictQueue(){ + const [audio, file, end, chunkStart] = predictQueue.shift(); // Dequeue chunk + setupCtx(audio, undefined, 'model', file).then(offlineCtx => { + let worker; + if (offlineCtx) { + offlineCtx.startRendering().then((resampled) => { + const myArray = resampled.getChannelData(0); workerInstance = ++workerInstance >= NUM_WORKERS ? 0 : workerInstance; worker = workerInstance; - const myArray = new Float32Array(Array.from({ length: chunkLength }).fill(0)); - feedChunksToModel(myArray, chunkStart, file, end); - }}).catch(error => { - console.warn(file, error); - }) - }) + feedChunksToModel(myArray, chunkStart, file, end, worker); + return + }).catch((error) => { + console.error(`PredictBuffer rendering failed: ${error}, file ${file}`); + updateFilesBeingProcessed(file); + return + }); + } else { + console.log('Short chunk', audio.length, 'padding'); + let chunkLength = STATE.model === 'birdnet' ? 144_000 : 72_000; + workerInstance = ++workerInstance >= NUM_WORKERS ? 0 : workerInstance; + worker = workerInstance; + const myArray = new Float32Array(Array.from({ length: chunkLength }).fill(0)); + feedChunksToModel(myArray, chunkStart, file, end); + }}).catch(error => { console.warn(file, error) }) } const getPredictBuffers = async ({ @@ -1348,12 +1344,11 @@ const getPredictBuffers = async ({ .format('wav') .audioChannels(1) // Set to mono .audioFrequency(sampleRate) // Set sample rate - //.outputOptions([`-bufsize ${highWaterMark}`]) .writeToStream(STREAM) command.on('error', error => { updateFilesBeingProcessed(file) - if (error.message.includes('SIGKILL')) DEBUG && console.log('FFMPEG process shut down') + if (error.message.includes('SIGKILL')) console.log('FFMPEG process shut down at user request') else { error.message = error.message + '|' + error.stack; } @@ -1372,18 +1367,21 @@ const getPredictBuffers = async ({ if (chunk === null || chunk.byteLength <= 1) { // EOF: deal with part-full buffers if (concatenatedBuffer.length){ + header || console.warn('no header for ' + file) const audio = isWavHeaderPresent(header, concatenatedBuffer) ? concatenatedBuffer: Buffer.concat([header, concatenatedBuffer]) predictQueue.push([audio, file, end, chunkStart]); processPredictQueue(); + } else { + updateFilesBeingProcessed(file) } DEBUG && console.log('All chunks sent for ', file); - STREAM.destroy(); + //STREAM.end(); resolve('finished') } else { try { - concatenatedBuffer = concatenatedBuffer.length ? Buffer.concat([concatenatedBuffer, chunk]) : chunk; + concatenatedBuffer = concatenatedBuffer.byteLength ? Buffer.concat([concatenatedBuffer, chunk]) : chunk; header ??= lookForHeader(concatenatedBuffer); } catch (e) { console.log('Detached buffer?', e.message); @@ -1409,6 +1407,7 @@ const getPredictBuffers = async ({ } function lookForHeader(buffer){ + //if (buffer.length < 4096) return undefined try { const wav = new wavefileReader.WaveFileReader(); wav.fromBuffer(buffer); @@ -1487,11 +1486,9 @@ const fetchAudioBuffer = async ({ const chunk = stream.read(); if (chunk === null || chunk.byteLength <= 1) { // Last chunk - const audio = isWavHeaderPresent(header, concatenatedBuffer) ? concatenatedBuffer : Buffer.concat([header, concatenatedBuffer]); + const audio = concatenatedBuffer; setupCtx(audio, sampleRate, 'UI').then(offlineCtx => { offlineCtx.startRendering().then(resampled => { - stream.end(); - stream.destroy(); resolve(resampled); }).catch((error) => { console.error(`FetchAudio rendering failed: ${error}`); @@ -1499,11 +1496,9 @@ const fetchAudioBuffer = async ({ }).catch( (error) => { reject(error.message) stream.destroy(); - }); - + }); } else { // other chunks - header ??= lookForHeader(chunk) concatenatedBuffer = concatenatedBuffer.length ? Buffer.concat([concatenatedBuffer, chunk]) : chunk; } });