diff --git a/js/state.js b/js/state.js index ef88ffe1..b9b030f1 100644 --- a/js/state.js +++ b/js/state.js @@ -32,7 +32,7 @@ export class State { this.list = 'everything', this.customList = undefined, this.local = true, - this.incrementor = 1, + this.incrementor = 250, this.UUID = 0, this.track = true } diff --git a/js/tracking.js b/js/tracking.js index c67ea88f..01b2dd08 100644 --- a/js/tracking.js +++ b/js/tracking.js @@ -1,5 +1,5 @@ const DEBUG = false; -const ID_SITE = 3; +const ID_SITE = 2; function trackEvent(uuid, event, action, name, value){ diff --git a/js/ui.js b/js/ui.js index 1a749679..f1688e80 100644 --- a/js/ui.js +++ b/js/ui.js @@ -72,7 +72,7 @@ const STATE = { } // Batch size map for slider -const BATCH_SIZE_LIST = [16, 32, 48, 64, 128]; +const BATCH_SIZE_LIST = [4, 8, 16, 32, 48, 64, 128]; // Get the modules loaded in preload.js const fs = window.module.fs; diff --git a/js/worker.js b/js/worker.js index 785966ac..da829199 100644 --- a/js/worker.js +++ b/js/worker.js @@ -8,7 +8,6 @@ const SunCalc = require('suncalc'); const ffmpeg = require('fluent-ffmpeg'); const png = require('fast-png'); const { utimesSync } = require('utimes'); -const staticFfmpeg = require('ffmpeg-static-electron'); const {writeToPath} = require('@fast-csv/format'); const merge = require('lodash.merge'); import { State } from './state.js'; @@ -103,8 +102,9 @@ Date.prototype.getWeekNumber = function(){ DEBUG && console.log(staticFfmpeg.path); -ffmpeg.setFfmpegPath(staticFfmpeg.path.replace('app.asar', 'app.asar.unpacked')); - +const ffmpegPath = require('@ffmpeg-installer/ffmpeg').path.replace('app.asar', 'app.asar.unpacked'); +//ffmpeg.setFfmpegPath(staticFfmpeg.path.replace('app.asar', 'app.asar.unpacked')); +ffmpeg.setFfmpegPath(ffmpegPath); let predictionsRequested = {}, predictionsReceived = {}, filesBeingProcessed = []; let diskDB, memoryDB; @@ -443,7 +443,7 @@ async function onLaunch({model = 'chirpity', batchSize = 32, threads = 1, backen STATE.update({ model: model }); await loadDB(appPath); // load the diskdb await createDB(); // now make the memoryDB - spawnPredictWorkers(model, list, batchSize, threads); + spawnPredictWorkers(model, list, 4, threads); } @@ -730,7 +730,7 @@ async function onAnalyse({ circleClicked = false }) { // Now we've asked for a new analysis, clear the aborted flag - aborted = false; STATE.incrementor = 1; + aborted = false; //STATE.incrementor = 1; predictionStart = new Date(); // Set the appropraite selection range if this is a selection analysis STATE.update({ selection: end ? getSelectionRange(filesInScope[0], start, end) : undefined }); @@ -1267,7 +1267,8 @@ const getPredictBuffers = async ({ predictionsRequested[file] = 0; let concatenatedBuffer = Buffer.alloc(0); const highWaterMark = 2 * sampleRate * BATCH_SIZE * WINDOW_SIZE; - const STREAM = new PassThrough({ highWaterMark: highWaterMark, end: true}); + //const STREAM = new PassThrough({ highWaterMark: highWaterMark, end: true}); + const STREAM = new PassThrough({end: false}); let chunkStart = start * sampleRate; return new Promise((resolve, reject) => { @@ -1278,7 +1279,7 @@ const getPredictBuffers = async ({ .audioChannels(1) // Set to mono .audioFrequency(sampleRate) // Set sample rate .outputOptions([`-bufsize ${highWaterMark}`]) - .output(STREAM) + .writeToStream(STREAM) command.on('error', error => { updateFilesBeingProcessed(file) @@ -1299,17 +1300,18 @@ const getPredictBuffers = async ({ STREAM.on('data', chunk => { if (aborted) { - command.kill() STREAM.destroy() return } if (chunk.byteLength <= 1) { - return + STREAM.end(); + STREAM.destroy(); + console.log('STREAM ended, destroyed') } else { // try/catch may no longer be necessary try { - concatenatedBuffer = Buffer.concat([concatenatedBuffer, chunk]); + concatenatedBuffer = concatenatedBuffer.length ? Buffer.concat([concatenatedBuffer, chunk]) : chunk; } catch (error) { console.warn(error) console.warn('concat bugger length', concatenatedBuffer.length, 'chunk.length', chunk.length, chunk.buffer.detached, concatenatedBuffer.buffer.detached) @@ -1319,20 +1321,22 @@ const getPredictBuffers = async ({ if (concatenatedBuffer.length >= highWaterMark) { STREAM.pause(); chunk = concatenatedBuffer.subarray(0, highWaterMark); - const remainingBuffer = Buffer.from(concatenatedBuffer.subarray(highWaterMark)); + concatenatedBuffer = concatenatedBuffer.subarray(highWaterMark); const audio = Buffer.concat([WAV_HEADER, chunk]) predictQueue.push([audio, file, end, chunkStart]); chunkStart += WINDOW_SIZE * BATCH_SIZE * sampleRate - processPredictQueue().then((resolve, reject) =>{ - concatenatedBuffer = remainingBuffer; - checkBacklog(STREAM) - } - ); - + processPredictQueue().then((resolve, reject) => checkBacklog(STREAM) ); } } }); + // STREAM.on('close', () => console.log('STREAM closed')) + // STREAM.on('pipe', () => console.log('STREAM piped')) + // STREAM.on('unpipe', () => console.log('STREAM unpiped')) + // STREAM.on('drain', () => console.log('STREAM drained')) + // STREAM.on('finish', () => console.log('STREAM resumed')) + // STREAM.on('resume', () => console.log('STREAM resumed')) + STREAM.on('end', () => { // EOF: deal with part-full buffers if (concatenatedBuffer.length){ @@ -1350,7 +1354,7 @@ const getPredictBuffers = async ({ err.code === 'ENOENT' && notifyMissingFile(file); }) - command.run(); + // command.run(); }).catch(error => console.log(error)); } @@ -1375,7 +1379,7 @@ const fetchAudioBuffer = async ({ .audioChannels(1) // Set to mono .audioFrequency(24_000) // Set sample rate to 24000 Hz (always - this is for wavesurfer) .output(stream, { end:true }); - if (STATE.audio.normalise) command = command.audioFilter("loudnorm=I=-16:LRA=11:TP=-1.5"); + if (STATE.audio.normalise) command = command.audioFilter("loudnorm=I=-16:LRA=11:TP=-1.5:offset=" + STATE.audio.gain); command.on('error', error => { UI.postMessage({event: 'generate-alert', message: error.message}) diff --git a/package.json b/package.json index 68e7c6a2..71e1a5e4 100644 --- a/package.json +++ b/package.json @@ -45,13 +45,7 @@ "!README.md", "!saved_model$(/*)", "!**/*.*js.map", - "!**/*.*.md", - "node_modules/ffmpeg-static-electron/bin/${os}/${arch}/ffmpeg", - "node_modules/ffmpeg-static-electron/index.js", - "node_modules/ffmpeg-static-electron/package.json", - "node_modules/ffprobe-static-electron/bin/${os}/${arch}/ffmpeg", - "node_modules/ffprobe-static-electron/index.js", - "node_modules/ffprobe-static-electron/package.json" + "!**/*.*.md" ], "extraResources": [ "./Help/example.mp3" @@ -103,12 +97,7 @@ "target": "pkg", "arch": "arm64" }, - "files": [ - "!node_modules/ffmpeg-static-electron/bin/win${/*}", - "!node_modules/ffmpeg-static-electron/bin/linux${/*}", - "!node_modules/ffprobe-static-electron/bin/win${/*}", - "!node_modules/ffprobe-static-electron/bin/linux${/*}" - ], + "icon": "img/icon/icon.icns", "category": "public.app-category.utilities", "fileAssociations": [ @@ -133,14 +122,6 @@ "icon": "./packages/img/icon/icon.icns", "target": [ "AppImage" - ], - "files": [ - "!node_modules/ffmpeg-static-electron/bin/linux/ia32${/*}", - "!node_modules/ffmpeg-static-electron/bin/win${/*}", - "!node_modules/ffmpeg-static-electron/bin/mac${/*}", - "!node_modules/ffprobe-static-electron/bin/linux/ia32${/*}", - "!node_modules/ffprobe-static-electron/bin/win${/*}", - "!node_modules/ffprobe-static-electron/bin/mac${/*}" ] }, "nsis": { @@ -153,17 +134,8 @@ ], "verifyUpdateCodeSignature": false, "asar": true, - "icon": "./img/icon/icon.png", - "files": [ - "node_modules/ffmpeg-static-electron/bin/win/${arch}/ffmpeg", - "!node_modules/ffmpeg-static-electron/bin/win/ia32${/*}", - "!node_modules/ffmpeg-static-electron/bin/linux${/*}", - "!node_modules/ffmpeg-static-electron/bin/mac${/*}", - "node_modules/ffprobe-static-electron/bin/win/${arch}/ffprobe", - "!node_modules/ffprobe-static-electron/bin/win/ia32${/*}", - "!node_modules/ffprobe-static-electron/bin/linux${/*}", - "!node_modules/ffprobe-static-electron/bin/mac${/*}" - ] + "icon": "./img/icon/icon.png" + } }, "keywords": [ @@ -203,7 +175,7 @@ "electron-settings": "^4.0.2", "electron-updater": "6.1.4", "fast-png": "^6.1.0", - "ffmpeg-static-electron": "^2.0.3", + "@ffmpeg-installer/ffmpeg": "^1.1.0", "fluent-ffmpeg": "^2.1.2", "lodash.merge": "^4.6.2", "sqlite3": "5.1.6",