Removed ffmpeg-static electron and now use @ffmpeg-installer/ffmpeg

added batch sizes 4 & 8 back to the batch size map
Mattk70 · Mar 27, 2024 · f4cb79b · f4cb79b
1 parent 82eeb33
commit f4cb79b
Show file tree

Hide file tree

Showing 5 changed files with 31 additions and 55 deletions.
diff --git a/js/state.js b/js/state.js
@@ -32,7 +32,7 @@ export class State {
             this.list = 'everything',
             this.customList = undefined,
             this.local = true,
-            this.incrementor = 1,
+            this.incrementor = 250,
             this.UUID = 0,
             this.track = true
     }

diff --git a/js/tracking.js b/js/tracking.js
@@ -1,5 +1,5 @@
 const DEBUG = false;
-const ID_SITE = 3;
+const ID_SITE = 2;
 
 
 function trackEvent(uuid, event, action, name, value){

diff --git a/js/ui.js b/js/ui.js
@@ -72,7 +72,7 @@ const STATE = {
 }
 
 // Batch size map for slider
-const BATCH_SIZE_LIST = [16, 32, 48, 64, 128];
+const BATCH_SIZE_LIST = [4, 8, 16, 32, 48, 64, 128];
 
 // Get the modules loaded in preload.js
 const fs = window.module.fs;

diff --git a/js/worker.js b/js/worker.js
@@ -8,7 +8,6 @@ const SunCalc = require('suncalc');
 const ffmpeg = require('fluent-ffmpeg');
 const png = require('fast-png');
 const { utimesSync } = require('utimes');
-const staticFfmpeg = require('ffmpeg-static-electron');
 const {writeToPath} = require('@fast-csv/format');
 const merge = require('lodash.merge');
 import { State } from './state.js';
@@ -103,8 +102,9 @@ Date.prototype.getWeekNumber = function(){
 
 
 DEBUG && console.log(staticFfmpeg.path);
-ffmpeg.setFfmpegPath(staticFfmpeg.path.replace('app.asar', 'app.asar.unpacked'));
-
+const ffmpegPath = require('@ffmpeg-installer/ffmpeg').path.replace('app.asar', 'app.asar.unpacked');
+//ffmpeg.setFfmpegPath(staticFfmpeg.path.replace('app.asar', 'app.asar.unpacked'));
+ffmpeg.setFfmpegPath(ffmpegPath);
 let predictionsRequested = {}, predictionsReceived = {}, filesBeingProcessed = [];
 let diskDB, memoryDB;
 
@@ -443,7 +443,7 @@ async function onLaunch({model = 'chirpity', batchSize = 32, threads = 1, backen
     STATE.update({ model: model });
     await loadDB(appPath); // load the diskdb
     await createDB(); // now make the memoryDB
-    spawnPredictWorkers(model, list, batchSize, threads);
+    spawnPredictWorkers(model, list, 4, threads);
 }
 
 
@@ -730,7 +730,7 @@ async function onAnalyse({
     circleClicked = false
 }) {
     // Now we've asked for a new analysis, clear the aborted flag
-    aborted = false; STATE.incrementor = 1;
+    aborted = false; //STATE.incrementor = 1;
     predictionStart = new Date();
     // Set the appropraite selection range if this is a selection analysis
     STATE.update({ selection: end ? getSelectionRange(filesInScope[0], start, end) : undefined });
@@ -1267,7 +1267,8 @@ const getPredictBuffers = async ({
     predictionsRequested[file] = 0;
     let concatenatedBuffer = Buffer.alloc(0);
     const highWaterMark = 2 * sampleRate * BATCH_SIZE * WINDOW_SIZE; 
-    const STREAM = new PassThrough({ highWaterMark: highWaterMark, end: true});
+    //const STREAM = new PassThrough({ highWaterMark: highWaterMark, end: true});
+    const STREAM = new PassThrough({end: false});
 
     let chunkStart = start * sampleRate;
     return new Promise((resolve, reject) => {
@@ -1278,7 +1279,7 @@ const getPredictBuffers = async ({
             .audioChannels(1) // Set to mono
             .audioFrequency(sampleRate) // Set sample rate 
             .outputOptions([`-bufsize ${highWaterMark}`])
-            .output(STREAM)
+            .writeToStream(STREAM)
 
         command.on('error', error => {
             updateFilesBeingProcessed(file)
@@ -1299,17 +1300,18 @@ const getPredictBuffers = async ({
         STREAM.on('data', chunk => {
 
             if (aborted) {
-                command.kill()
                 STREAM.destroy()
                 return
             }
             if (chunk.byteLength <= 1) {
-                return
+                STREAM.end();
+                STREAM.destroy();
+                console.log('STREAM ended, destroyed')
             }
             else {
                 // try/catch may no longer be necessary
                 try {
-                    concatenatedBuffer = Buffer.concat([concatenatedBuffer, chunk]);
+                    concatenatedBuffer = concatenatedBuffer.length ?  Buffer.concat([concatenatedBuffer, chunk]) : chunk;
                 } catch (error) {
                     console.warn(error)
                     console.warn('concat bugger length', concatenatedBuffer.length, 'chunk.length', chunk.length, chunk.buffer.detached, concatenatedBuffer.buffer.detached)
@@ -1319,20 +1321,22 @@ const getPredictBuffers = async ({
                 if (concatenatedBuffer.length >= highWaterMark) {
                     STREAM.pause();
                     chunk = concatenatedBuffer.subarray(0, highWaterMark);
-                    const remainingBuffer = Buffer.from(concatenatedBuffer.subarray(highWaterMark));
+                    concatenatedBuffer = concatenatedBuffer.subarray(highWaterMark);
                     const audio = Buffer.concat([WAV_HEADER, chunk])
                     predictQueue.push([audio, file, end, chunkStart]);
                     chunkStart += WINDOW_SIZE * BATCH_SIZE * sampleRate
-                    processPredictQueue().then((resolve, reject) =>{
-                        concatenatedBuffer = remainingBuffer;
-                        checkBacklog(STREAM) 
-                    }
-                    );
-
+                    processPredictQueue().then((resolve, reject) => checkBacklog(STREAM) );
                 }
             }
         });
 
+        // STREAM.on('close', () => console.log('STREAM closed'))
+        // STREAM.on('pipe', () => console.log('STREAM piped'))
+        // STREAM.on('unpipe', () => console.log('STREAM unpiped'))
+        // STREAM.on('drain', () => console.log('STREAM drained'))
+        // STREAM.on('finish', () => console.log('STREAM resumed'))
+        // STREAM.on('resume', () => console.log('STREAM resumed'))
+
         STREAM.on('end', () => {
             // EOF: deal with part-full buffers
             if (concatenatedBuffer.length){
@@ -1350,7 +1354,7 @@ const getPredictBuffers = async ({
             err.code === 'ENOENT' && notifyMissingFile(file);
         })
 
-        command.run();
+        // command.run();
     }).catch(error => console.log(error));
 }
 
@@ -1375,7 +1379,7 @@ const fetchAudioBuffer = async ({
             .audioChannels(1) // Set to mono
             .audioFrequency(24_000) // Set sample rate to 24000 Hz (always - this is for wavesurfer)
             .output(stream, { end:true });
-        if (STATE.audio.normalise) command = command.audioFilter("loudnorm=I=-16:LRA=11:TP=-1.5");
+        if (STATE.audio.normalise) command = command.audioFilter("loudnorm=I=-16:LRA=11:TP=-1.5:offset=" + STATE.audio.gain);
 
         command.on('error', error => {
             UI.postMessage({event: 'generate-alert', message: error.message})

diff --git a/package.json b/package.json
@@ -45,13 +45,7 @@
       "!README.md",
       "!saved_model$(/*)",
       "!**/*.*js.map",
-      "!**/*.*.md",
-      "node_modules/ffmpeg-static-electron/bin/${os}/${arch}/ffmpeg",
-      "node_modules/ffmpeg-static-electron/index.js",
-      "node_modules/ffmpeg-static-electron/package.json",
-      "node_modules/ffprobe-static-electron/bin/${os}/${arch}/ffmpeg",
-      "node_modules/ffprobe-static-electron/index.js",
-      "node_modules/ffprobe-static-electron/package.json"
+      "!**/*.*.md"
     ],
     "extraResources": [
       "./Help/example.mp3"
@@ -103,12 +97,7 @@
         "target": "pkg",
         "arch": "arm64"
       },
-      "files": [
-        "!node_modules/ffmpeg-static-electron/bin/win${/*}",
-        "!node_modules/ffmpeg-static-electron/bin/linux${/*}",
-        "!node_modules/ffprobe-static-electron/bin/win${/*}",
-        "!node_modules/ffprobe-static-electron/bin/linux${/*}"
-      ],
+
       "icon": "img/icon/icon.icns",
       "category": "public.app-category.utilities",
       "fileAssociations": [
@@ -133,14 +122,6 @@
       "icon": "./packages/img/icon/icon.icns",
       "target": [
         "AppImage"
-      ],
-      "files": [
-        "!node_modules/ffmpeg-static-electron/bin/linux/ia32${/*}",
-        "!node_modules/ffmpeg-static-electron/bin/win${/*}",
-        "!node_modules/ffmpeg-static-electron/bin/mac${/*}",
-        "!node_modules/ffprobe-static-electron/bin/linux/ia32${/*}",
-        "!node_modules/ffprobe-static-electron/bin/win${/*}",
-        "!node_modules/ffprobe-static-electron/bin/mac${/*}"
       ]
     },
     "nsis": {
@@ -153,17 +134,8 @@
       ],
       "verifyUpdateCodeSignature": false,
       "asar": true,
-      "icon": "./img/icon/icon.png",
-      "files": [
-        "node_modules/ffmpeg-static-electron/bin/win/${arch}/ffmpeg",
-        "!node_modules/ffmpeg-static-electron/bin/win/ia32${/*}",
-        "!node_modules/ffmpeg-static-electron/bin/linux${/*}",
-        "!node_modules/ffmpeg-static-electron/bin/mac${/*}",
-        "node_modules/ffprobe-static-electron/bin/win/${arch}/ffprobe",
-        "!node_modules/ffprobe-static-electron/bin/win/ia32${/*}",
-        "!node_modules/ffprobe-static-electron/bin/linux${/*}",
-        "!node_modules/ffprobe-static-electron/bin/mac${/*}"
-      ]
+      "icon": "./img/icon/icon.png"
+
     }
   },
   "keywords": [
@@ -203,7 +175,7 @@
     "electron-settings": "^4.0.2",
     "electron-updater": "6.1.4",
     "fast-png": "^6.1.0",
-    "ffmpeg-static-electron": "^2.0.3",
+    "@ffmpeg-installer/ffmpeg": "^1.1.0",
     "fluent-ffmpeg": "^2.1.2",
     "lodash.merge": "^4.6.2",
     "sqlite3": "5.1.6",