From f063941d24a63242ede39e0e9368a7e31850b7b8 Mon Sep 17 00:00:00 2001 From: Mattk70 Date: Tue, 6 Feb 2024 10:26:29 +0000 Subject: [PATCH] All model calls return the worker instance that processed the task database errors now log SQL triggering error Promise.all used in model.js for syncing indices and values, Dramatically increases GPU speed (2x boost: 1170x on 3090!!) worker instance no longer passed down from processNextFile, it's set in feedChunks to model predictworkers list adds isReady to isAvailable to fix getting list on launch all predictworker messages from worker include worker instance Added wiatForWorker to finx issue with no ready / available workers returning undefined. --- js/BirdNet2.4.js | 15 ++- js/database.js | 8 +- js/model.js | 252 +++++++++++++++++++++++------------------------ js/worker.js | 195 +++++++++++++++++++----------------- 4 files changed, 244 insertions(+), 226 deletions(-) diff --git a/js/BirdNet2.4.js b/js/BirdNet2.4.js index 59f0b492..3167a738 100644 --- a/js/BirdNet2.4.js +++ b/js/BirdNet2.4.js @@ -79,6 +79,7 @@ const NOT_BIRDS = [ "Alouatta pigra_Mexican Black Howler Monkey", "Tamias striatus_Eastern Chipmunk", "Tamiasciurus hudsonicus_Red Squirrel"]; + const MYSTERIES = ['Unknown Sp._Unknown Sp.']; const GRAYLIST = []; const GOLDEN_LIST = [] @@ -92,9 +93,11 @@ const CONFIG = { onmessage = async (e) => { const modelRequest = e.data.message; + const worker = e.data.worker; let response; try { switch (modelRequest) { + case "load": { const version = e.data.model; DEBUG && console.log("load request to worker"); @@ -144,7 +147,8 @@ onmessage = async (e) => { sampleRate: myModel.config.sampleRate, chunkLength: myModel.chunkLength, backend: tf.getBackend(), - labels: labels + labels: labels, + worker: worker }); }); break; @@ -199,7 +203,8 @@ onmessage = async (e) => { channels: 1, image: image, file: specFile, - filepath: filepath + filepath: filepath, + worker: worker }; postMessage(response); break; @@ -218,7 +223,8 @@ onmessage = async (e) => { lat: myModel.lat, lon: myModel.lon, week: myModel.week, - updateResults: false + updateResults: false, + worker: worker }); break; } @@ -456,6 +462,7 @@ class Model { const finalPrediction = newPrediction || prediction; //new const { indices, values } = tf.topk(finalPrediction, 5, true) + // const [topIndices, topValues] = await Promise.all([indices.arraySync(), values.arraySync()]).catch((err => console.log(err))); const topIndices = indices.arraySync(); const topValues = values.arraySync(); indices.dispose(); @@ -513,7 +520,7 @@ class Model { }; async predictChunk(audioBuffer, start, fileStart, file, threshold, confidence) { - DEBUG && console.log('predictCunk begin', tf.memory().numTensors); + DEBUG && console.log('predictCunk begin', tf.memory()); audioBuffer = tf.tensor1d(audioBuffer); // check if we need to pad diff --git a/js/database.js b/js/database.js index 3ee58b1e..02430b0e 100644 --- a/js/database.js +++ b/js/database.js @@ -6,7 +6,7 @@ const sqlite3 = DEBUG ? require('sqlite3').verbose() : require('sqlite3'); sqlite3.Database.prototype.runAsync = function (sql, ...params) { return new Promise((resolve, reject) => { this.run(sql, params, function (err) { - if (err) return reject(console.log(err)); + if (err) return reject(console.log(err, sql)); resolve(this); }); }); @@ -15,7 +15,7 @@ sqlite3.Database.prototype.runAsync = function (sql, ...params) { sqlite3.Database.prototype.allAsync = function (sql, ...params) { return new Promise((resolve, reject) => { this.all(sql, params, (err, rows) => { - if (err) return reject(console.log(err)); + if (err) return reject(console.log(err, sql)); resolve(rows); }); }); @@ -25,7 +25,7 @@ sqlite3.Statement.prototype.allAsync = function (...params) { if (DEBUG) console.log('SQL\n', this.sql, '\nParams\n', params) return new Promise((resolve, reject) => { this.all(params, (err, rows) => { - if (err) return reject(console.log(err)); + if (err) return reject(console.log(err, sql)); if (DEBUG) console.log('\nRows:', rows) resolve(rows); }); @@ -35,7 +35,7 @@ sqlite3.Statement.prototype.allAsync = function (...params) { sqlite3.Database.prototype.getAsync = function (sql, ...params) { return new Promise((resolve, reject) => { this.get(sql, params, (err, row) => { - if (err) return reject(console.log(err)); + if (err) return reject(console.log(err, sql)); resolve(row); }); }); diff --git a/js/model.js b/js/model.js index a934a2e0..181867fb 100644 --- a/js/model.js +++ b/js/model.js @@ -21,141 +21,135 @@ const CONFIG = { onmessage = async (e) => { const modelRequest = e.data.message; + const worker = e.data.worker; let response; try { switch (modelRequest) { - case "load": {const version = e.data.model; -if (DEBUG) { - console.log("load request to worker"); -} -const { height: height, width: width, labels: labels, location: location } = JSON.parse(fs.readFileSync(path.join(__dirname, `../${version}_model_config.json`), "utf8")); -const appPath = "../" + location + "/"; -const list = e.data.list; -const batch = e.data.batchSize; -const backend = e.data.backend; -// labels.push(...MYSTERIES); -// postMessage({ -// message: "labels", -// labels: labels -// }); -if (DEBUG) { - console.log(`model received load instruction. Using list: ${list}, batch size ${batch}`); -} -tf.setBackend(backend).then(async () => { - if (backend === "webgl") { - tf.env().set("WEBGL_FORCE_F16_TEXTURES", true); - tf.env().set("WEBGL_PACK", true); - tf.env().set("WEBGL_EXP_CONV", true); - tf.env().set("TOPK_K_CPU_HANDOFF_THRESHOLD", 128); - tf.env().set("TOPK_LAST_DIM_CPU_HANDOFF_SIZE_THRESHOLD", 0); - } - tf.enableProdMode(); - if (DEBUG) { - console.log(tf.env()); - console.log(tf.env().getFlags()); - } - myModel = new Model(appPath, list, version); - myModel.height = height; - myModel.width = width; - myModel.list = e.data.list; - myModel.lat = parseFloat(e.data.lat); - myModel.lon = parseFloat(e.data.lon); - myModel.week = parseInt(e.data.week); - myModel.speciesThreshold = parseFloat(e.data.threshold); - myModel.labels = labels; - await myModel.loadModel(); - // postMessage({ - // message: "update-list", - // blocked: BLOCKED_IDS, - // lat: myModel.lat, - // lon: myModel.lon, - // week: myModel.week, - // updateResults: false - // }); - myModel.warmUp(batch); - BACKEND = tf.getBackend(); - postMessage({ - message: "model-ready", - sampleRate: myModel.config.sampleRate, - chunkLength: myModel.chunkLength, - backend: tf.getBackend(), - labels: labels - }); -}); -break; -} - case "predict": {if (myModel.model_loaded) { - const { chunks: chunks, start: start, fileStart: fileStart, file: file, snr: snr, confidence: confidence, worker: worker, context: context, resetResults: resetResults } = e.data; - myModel.useContext = context; - myModel.selection = !resetResults; - const [result,filename,startPosition] = await myModel.predictChunk(chunks, start, fileStart, file, snr, confidence / 1000); - response = { - message: "prediction", - file: filename, - result: result, - fileStart: startPosition, - worker: worker, - selection: myModel.selection - }; - postMessage(response); - myModel.result = []; -} -break; -} - case "get-spectrogram": {const buffer = e.data.buffer; -if (buffer.length < myModel.chunkLength) { - return; -} -const specFile = e.data.file; -const filepath = e.data.filepath; -const spec_height = e.data.height; -const spec_width = e.data.width; -let image; -const signal = tf.tensor1d(buffer, "float32"); -const bufferTensor = myModel.normalise_audio(signal); -signal.dispose(); -const imageTensor = tf.tidy(() => { - return myModel.makeSpectrogram(bufferTensor); -}); -image = tf.tidy(() => { - let spec = myModel.fixUpSpecBatch(tf.expandDims(imageTensor, 0), spec_height, spec_width); - const spec_max = tf.max(spec); - return spec.mul(255).div(spec_max).dataSync(); -}); -bufferTensor.dispose(); -imageTensor.dispose(); -response = { - message: "spectrogram", - width: myModel.inputShape[2], - height: myModel.inputShape[1], - channels: myModel.inputShape[3], - image: image, - file: specFile, - filepath: filepath -}; -postMessage(response); -break; -} + case "load": { + const version = e.data.model; + if (DEBUG) { + console.log("load request to worker"); + } + const { height: height, width: width, labels: labels, location: location } = JSON.parse(fs.readFileSync(path.join(__dirname, `../${version}_model_config.json`), "utf8")); + const appPath = "../" + location + "/"; + const list = e.data.list; + const batch = e.data.batchSize; + const backend = e.data.backend; + if (DEBUG) { + console.log(`model received load instruction. Using list: ${list}, batch size ${batch}`); + } + tf.setBackend(backend).then(async () => { + if (backend === "webgl") { + tf.env().set("WEBGL_FORCE_F16_TEXTURES", true); + tf.env().set("WEBGL_PACK", true); + tf.env().set("WEBGL_EXP_CONV", true); + tf.env().set("TOPK_K_CPU_HANDOFF_THRESHOLD", 128); + tf.env().set("TOPK_LAST_DIM_CPU_HANDOFF_SIZE_THRESHOLD", 0); + } + tf.enableProdMode(); + if (DEBUG) { + console.log(tf.env()); + console.log(tf.env().getFlags()); + } + myModel = new Model(appPath, list, version); + myModel.height = height; + myModel.width = width; + myModel.list = e.data.list; + myModel.lat = parseFloat(e.data.lat); + myModel.lon = parseFloat(e.data.lon); + myModel.week = parseInt(e.data.week); + myModel.speciesThreshold = parseFloat(e.data.threshold); + myModel.labels = labels; + await myModel.loadModel(); + myModel.warmUp(batch); + BACKEND = tf.getBackend(); + postMessage({ + message: "model-ready", + sampleRate: myModel.config.sampleRate, + chunkLength: myModel.chunkLength, + backend: tf.getBackend(), + labels: labels, + worker: worker + }); + }); + break; + } + case "predict": { + if (myModel.model_loaded) { + const { chunks: chunks, start: start, fileStart: fileStart, file: file, snr: snr, confidence: confidence, context: context, resetResults: resetResults } = e.data; + myModel.useContext = context; + myModel.selection = !resetResults; + const [result,filename,startPosition] = await myModel.predictChunk(chunks, start, fileStart, file, snr, confidence / 1000); + response = { + message: "prediction", + file: filename, + result: result, + fileStart: startPosition, + worker: worker, + selection: myModel.selection + }; + postMessage(response); + myModel.result = []; + } + break; + } + case "get-spectrogram": { + const buffer = e.data.buffer; + if (buffer.length < myModel.chunkLength) { + return; + } + const specFile = e.data.file; + const filepath = e.data.filepath; + const spec_height = e.data.height; + const spec_width = e.data.width; + let image; + const signal = tf.tensor1d(buffer, "float32"); + const bufferTensor = myModel.normalise_audio(signal); + signal.dispose(); + const imageTensor = tf.tidy(() => { + return myModel.makeSpectrogram(bufferTensor); + }); + image = tf.tidy(() => { + let spec = myModel.fixUpSpecBatch(tf.expandDims(imageTensor, 0), spec_height, spec_width); + const spec_max = tf.max(spec); + return spec.mul(255).div(spec_max).dataSync(); + }); + bufferTensor.dispose(); + imageTensor.dispose(); + response = { + message: "spectrogram", + width: myModel.inputShape[2], + height: myModel.inputShape[1], + channels: myModel.inputShape[3], + image: image, + file: specFile, + filepath: filepath, + worker: worker + }; + postMessage(response); + break; + } case "list": { myModel.list = e.data.list; myModel.lat = parseFloat(e.data.lat); myModel.lon = parseFloat(e.data.lon); myModel.week = parseInt(e.data.week) || myModel.week; myModel.speciesThreshold = parseFloat(e.data.threshold); -if (DEBUG) { - console.log(`Setting list to ${myModel.list}`); -} -await myModel.setList(); -postMessage({ - message: "update-list", - blocked: BLOCKED_IDS, - lat: myModel.lat, - lon: myModel.lon, - week: myModel.week, - updateResults: true -}); -break; -} + if (DEBUG) { + console.log(`Setting list to ${myModel.list}`); + } + await myModel.setList(); + postMessage({ + message: "update-list", + blocked: BLOCKED_IDS, + lat: myModel.lat, + lon: myModel.lon, + week: myModel.week, + updateResults: true, + worker: worker + }); + break; + } } } // If worker was respawned @@ -386,9 +380,9 @@ class Model { const finalPrediction = newPrediction || prediction; const { indices, values } = tf.topk(finalPrediction, 5, true) //const adjusted_values = tf.div(1, tf.add(1, tf.exp(tf.mul(tf.neg(10), values.sub(0.6))))); - - const topIndices = indices.arraySync(); - const topValues = values.arraySync(); + const [topIndices, topValues] = await Promise.all([indices.array(), values.array()]).catch((err => console.log(err))); + // const topIndices = indices.arraySync(); + // const topValues = await values.arraySync(); indices.dispose(); values.dispose(); diff --git a/js/worker.js b/js/worker.js index 4242da73..91e4a3dc 100644 --- a/js/worker.js +++ b/js/worker.js @@ -17,7 +17,6 @@ let WINDOW_SIZE = 3; let NUM_WORKERS; let workerInstance = 0; let TEMP, appPath, CACHE_LOCATION, BATCH_SIZE, LABELS, BACKEND, batchChunksToSend = {}; -let SEEN_LIST_UPDATE = false // Prevents list updates from every worker on every change const DEBUG = false; @@ -67,7 +66,8 @@ const createDB = async (file) => { await db.runAsync('CREATE TABLE species(id INTEGER PRIMARY KEY, sname TEXT NOT NULL, cname TEXT NOT NULL)'); await db.runAsync(`CREATE TABLE files(id INTEGER PRIMARY KEY, name TEXT,duration REAL,filestart INTEGER, locationID INTEGER, UNIQUE (name))`); await db.runAsync(`CREATE TABLE locations( id INTEGER PRIMARY KEY, lat REAL NOT NULL, lon REAL NOT NULL, place TEXT NOT NULL, UNIQUE (lat, lon))`); - + // await db.runAsync('CREATE TABLE blocked_species (lat REAL, lon REAL, week INTEGER, list TEXT NOT NULL, speciesID INTEGER NOT NULL, PRIMARY KEY (lat,lon,week,list,speciesID))'); + //await db.runAsync('CREATE INDEX blocked_species_idx on blocked_species(lat,lon,week)'); // Ensure place names are unique too await db.runAsync('CREATE UNIQUE INDEX idx_unique_place ON locations(lat, lon)'); await db.runAsync(`CREATE TABLE records( dateTime INTEGER, position INTEGER, fileID INTEGER, speciesID INTEGER, confidence INTEGER, label TEXT, comment TEXT, end INTEGER, callCount INTEGER, isDaylight INTEGER, UNIQUE (dateTime, fileID, speciesID), CONSTRAINT fk_files FOREIGN KEY (fileID) REFERENCES files(id) ON DELETE CASCADE, FOREIGN KEY (speciesID) REFERENCES species(id))`); @@ -129,15 +129,17 @@ async function loadDB(path) { await createDB(file); } else if (diskDB?.filename !== file) { diskDB = new sqlite3.Database(file); - // Add the blocked_species table if it is not present - const {changes} = await diskDB.runAsync(`CREATE TABLE IF NOT EXISTS blocked_species ( - fileID INTEGER, - speciesID INTEGER, - PRIMARY KEY (fileID, speciesID), - FOREIGN KEY (fileID) REFERENCES files(id) ON DELETE CASCADE, - FOREIGN KEY (speciesID) REFERENCES species(id) ON DELETE CASCADE - )`) - changes && console.log('Created a new blocked_species table'); + // // Add the blocked_species table if it is not present + // const {changes} = await diskDB.runAsync(` + // CREATE TABLE IF NOT EXISTS blocked_species ( + // lat REAL NOT NULL, + // lon REAL NOT NULL, + // week INTEGER NOT NULL, + // list TEXT NOT NULL, + // speciesID INTEGER NOT NULL, + // PRIMARY KEY (lat,lon,week,speciesID))`); + // await diskDB.runAsync('CREATE INDEX IF NOT EXISTS blocked_species_idx on blocked_species(lat,lon,week)') + // changes && console.log('Created a new blocked_species table and index'); STATE.update({ db: diskDB }); await diskDB.runAsync('VACUUM'); await diskDB.runAsync('PRAGMA foreign_keys = ON'); @@ -372,7 +374,6 @@ case "update-file-start": {await onUpdateFileStart(args); } case "update-list": { UI.postMessage({ event: "show-spinner" }); - SEEN_LIST_UPDATE = false; STATE.list = args.list; setBlockedIDs(STATE.lat, STATE.lon, STATE.week) break; @@ -520,14 +521,14 @@ const prepSummaryStatement = (blocked) => { } if (blocked.length) { - // const excluded = prepParams(blocked); - // summaryStatement += ` AND speciesID NOT IN (${excluded}) `; - ` AND NOT EXISTS ( - SELECT 1 - FROM blocked_species - WHERE blocked_species.fileID = files.id - AND blocked_species.speciesID = records.speciesID - ) ` + const excluded = prepParams(blocked); + summaryStatement += ` AND speciesID NOT IN (${excluded}) `; + // ` AND NOT EXISTS ( + // SELECT 1 + // FROM blocked_species + // WHERE blocked_species.fileID = files.id + // AND blocked_species.speciesID = records.speciesID + // ) ` } if (STATE.detect.nocmig){ summaryStatement += ' AND COALESCE(isDaylight, 0) != 1 '; @@ -637,7 +638,7 @@ const prepSummaryStatement = (blocked) => { resultStatement += ` AND locationID = ${STATE.locationID} `; } if (STATE.detect.nocmig){ - resultStatement += ' AND COALESCE(isDaylight, 0) != 1 '; + resultStatement += ' AND COALESCE(isDaylight, 0) != 1 '; // Backward compatibility for < v0.9. } resultStatement += `) @@ -1212,7 +1213,7 @@ const prepSummaryStatement = (blocked) => { */ const getPredictBuffers = async ({ - file = '', start = 0, end = undefined, worker = undefined + file = '', start = 0, end = undefined }) => { let chunkLength = STATE.model === 'birdnet' ? 144_000 : 72_000; // Ensure max and min are within range @@ -1261,7 +1262,7 @@ const prepSummaryStatement = (blocked) => { if (++workerInstance >= NUM_WORKERS) { workerInstance = 0; } - worker = workerInstance; + let worker = workerInstance; feedChunksToModel(myArray, chunkStart, file, end, worker); chunkStart += WINDOW_SIZE * BATCH_SIZE * sampleRate; // Now the async stuff is done ==> @@ -1284,7 +1285,7 @@ const prepSummaryStatement = (blocked) => { } // Create array with 0's (short segment of silence that will trigger the finalChunk flag const myArray = new Float32Array(Array.from({length: chunkLength}).fill(0)); - feedChunksToModel(myArray, chunkStart, file, end, worker); + feedChunksToModel(myArray, chunkStart, file, end); readStream.resume(); } }) @@ -1385,9 +1386,8 @@ const prepSummaryStatement = (blocked) => { file = '', start = 0, end = metadata[file].duration, - worker = undefined }) { - await getPredictBuffers({ file: file, start: start, end: end, worker: undefined }); + await getPredictBuffers({ file: file, start: start, end: end }); UI.postMessage({ event: 'update-audio-duration', value: metadata[file].duration }); } @@ -1427,7 +1427,8 @@ const prepSummaryStatement = (blocked) => { file: parts.base, buffer: buffer, height: 256, - width: 384 + width: 384, + worker: workerInstance }, [buffer.buffer]); } } @@ -1514,7 +1515,8 @@ const prepSummaryStatement = (blocked) => { file: file, buffer: buffer, height: height, - width: width + width: width, + worker: workerInstance }, [buffer.buffer]); count++; } @@ -1707,11 +1709,11 @@ const prepSummaryStatement = (blocked) => { function spawnWorkers(model, list, batchSize, threads) { NUM_WORKERS = threads; // And be ready to receive the list: - SEEN_LIST_UPDATE = false; for (let i = 0; i < threads; i++) { const workerSrc = model === 'v3' ? 'BirdNet' : model === 'birdnet' ? 'BirdNet2.4' : 'model'; const worker = new Worker(`./js/${workerSrc}.js`, { type: 'module' }); - worker.isAvailable = false; + worker.isAvailable = true; + worker.isReady = false; predictWorkers.push(worker) console.log('loading a worker') worker.postMessage({ @@ -1723,7 +1725,8 @@ const prepSummaryStatement = (blocked) => { lat: STATE.lat, lon: STATE.lon, week: STATE.week, - threshold: STATE.speciesThreshold + threshold: STATE.speciesThreshold, + worker: i }) worker.onmessage = async (e) => { await parseMessage(e) @@ -1979,19 +1982,24 @@ const prepSummaryStatement = (blocked) => { let SEEN_MODEL_READY = false; async function parseMessage(e) { const response = e.data; + // Update this worker's avaialability + predictWorkers[response.worker].isAvailable = true; + response.worker switch (response['message']) { - case "model-ready": {if ( !SEEN_MODEL_READY) { - SEEN_MODEL_READY = true; - sampleRate = response["sampleRate"]; - const backend = response["backend"]; - console.log(backend); - setBlockedIDs(STATE.lat,STATE.lon,STATE.week) - UI.postMessage({ - event: "model-ready", - message: "ready", - backend: backend, - sampleRate: sampleRate - }); + case "model-ready": { + predictWorkers[response.worker].isReady = true; + if ( !SEEN_MODEL_READY) { + SEEN_MODEL_READY = true; + sampleRate = response["sampleRate"]; + const backend = response["backend"]; + console.log(backend); + setBlockedIDs(STATE.lat,STATE.lon,STATE.week) + UI.postMessage({ + event: "model-ready", + message: "ready", + backend: backend, + sampleRate: sampleRate + }); } break; } @@ -2000,7 +2008,7 @@ const prepSummaryStatement = (blocked) => { predictWorkers[response.worker].isAvailable = true; let worker = await parsePredictions(response); if (predictionsReceived[response.file] === predictionsRequested[response.file]) { - const limit = 100; + const limit = 10; clearCache(CACHE_LOCATION, limit); if (filesBeingProcessed.length) { processNextFile({ @@ -2020,41 +2028,42 @@ const prepSummaryStatement = (blocked) => { break; } case "update-list": { - if ( !SEEN_LIST_UPDATE) { - if (STATE.list === 'location'){ - // Let's create our list cache - const {week, lat, lon, blocked} = response; - const location = lat.toFixed(2) + lon.toFixed(2); - if (! (STATE.blocked[week] && STATE.blocked[week][location])) { - STATE.blocked[week] = {}; - STATE.blocked[week][location] = blocked; - } else { - DEBUG && console.log("Unnecesary call to generate location list") - } + const {week, lat, lon, blocked} = response; + if (STATE.list === 'location'){ + // Let's create our list cache + + const location = lat.toFixed(2) + lon.toFixed(2); + if (! (STATE.blocked[week] && STATE.blocked[week][location])) { + STATE.blocked[week] = {}; + STATE.blocked[week][location] = blocked; } else { - STATE.blocked = response.blocked; + DEBUG && console.log("Unnecesary call to generate location list") } - STATE.globalOffset = 0; - try { - const fileID = await STATE.db.getAsync('SELECT id AS fileID FROM files WHERE name = ?', response.file); - await STATE.db.runAsync('BEGIN'); - let stmt = STATE.db.prepare("INSERT INTO blocked_species (fileID, speciesID) VALUES (?, ?);"); - response.blocked.forEach(speciesID => { - stmt.run(fileID, speciesID); - }) - await STATE.db.runAsync('END'); - } catch (error) { - console.log('setting blocked list didn\'t work', error) - } - - - SEEN_LIST_UPDATE = true; - UI.postMessage({ event: "results-complete" }); - if (response["updateResults"] && STATE.db) { - await Promise.all([getResults(), getSummary()]); - if (["explore", "chart"].includes(STATE.mode)) { - getDetectedSpecies(); - } + } else { + STATE.blocked = blocked; + } + STATE.globalOffset = 0; + // try { + // const list = STATE.list; + // let SQLweek = week, SQLlat = lat, SQLlon = lon; + // if (list !== 'location') { + // SQLlat = SQLlon = SQLweek = null; + // } + // await STATE.db.runAsync('BEGIN'); + // let stmt = STATE.db.prepare("INSERT OR IGNORE INTO blocked_species (lat, lon, week, list, model, speciesID) VALUES (?, ?, ?, ?, ?)"); + // response.blocked.forEach(speciesID => { + // stmt.run(SQLlat, SQLlon, SQLweek, list, speciesID); + // }) + // await STATE.db.runAsync('END'); + // } catch (error) { + // console.log('setting blocked list didn\'t work', error) + // } + + UI.postMessage({ event: "results-complete" }); + if (response["updateResults"] && STATE.db) { + await Promise.all([getResults(), getSummary()]); + if (["explore", "chart"].includes(STATE.mode)) { + getDetectedSpecies(); } } break; @@ -2663,13 +2672,6 @@ const prepSummaryStatement = (blocked) => { */ const getValidSpecies = async (file) => { const blocked = getBlockedIDs(file); - if (!SEEN_LIST_UPDATE){ - UI.postMessage({ - event: 'generate-alert', - message: 'Still preparing the detection list, try again in a few moments' - }); - return - } let excluded, included; let sql = `SELECT cname, sname FROM species`; if (blocked.length) { @@ -2994,16 +2996,31 @@ const prepSummaryStatement = (blocked) => { return blocked; } - function setBlockedIDs(lat,lon,week){ - SEEN_LIST_UPDATE = false; + async function setBlockedIDs(lat,lon,week){ // Look for an idle worker, or push to the first worker queue - const readyWorker = predictWorkers.find(obj => obj.isAvailable) || predictWorkers[0]; - readyWorker.postMessage({ + const readyWorker = await waitForWorker(predictWorkers); + predictWorkers[readyWorker].postMessage({ message: "list", list: STATE.list, lat: lat, lon: lon, week: week, - threshold: STATE.speciesThreshold + threshold: STATE.speciesThreshold, + worker: readyWorker + }); + } + + function waitForWorker(predictWorkers) { + let count = 0 + return new Promise(resolve => { + const checkAvailability = () => { + const readyWorker = predictWorkers.findIndex(obj => obj.isAvailable && obj.isReady) ; + if (readyWorker > -1) { + resolve(readyWorker); + } else { + setTimeout(checkAvailability, 100); // Check again in 100 milliseconds + } + }; + checkAvailability(); }); } \ No newline at end of file