From ec4bb2f1dfa038878536edd895c4b4ee534cf5fc Mon Sep 17 00:00:00 2001 From: Mattk70 Date: Tue, 6 Feb 2024 22:37:38 +0000 Subject: [PATCH] Extracted list generatino to seperate worker. Promisified that worker. --- js/BirdNet2.4.js | 32 ++++--- js/database.js | 2 +- js/listWorker.js | 217 +++++++++++++++++++++++++++++++++++++++++++ js/worker.js | 237 +++++++++++++++++++++++++++++------------------ 4 files changed, 383 insertions(+), 105 deletions(-) create mode 100644 js/listWorker.js diff --git a/js/BirdNet2.4.js b/js/BirdNet2.4.js index 3167a738..e6e03376 100644 --- a/js/BirdNet2.4.js +++ b/js/BirdNet2.4.js @@ -81,9 +81,7 @@ const NOT_BIRDS = [ "Tamiasciurus hudsonicus_Red Squirrel"]; const MYSTERIES = ['Unknown Sp._Unknown Sp.']; -const GRAYLIST = []; -const GOLDEN_LIST = [] -let BLOCKED_IDS = []; +let INCLUDED_IDS = []; let SUPPRESSED_IDS = []; let ENHANCED_IDS = []; const CONFIG = { @@ -219,11 +217,11 @@ onmessage = async (e) => { await myModel.setList(); postMessage({ message: "update-list", - blocked: BLOCKED_IDS, + included: INCLUDED_IDS, lat: myModel.lat, lon: myModel.lon, week: myModel.week, - updateResults: false, + updateResults: true, worker: worker }); break; @@ -285,8 +283,9 @@ class Model { } async setList() { - BLOCKED_IDS = []; - if (this.list === "everything") return + if (this.list === "everything") { + INCLUDED_IDS = this.labels.map((_, index) => index); + } else if (this.list === 'location'){ const lat = this.lat; const lon = this.lon; @@ -300,19 +299,28 @@ class Model { for (let i = 0; i < mdata_probs.length; i++) { if (mdata_probs[i] > this.speciesThreshold) { count++; + INCLUDED_IDS.push(i); DEBUG && console.log("including:", this.labels[i] + ': ' + mdata_probs[i]); + } else { DEBUG && console.log("Excluding:", this.labels[i] + ': ' + mdata_probs[i]); - // Hack to add Dotterel?? - //if (! this.labels[i].includes('Dotterel')) - BLOCKED_IDS.push(i) } } DEBUG && console.log('Total species considered at this location: ', count) } else { - // find the position of the blocked items in the label list - NOT_BIRDS.forEach(notBird => BLOCKED_IDS.push(this.labels.indexOf(notBird))) + // Function to extract the first element after splitting on '_' + const getFirstElement = label => label.split('_')[0]; + + // Create a list of included labels' indices + const t0 = Date.now() + INCLUDED_IDS = this.labels + .map((label, index) => { + const firstPart = getFirstElement(label); + return NOT_BIRDS.some(excludedLabel => getFirstElement(excludedLabel) === firstPart) ? null : index; + }) + .filter(index => index !== null); + console.log('filtering took', Date.now() - t0, 'ms') } } diff --git a/js/database.js b/js/database.js index 02430b0e..1ebe3f02 100644 --- a/js/database.js +++ b/js/database.js @@ -25,7 +25,7 @@ sqlite3.Statement.prototype.allAsync = function (...params) { if (DEBUG) console.log('SQL\n', this.sql, '\nParams\n', params) return new Promise((resolve, reject) => { this.all(params, (err, rows) => { - if (err) return reject(console.log(err, sql)); + if (err) return reject(console.log(err, this.sql)); if (DEBUG) console.log('\nRows:', rows) resolve(rows); }); diff --git a/js/listWorker.js b/js/listWorker.js new file mode 100644 index 00000000..845700a0 --- /dev/null +++ b/js/listWorker.js @@ -0,0 +1,217 @@ +const tf = require('@tensorflow/tfjs-node'); +const fs = require('node:fs'); +const path = require('node:path'); +let DEBUG = false; +let BACKEND; + +//GLOBALS +let listModel; +let NOT_BIRDS; +const MIGRANTS = new Set(["Pluvialis dominica_American Golden Plover", "Acanthis hornemanni_Arctic Redpoll", "Sterna paradisaea_Arctic Tern", "Recurvirostra avosetta_Avocet", "Porzana pusilla_Baillon's Crake", "Limosa lapponica_Bar-tailed Godwit", "Tyto alba_Barn Owl", "Branta leucopsis_Barnacle Goose", "Cygnus columbianus_Bewick's Swan", "Botaurus stellaris_Bittern (call)", "Chroicocephalus ridibundus_Black-headed Gull", "Podiceps nigricollis_Black-necked Grebe", "Limosa limosa_Black-tailed Godwit", "Turdus merula_Blackbird (flight call)", "Sylvia atricapilla_Blackcap (call)", "Fringilla montifringilla_Brambling", "Branta bernicla_Brent Goose", "Branta canadensis_Canada Goose", "Larus cachinnans_Caspian Gull", "Phylloscopus collybita_Chiffchaff (call)", "Loxia curvirostra_Common Crossbill", "Larus canus_Common Gull", "Acanthis flammea_Common Redpoll", "Actitis hypoleucos_Common Sandpiper", "Melanitta nigra_Common Scoter", "Sterna hirundo_Common Tern", "Fulica atra_Coot", "Emberize calandre_Corn Bunting (call)", "Crex crex_Corncrake", "Cuculus canorus_Cuckoo (call)", "Calidris ferruginea_Curlew Sandpiper", "Numenius arquata_Curlew", "Charadrius morinellus_Dotterel", "Calidris alpina_Dunlin", "Prunella modularis_Dunnock (call)", "Alopochen aegyptiaca_Egyptian Goose", "Turdus pilaris_Fieldfare (call)", "Mareca strepera_Gadwall", "Sylvia borin_Garden Warbler (call)", "Spatula querquedula_Garganey", "Regulus regulus_Goldcrest (call)", "Regulus ignicapilla_Firecrest (call)", "Pluvialis apricaria_Golden Plover", "Bucephala clangula_Goldeneye", "Mergus merganser_Goosander", "Locustella naevia_Grasshopper Warbler (call)", "Larus marinus_Great Black-backed Gull", "Podiceps cristatus_Great Crested Grebe", "Tringa ochropus_Green Sandpiper", "Tringa nebularia_Greenshank", "Ardea cinerea_Grey Heron", "Perdix perdix_Grey Partridge", "Phalaropus fulicarius_Grey", "Pluvialis squatarola_Grey Plover", "Motacilla cinerea_Grey Wagtail ", "Anser anser_Greylag Goose", "Delichon urbicum_House Martin", "Coccothraustes coccothraustes_Hawfinch (call)", "Larus argentatus_Herring Gull", "Lymnocryptes minimus_Jack Snipe", "Alcedo atthis_Kingfisher", "Calidris canutus_Knot", "Calcarius lapponicus_Lapland Bunting (call)", "Larus fuscus_Lesser Black-backed Gull", "Acanthis cabaret_Lesser Redpoll ", "Curraca curruca_Lesser Whitethroat (call)", "Linaria cannabina_Linnet", "Ixobrychus minutus_Little Bittern (call)", "Egretta garzetta_Little Egret", "Tachybaptus ruficollis_Little Grebe", "Hydrocoloeus minutus_Little Gull", "Athene noctua_Little Owl", "Charadrius dubius_Little Ringed Plover", "Calidris minuta_Little Stint ", "Sternula albifrons_Little Tern", "Asio otus_Long-eared Owl", "Clangula hyemalis_Long-tailed Duck", "Anas platyrhynchos_Mallard", "Aix galericulata_Mandarin Duck", "Anthus pratensis_Meadow Pipit (call)", "Ichthyaetus melanocephalus_Mediterranean Gull", "Turdus viscivorus_Mistle Thrush (call)", "Gallinula chloropus_Moorhen", "Nycticorax nycticorax_Night Heron", "Luscinia megarhynchos_Nightingale (call)", "Luscinia megarhynchos_Nightingale (song)", "Caprimulgus europaeus_Nightjar (call)", "Anthus hodgsoni_Olive-backed Pipit (call)", "Emberiza hortulana_Ortolan Bunting (call)", "Emberiza pusilla_Little Bunting (call)", "Haematopus ostralegus_Oystercatcher", "Ficedula hypoleuca_Pied Flycatcher (call)", "Motacilla alba_Pied Wagtail", "Anser brachyrhynchus_Pink-footed Goose", "Anas acuta_Pintail", "Aythya ferina_Pochard", "Calidris maritima_Purple Sandpiper", "Coturnix coturnix_Quail (call)", "Coturnix coturnix_Quail (song)", "Mergus serrator_Red-breasted Merganser", "Netta rufina_Red-crested Pochard", "Alectoris rufa_Red-legged Partridge", "Tringa totanus_Redshank", "Phoenicurus phoenicurus_Redstart (call)", "Turdus iliacus_Redwing (call)", "Emberiza schoeniclus_Reed Bunting (call)", "Acrocephalus scirpaceus_Reed Warbler (call)", "Anthus richardi_Richard's Pipit (call)", "Turdus torquatus_Ring Ouzel (call)", "Charadrius hiaticula_Ringed Plover", "Erithacus rubecula_Robin (flight call)", "Anthus petrosus_Rock Pipit", "Sterna dougallii_Roseate Tern", "Calidris pugnax_Ruff", "Riparia riparia_Sand Martin", "Calidris alba_Sanderling", "Thalasseus sandvicensis_Sandwich Tern", "Aythya marila_Scaup", "Loxia scotica_Scottish Crossbill", "Acrocephalus schoenobaenus_Sedge Warbler", "Tadorna tadorna_Shelduck", "Asio flammeus_Short-eared Owl", "Spatula clypeata_Shoveler", "Spinus spinus_Siskin", "Alauda arvensis_Skylark (call)", "Gallinago gallinago_Snipe", "Plectrophenax nivalis_Snow Bunting", "Turdus philomelos_Song Thrush (call)", "Porzana porzana_Spotted Crake", "Muscicapa striata_Spotted Flycatcher", "Tringa erythropus_Spotted Redshank (call)", "Burhinus oedicnemus_Stone-curlew", "Saxicola rubicola_Stonechat", "Hirundo rustica_Swallow", "Apus apus_Swift", "Anser fabalis_Taiga Bean Goose", "Strix aluco_Tawny Owl", "Anas crecca_Teal", "Anthus trivialis_Tree Pipit (call)", "Aythya fuligula_Tufted Duck", "Anser serrirostris_Tundra Bean Goose", "Arenaria interpres_Turnstone", "Anthus spinoletta_Water Pipit", "Rallus aquaticus_Water Rail", "Numenius phaeopus_Whimbrel", "Anser albifrons_White-fronted Goose", "Sylvia communis_Whitethroat (call)", "Cygnus cygnus_Whooper Swan", "Mareca penelope_Wigeon", "Phylloscopus trochilus_Willow Warbler (call)", "Tringa glareola_Wood Sandpiper", "Scolopax rusticola_Woodcock", "Lullula arborea_Woodlark (call)", "Larus michahellis_Yellow-legged Gull", "Motacilla flava_Yellow Wagtail", "Emberiza citrinella_Yellowhammer (call)"]); +const CHIRPITY_NOT_BIRDS = ['Ambient Noise_Ambient Noise', 'Animal_Animal', 'Cat_Cat', 'Church Bells_Church Bells', 'Cough_Cough', 'Dog_Dog', 'Human_Human', 'Laugh_Laugh', 'No call_No call', 'Rain_Rain', 'Red Fox_Red Fox', 'Sneeze_Sneeze', 'Snoring_Snoring', 'Thunder_Thunder', 'Vehicle_Vehicle', 'Water Drops_Water Drops', 'Waves_Waves', 'Wind_Wind']; +const BIRDNET_NOT_BIRDS = [ + 'Dog_Dog', + 'Environmental_Environmental', + 'Engine_Engine', + 'Fireworks_Fireworks', + 'Gun_Gun', + 'Human non-vocal_Human non-vocal', + 'Human vocal_Human vocal', + 'Human whistle_Human whistle', + 'Miogryllus saussurei_Miogryllus saussurei', + 'Noise_Noise', + 'Power tools_Power tools', + 'Siren_Siren', + "Canis latrans_Coyote", + "Canis lupus_Gray Wolf", + "Gastrophryne carolinensis_Eastern Narrow-mouthed Toad", + "Gastrophryne olivacea_Great Plains Narrow-mouthed Toad", + "Incilius valliceps_Gulf Coast Toad", + "Anaxyrus americanus_American Toad", + "Anaxyrus canorus_Yosemite Toad", + "Anaxyrus cognatus_Great Plains Toad", + "Anaxyrus fowleri_Fowler's Toad", + "Anaxyrus houstonensis_Houston Toad", + "Anaxyrus microscaphus_Arizona Toad", + "Anaxyrus quercicus_Oak Toad", + "Anaxyrus speciosus_Texas Toad", + "Anaxyrus terrestris_Southern Toad", + "Anaxyrus woodhousii_Woodhouse's Toad", + "Dryophytes andersonii_Pine Barrens Treefrog", + "Dryophytes arenicolor_Canyon Treefrog", + "Dryophytes avivoca_Bird-voiced Treefrog", + "Dryophytes chrysoscelis_Cope's Gray Treefrog", + "Dryophytes cinereus_Green Treefrog", + "Dryophytes femoralis_Pine Woods Treefrog", + "Dryophytes gratiosus_Barking Treefrog", + "Dryophytes squirellus_Squirrel Treefrog", + "Dryophytes versicolor_Gray Treefrog", + "Eleutherodactylus planirostris_Greenhouse Frog", + "Hyliola regilla_Pacific Chorus Frog", + "Lithobates catesbeianus_American Bullfrog", + "Lithobates clamitans_Green Frog", + "Lithobates palustris_Pickerel Frog", + "Lithobates sylvaticus_Wood Frog", + "Pseudacris brimleyi_Brimley's Chorus Frog", + "Pseudacris clarkii_Spotted Chorus Frog", + "Pseudacris crucifer_Spring Peeper", + "Pseudacris feriarum_Upland Chorus Frog", + "Pseudacris nigrita_Southern Chorus Frog", + "Pseudacris ocularis_Little Grass Frog", + "Pseudacris ornata_Ornate Chorus Frog", + "Pseudacris streckeri_Strecker's Chorus Frog", + "Pseudacris triseriata_Striped Chorus Frog", + "Acris crepitans_Northern Cricket Frog", + "Acris gryllus_Southern Cricket Frog", + "Eunemobius carolinus_Carolina Ground Cricket", + "Eunemobius confusus_Confused Ground Cricket", + "Gryllus assimilis_Gryllus assimilis", + "Gryllus fultoni_Southern Wood Cricket", + "Gryllus pennsylvanicus_Fall Field Cricket", + "Gryllus rubens_Southeastern Field Cricket", + "Neonemobius cubensis_Cuban Ground Cricket", + "Oecanthus celerinictus_Fast-calling Tree Cricket", + "Oecanthus exclamationis_Davis's Tree Cricket", + "Oecanthus fultoni_Snowy Tree Cricket", + "Oecanthus nigricornis_Blackhorned Tree Cricket", + "Oecanthus niveus_Narrow-winged Tree Cricket", + "Oecanthus pini_Pine Tree Cricket", + "Oecanthus quadripunctatus_Four-spotted Tree Cricket", + "Orocharis saltator_Jumping Bush Cricket", + "Alouatta pigra_Mexican Black Howler Monkey", + "Tamias striatus_Eastern Chipmunk", + "Tamiasciurus hudsonicus_Red Squirrel"]; + +const MYSTERIES = ['Unknown Sp._Unknown Sp.']; + + +const birdnetlabelFile = `../labels/V2.4/BirdNET_GLOBAL_6K_V2.4_Labels_en.txt`; +const BIRDNET_LABELS = await fetch(birdnetlabelFile).then(response => { + if (! response.ok) throw new Error('Network response was not ok'); + return response.text(); + }).then(filecontents => { + return filecontents.trim().split(/\r?\n/); + }).catch(error =>{ + console.error('There was a problem fetching the label file:', error); + }) + +let config = JSON.parse(fs.readFileSync(path.join(__dirname, '../chirpity_model_config.json'), "utf8")); +const CHIRPITY_LABELS = config.labels; +config = undefined; + + +/* USAGE EXAMPLES: +listWorker.postMessage({message: 'load'}) +listWorker.postMessage({message: 'get-list', model: 'chirpity', listType: 'location', useWeek: true, lat: 52.0, lon: -0.5, week: 40, threshold: 0.01 }) +*/ + +onmessage = async (e) => { + DEBUG && console.log('got a message', e.data) + const {message} = e.data; + let response; + try { + switch (message) { + + case "get-list": { + const {model, listType, useWeek} = e.data; + NOT_BIRDS = model === 'birdnet' ? BIRDNET_NOT_BIRDS : CHIRPITY_NOT_BIRDS; + listModel.labels = model === 'birdnet' ? BIRDNET_LABELS : CHIRPITY_LABELS; + let lat = parseFloat(e.data.lat); + let lon = parseFloat(e.data.lon); + let week = parseInt(e.data.week); + let threshold = parseFloat(e.data.threshold); + DEBUG && console.log(`Setting list to ${listType}`); + const includedIDs = await listModel.setList({lat, lon, week, listType, useWeek, threshold}); + postMessage({ + message: "your-list-sir", + included: includedIDs, + lat: listModel.lat, + lon: listModel.lon, + week: listModel.week + }); + break; + } + } + } + // If worker was respawned + catch (error) { + console.log(error) + } +}; + +class Model { + constructor(appPath) { + this.model_loaded = false; + this.appPath = appPath; + this.labels = undefined; // labels in the model we're filtering + } + + async loadModel() { + if (this.model_loaded === false) { + // Model files must be in a different folder than the js, assets files + if (DEBUG) console.log('loading model from', this.appPath); + this.metadata_model = await tf.loadGraphModel(this.appPath); + // const mdata_label_path = path.join(__dirname, '..','BirdNET_GLOBAL_6K_V2.4_Model_TFJS','static','model','labels.json') + this.mdata_labels = BIRDNET_LABELS; //JSON.parse(fs.readFileSync(mdata_label_path, "utf8")); // Labels used in the metadata model + } + } + + async setList({lat, lon, week, listType, useWeek, threshold}) { + let includedIDs = []; + week = useWeek ? week : -1; + if (listType === "everything") { + includedIDs = this.labels.map((_, index) => index); + } + + else if (listType === 'location'){ + DEBUG && console.log('lat', lat, 'lon', lon, 'week', week) + this.mdata_input = tf.tensor([lat, lon, week]).expandDims(0); + const mdata_prediction = this.metadata_model.predict(this.mdata_input); + const mdata_probs = await mdata_prediction.data(); + let count = 0; + for (let i = 0; i < mdata_probs.length; i++) { + if (mdata_probs[i] > threshold) { + count++; + includedIDs.push(i); + DEBUG && console.log("including:", this.labels[i] + ': ' + mdata_probs[i]); + + } else { + DEBUG && console.log("Excluding:", this.labels[i] + ': ' + mdata_probs[i]); + } + } + DEBUG && console.log('Total species considered at this location: ', count) + } + else { + // Function to extract the first element after splitting on '_' + const getFirstElement = label => label.split('_')[0]; + + // Create a list of included labels' indices + const t0 = Date.now() + includedIDs = this.labels.map((label, index) => { + const firstPart = getFirstElement(label); + return NOT_BIRDS.some(excludedLabel => getFirstElement(excludedLabel) === firstPart) ? null : index; + }).filter(index => index !== null); + console.log('filtering took', Date.now() - t0, 'ms') + } + return includedIDs; + } +} + +async function _init_(){ + DEBUG && console.log("load loading metadata_model"); + // const appPath = "../" + location + "/"; + DEBUG && console.log(`List generating model received load instruction.`); + tf.setBackend('tensorflow').then(async () => { + tf.enableProdMode(); + if (DEBUG) { + console.log(tf.env()); + console.log(tf.env().getFlags()); + } + listModel = new Model('../BirdNET_GLOBAL_6K_V2.4_Model_TFJS/static/model/mdata/model.json'); + + await listModel.loadModel(); + postMessage({ message: "list-model-ready"}); + }); +} + +await _init_(); \ No newline at end of file diff --git a/js/worker.js b/js/worker.js index 91e4a3dc..a87c2de7 100644 --- a/js/worker.js +++ b/js/worker.js @@ -17,7 +17,7 @@ let WINDOW_SIZE = 3; let NUM_WORKERS; let workerInstance = 0; let TEMP, appPath, CACHE_LOCATION, BATCH_SIZE, LABELS, BACKEND, batchChunksToSend = {}; - +let LIST_WORKER; const DEBUG = false; const DATASET = false; @@ -253,6 +253,9 @@ async function handleMessage(e) { switch (action) { case "_init_": { const {model, batchSize, threads, backend, list} = args; + const t0 = Date.now(); + LIST_WORKER = await spawnListWorker(); + console.log('List worker took', Date.now() - t0, 'ms to load'); await onLaunch({model: model, batchSize: batchSize, threads: threads, backend: backend, list: list}); break; } @@ -344,7 +347,7 @@ case "load-model": { // metadata = {}; ipcRenderer.invoke('clear-cache', CACHE_LOCATION) - STATE.blocked = []; + STATE.included = []; } predictWorkers.length && terminateWorkers(); await onLaunch(args); @@ -375,7 +378,7 @@ case "update-file-start": {await onUpdateFileStart(args); case "update-list": { UI.postMessage({ event: "show-spinner" }); STATE.list = args.list; - setBlockedIDs(STATE.lat, STATE.lon, STATE.week) + await setIncludedIDs(STATE.lat, STATE.lon, STATE.week) break; } case 'update-locale': { @@ -386,8 +389,8 @@ case 'update-locale': { case "update-state": { TEMP = args.temp || TEMP; appPath = args.path || appPath; - // If we change the speciesThreshold, we need to invalidate the blocked id cache - if (args.speciesThreshold) STATE.blocked = {}; + // If we change the speciesThreshold, we need to invalidate the included id cache + if (args.speciesThreshold) STATE.included = {}; STATE.update(args); break; } @@ -411,6 +414,7 @@ async function onChangeMode(mode) { }); } +const filtersApplied = () => STATE.included.length < LABELS.length -1; /** * onLaunch called when Application is first opened or when model changed @@ -427,9 +431,64 @@ async function onLaunch({model = 'chirpity', batchSize = 32, threads = 1, backen STATE.update({ model: model }); await loadDB(appPath); // load the diskdb await createDB(); // now make the memoryDB - spawnWorkers(model, list, batchSize, threads); + spawnPredictWorkers(model, list, batchSize, threads); } + +// function spawnListWorker() { +// const worker = new Worker('./js/listWorker.js', { type: 'module' }); + +// return function listWorker(message) { +// return new Promise((resolve, reject) => { +// worker.onmessage = function(event) { +// resolve(event.data); +// }; + +// worker.onerror = function(error) { +// reject(error); +// }; + +// console.log('posting message') +// worker.postMessage(message); +// }); +// }; +// } + +async function spawnListWorker() { + const worker_1 = await new Promise((resolve, reject) => { + const worker = new Worker('./js/listWorker.js', { type: 'module' }); + + worker.onmessage = function (event) { + // Resolve the promise once the worker sends a message indicating it's ready + if (event.data.message === 'list-model-ready') { + resolve(worker); + } + }; + + worker.onerror = function (error) { + reject(error); + }; + + // Start the worker + worker.postMessage('start'); + }); + return function listWorker(message_1) { + return new Promise((resolve_1, reject_1) => { + worker_1.onmessage = function (event_1) { + resolve_1(event_1.data); + }; + + worker_1.onerror = function (error_1) { + reject_1(error_1); + }; + + console.log('posting message'); + worker_1.postMessage(message_1); + }); + }; +} + + /** * Generates a list of supported audio files, recursively searching directories. * Sends this list to the UI @@ -486,7 +545,7 @@ const prepParams = (list) => list.map(item => '?').join(','); * @returns a string, like (?,?,?) */ -const getSummaryParams = (blocked) => { +const getSummaryParams = (included) => { const range = STATE.mode === 'explore' ? STATE.explore.range : STATE.selection?.range; const useRange = range?.start; @@ -496,13 +555,13 @@ const getSummaryParams = (blocked) => { extraParams.push(...STATE.filesToAnalyse); } else if (useRange) params.push(range.start, range.end); - extraParams.push(...blocked); + filtersApplied() && extraParams.push(...included); STATE.locationID && extraParams.push(STATE.locationID); params.push(...extraParams); return params } -const prepSummaryStatement = (blocked) => { +const prepSummaryStatement = (included) => { const range = STATE.mode === 'explore' ? STATE.explore.range : undefined; const useRange = range?.start; let summaryStatement = ` @@ -520,9 +579,9 @@ const prepSummaryStatement = (blocked) => { summaryStatement += ' AND dateTime BETWEEN ? AND ? '; } - if (blocked.length) { - const excluded = prepParams(blocked); - summaryStatement += ` AND speciesID NOT IN (${excluded}) `; + if (filtersApplied()) { + const includedParams = prepParams(included); + summaryStatement += ` AND speciesID IN (${includedParams}) `; // ` AND NOT EXISTS ( // SELECT 1 // FROM blocked_species @@ -564,7 +623,7 @@ const prepSummaryStatement = (blocked) => { params.push(species); SQL += ' AND speciesID = (SELECT id from species WHERE cname = ?) '; }// This will overcount as there may be a valid species ranked above it - else if (STATE.blocked.length) SQL += ` AND speciesID not in (${STATE.blocked}) `; + else if (filtersApplied()) SQL += ` AND speciesID IN (${STATE.included}) `; if (useRange) SQL += ` AND dateTime BETWEEN ${range.start} AND ${range.end} `; if (STATE.detect.nocmig) SQL += ' AND COALESCE(isDaylight, 0) != 1 '; if (STATE.locationID) SQL += ` AND locationID = ${STATE.locationID}`; @@ -581,11 +640,11 @@ const prepSummaryStatement = (blocked) => { - const getResultsParams = (species, confidence, offset, limit, topRankin, blocked) => { + const getResultsParams = (species, confidence, offset, limit, topRankin, included) => { const params = []; params.push(confidence); ['analyse', 'archive'].includes(STATE.mode) && !STATE.selection && params.push(...STATE.filesToAnalyse); - blocked.length && params.push(...blocked); + filtersApplied() && params.push(...included); params.push(topRankin); species && params.push(species); @@ -593,7 +652,7 @@ const prepSummaryStatement = (blocked) => { return params } - const prepResultsStatement = (species, noLimit, blocked) => { + const prepResultsStatement = (species, noLimit, included) => { let resultStatement = ` WITH ranked_records AS ( SELECT @@ -632,7 +691,7 @@ const prepSummaryStatement = (blocked) => { if (useRange) { resultStatement += ` AND dateTime BETWEEN ${range.start} AND ${range.end} `; } - if (blocked.length) resultStatement += ` AND speciesID NOT IN (${prepParams(blocked)}) `; + if (filtersApplied()) resultStatement += ` AND speciesID IN (${prepParams(included)}) `; if (STATE.selection) resultStatement += ` AND name = '${FILE_QUEUE[0]}' `; if (STATE.locationID) { resultStatement += ` AND locationID = ${STATE.locationID} `; @@ -763,7 +822,7 @@ const prepSummaryStatement = (blocked) => { if (filesBeingProcessed.length) { //restart the worker terminateWorkers(); - spawnWorkers(model, list, BATCH_SIZE, NUM_WORKERS) + spawnPredictWorkers(model, list, BATCH_SIZE, NUM_WORKERS) } filesBeingProcessed = []; predictionsReceived = {}; @@ -880,7 +939,7 @@ const prepSummaryStatement = (blocked) => { await setMetadata({ file: file, proxy: proxy, source_file: source_file }); /*This is where we add week checking... GENERATING A WEEK SPECIFIC LIST FOR A LOCATION IS A *REALLY* EXPENSIVE TASK. - LET'S CACHE BLOCKED IDS FOR WEEK AND LOCATION. NEED TO ADAPT STATE.BLOCKED_IDS + LET'S CACHE included IDS FOR WEEK AND LOCATION. NEED TO ADAPT STATE.BLOCKED_IDS SO IT CAN BE USED THIS WAY. DEFAULT KEY -1. STRUCTURE: BLOCKED_IDS.week.location = []; */ @@ -888,8 +947,8 @@ const prepSummaryStatement = (blocked) => { const meta = metadata[file]; const week = STATE.useWeek ? new Date(meta.fileStart).getWeekNumber() : "-1"; const location = STATE.lat + STATE.lon; - if (! (STATE.blocked[week] && STATE.blocked[week][location])) { - setBlockedIDs(STATE.lat,STATE.lon,week) + if (! (STATE.included[week] && STATE.included[week][location])) { + await setIncludedIDs(STATE.lat,STATE.lon,week) } } } @@ -1457,9 +1516,9 @@ const prepSummaryStatement = (blocked) => { JOIN species ON species.id = records.speciesID JOIN files ON records.fileID = files.id - WHERE speciesID NOT IN (${prepParams(STATE.blocked)}) + ${filtersApplied() ? `WHERE speciesID IN (${prepParams(STATE.included)}` : ''}) AND confidence >= ${STATE.detect.confidence}`; - let params = STATE.blocked; + let params = filtersApplied() ? STATE.included : []; if (species) { db2ResultSQL += ` AND species.cname = ?`; params.push(species) @@ -1706,7 +1765,7 @@ const prepSummaryStatement = (blocked) => { /// Workers From the MDN example5 - function spawnWorkers(model, list, batchSize, threads) { + function spawnPredictWorkers(model, list, batchSize, threads) { NUM_WORKERS = threads; // And be ready to receive the list: for (let i = 0; i < threads; i++) { @@ -1904,7 +1963,7 @@ const prepSummaryStatement = (blocked) => { const parsePredictions = async (response) => { let file = response.file; - const blocked = getBlockedIDs(file); + const included = await getIncludedIDs(file); const latestResult = response.result, db = STATE.db; DEBUG && console.log('worker being used:', response.worker); if (! STATE.selection) await generateInsertQuery(latestResult, file); @@ -1920,7 +1979,7 @@ const prepSummaryStatement = (blocked) => { if (confidence < 0.05) break; confidence*=1000; let speciesID = speciesIDArray[j]; - updateUI = (confidence > STATE.detect.confidence && ! blocked.includes(speciesID)); + updateUI = (confidence > STATE.detect.confidence && included.includes(speciesID)); if (STATE.selection || updateUI) { let end, confidenceRequired; if (STATE.selection) { @@ -1993,7 +2052,6 @@ const prepSummaryStatement = (blocked) => { sampleRate = response["sampleRate"]; const backend = response["backend"]; console.log(backend); - setBlockedIDs(STATE.lat,STATE.lon,STATE.week) UI.postMessage({ event: "model-ready", message: "ready", @@ -2028,19 +2086,19 @@ const prepSummaryStatement = (blocked) => { break; } case "update-list": { - const {week, lat, lon, blocked} = response; + const {week, lat, lon, included} = response; if (STATE.list === 'location'){ // Let's create our list cache const location = lat.toFixed(2) + lon.toFixed(2); - if (! (STATE.blocked[week] && STATE.blocked[week][location])) { - STATE.blocked[week] = {}; - STATE.blocked[week][location] = blocked; + if (! (STATE.included[week] && STATE.included[week][location])) { + STATE.included[week] = {}; + STATE.included[week][location] = included; } else { DEBUG && console.log("Unnecesary call to generate location list") } } else { - STATE.blocked = blocked; + STATE.included = included; } STATE.globalOffset = 0; // try { @@ -2051,12 +2109,12 @@ const prepSummaryStatement = (blocked) => { // } // await STATE.db.runAsync('BEGIN'); // let stmt = STATE.db.prepare("INSERT OR IGNORE INTO blocked_species (lat, lon, week, list, model, speciesID) VALUES (?, ?, ?, ?, ?)"); - // response.blocked.forEach(speciesID => { + // response.included.forEach(speciesID => { // stmt.run(SQLlat, SQLlon, SQLweek, list, speciesID); // }) // await STATE.db.runAsync('END'); // } catch (error) { - // console.log('setting blocked list didn\'t work', error) + // console.log('setting included list didn\'t work', error) // } UI.postMessage({ event: "results-complete" }); @@ -2227,8 +2285,8 @@ const prepSummaryStatement = (blocked) => { } = {}) => { const db = STATE.db; - const blocked = STATE.selection ? [] : getBlockedIDs(); - prepSummaryStatement(blocked); + const included = STATE.selection ? [] : await getIncludedIDs(); + prepSummaryStatement(included); const offset = species ? STATE.filteredOffset[species] : STATE.globalOffset; let range, files = []; if (['explore', 'chart'].includes(STATE.mode)) { @@ -2238,7 +2296,7 @@ const prepSummaryStatement = (blocked) => { } t0 = Date.now(); - const params = getSummaryParams(blocked); + const params = getSummaryParams(included); const summary = await STATE.GET_SUMMARY_SQL.allAsync(...params); DEBUG && console.log("Get Summary took", (Date.now() - t0) / 1000, "seconds"); @@ -2291,9 +2349,9 @@ const prepSummaryStatement = (blocked) => { let index = offset; AUDACITY = {}; - const blocked = STATE.selection ? [] : getBlockedIDs(); - const params = getResultsParams(species, confidence, offset, limit, topRankin, blocked); - prepResultsStatement(species, limit === Infinity, blocked); + const included = STATE.selection ? [] : await getIncludedIDs(); + const params = getResultsParams(species, confidence, offset, limit, topRankin, included); + prepResultsStatement(species, limit === Infinity, included); const result = await STATE.GET_RESULT_SQL.allAsync(...params); if (format === 'text'){ @@ -2461,7 +2519,8 @@ const prepSummaryStatement = (blocked) => { }) return // nothing to do. Also will crash if trying to update disk from disk. } - const blocked = getBlockedIDs(args.file); + const included = await getIncludedIDs(args.file); + const filterClause = filtersApplied() ? `AND speciesID IN (${included} )` : '' await memoryDB.runAsync('BEGIN'); await memoryDB.runAsync(`INSERT OR IGNORE INTO disk.files SELECT * FROM files`); // Set the saved flag on files' metadata @@ -2475,7 +2534,7 @@ const prepSummaryStatement = (blocked) => { response = await memoryDB.runAsync(` INSERT OR IGNORE INTO disk.records SELECT * FROM records - WHERE confidence >= ${STATE.detect.confidence} AND speciesID NOT IN (${blocked})`); + WHERE confidence >= ${STATE.detect.confidence} ${filterClause} `); console.log(response?.changes + ' records added to disk database'); await memoryDB.runAsync('END'); console.log("transaction ended"); @@ -2643,7 +2702,7 @@ const prepSummaryStatement = (blocked) => { /** * getDetectedSpecies generates a list of species to use in dropdowns for chart and explore mode filters * It doesn't really make sense to use location specific filtering here, as there is a location filter in the - * page. For now, I'm just going skip the blocked IDs filter if location mode is selected + * page. For now, I'm just going skip the included IDs filter if location mode is selected */ const getDetectedSpecies = () => { const range = STATE.explore.range; @@ -2654,8 +2713,8 @@ const prepSummaryStatement = (blocked) => { JOIN files on records.fileID = files.id`; if (STATE.mode === 'explore') sql += ` WHERE confidence >= ${confidence}`; - if (STATE.list !== 'location' && STATE.blocked.length) { - sql += ` AND speciesID NOT IN (${STATE.blocked.join(',')})`; + if (STATE.list !== 'location' && filtersApplied()) { + sql += ` AND speciesID IN (${STATE.included.join(',')})`; } if (range?.start) sql += ` AND datetime BETWEEN ${range.start} AND ${range.end}`; sql += filterLocation(); @@ -2671,20 +2730,22 @@ const prepSummaryStatement = (blocked) => { * @returns Promise */ const getValidSpecies = async (file) => { - const blocked = getBlockedIDs(file); - let excluded, included; + const included = await getIncludedIDs(file); + let excludedSpecies, includedSpecies; let sql = `SELECT cname, sname FROM species`; - if (blocked.length) { - sql += ` WHERE id NOT IN (${blocked.join(',')})`; + // We'll ignore Unknown Sp. here, hence length < (LABELS.length *-1*) + + if (filtersApplied()) { + sql += ` WHERE id IN (${included.join(',')})`; } sql += ' GROUP BY cname ORDER BY cname'; - included = await diskDB.allAsync(sql) + includedSpecies = await diskDB.allAsync(sql) - if (blocked.length){ - sql = sql.replace('NOT IN', 'IN'); - excluded = await diskDB.allAsync(sql); + if (filtersApplied()){ + sql = sql.replace('IN', 'NOT IN'); + excludedSpecies = await diskDB.allAsync(sql); } - UI.postMessage({ event: 'valid-species-list', included: included, excluded: excluded }) + UI.postMessage({ event: 'valid-species-list', included: includedSpecies, excluded: excludedSpecies }) }; const onUpdateFileStart = async (args) => { @@ -2967,8 +3028,13 @@ const prepSummaryStatement = (blocked) => { UI.postMessage({ event: 'location-list', locations: locations, currentLocation: metadata[file]?.locationID }) } - function getBlockedIDs(file){ - let blocked, lat, lon, week; + /** + * Helper function to provide a list of valid species for the filter. Will look in the cache, or call setIncludedIDs to generate a new list + * @param {*} file + * @returns a list of IDs included in filtered results + */ + async function getIncludedIDs(file){ + let included, lat, lon, week; if (STATE.list === 'location'){ if (file){ file = metadata[file]; @@ -2982,45 +3048,32 @@ const prepSummaryStatement = (blocked) => { week = STATE.useWeek ? STATE.week : "-1"; } const location = lat.toString() + lon.toString(); - try { - blocked = STATE.blocked[week][location]; - } catch (error) { - DEBUG && console.log('error creating blocked species list:', error) - //blocked = STATE.blocked["-1"][location]; - setBlockedIDs(lat,lon,week) + if (Array.isArray(STATE.included) || STATE.included[week]?.[location] === undefined ) { + included = await setIncludedIDs(lat,lon,week) + } else { + included = STATE.included.week.location; } - } else { - blocked = STATE.blocked; + included = STATE.included; } - return blocked; + return included; } - async function setBlockedIDs(lat,lon,week){ - // Look for an idle worker, or push to the first worker queue - const readyWorker = await waitForWorker(predictWorkers); - predictWorkers[readyWorker].postMessage({ - message: "list", - list: STATE.list, - lat: lat, - lon: lon, - week: week, - threshold: STATE.speciesThreshold, - worker: readyWorker - }); + async function setIncludedIDs(lat,lon,week){ + // Use the list worker + t0 = Date.now(); + const message = await LIST_WORKER({ + message: 'get-list', + model: STATE.model, + listType: STATE.list, + lat: lat, + lon: lon, + week: week, + useWeek: STATE.useWeek, + threshold: STATE.speciesThreshold + }) + console.log(`setting the ${STATE.list} list took ${Date.now() -t0}ms`) + STATE.included = message.included; + UI.postMessage({ event: "results-complete" }); + return STATE.included } - - function waitForWorker(predictWorkers) { - let count = 0 - return new Promise(resolve => { - const checkAvailability = () => { - const readyWorker = predictWorkers.findIndex(obj => obj.isAvailable && obj.isReady) ; - if (readyWorker > -1) { - resolve(readyWorker); - } else { - setTimeout(checkAvailability, 100); // Check again in 100 milliseconds - } - }; - checkAvailability(); - }); - } \ No newline at end of file