From 8a2571c2443f8a8487399a3bb66c3caccc6ef4c9 Mon Sep 17 00:00:00 2001 From: Mattk70 Date: Sun, 19 Nov 2023 12:31:02 +0000 Subject: [PATCH] swapped to && pattern for simple ifs reversed order of getRestuls and getSummary when you delete a species, it filters on the next species in the summary Huge speed improvements filtering records - need to sort pagination - added a 500ms delay before spinner shows. - hide spinner when all results updated --- js/ui.js | 80 ++++++++++++++++++++++------------------------ js/worker.js | 89 +++++++++++++++++++++++----------------------------- 2 files changed, 78 insertions(+), 91 deletions(-) diff --git a/js/ui.js b/js/ui.js index edb301b9..b6c021b8 100644 --- a/js/ui.js +++ b/js/ui.js @@ -64,12 +64,14 @@ async function getPaths() { } // Function to show the loading spinner -function showLoadingSpinner() { - document.getElementById('loadingOverlay').classList.remove('d-none'); +function showLoadingSpinner(durationThreshold) { + window.loadingTimer = setTimeout(function () { + document.getElementById('loadingOverlay').classList.remove('d-none'); + }, durationThreshold); } - // Function to hide the loading spinner function hideLoadingSpinner() { + clearTimeout(window.loadingTimer); document.getElementById('loadingOverlay').classList.add('d-none'); } let version; @@ -799,11 +801,6 @@ const getSelectionResults = (fromDB) => { let start = region.start + bufferBegin; // Remove small amount of region to avoid pulling in results from 'end' let end = region.end + bufferBegin - 0.001; - // const mod = end - start % 3; - // if (end - start < 1.5 || mod < 1.5) { - // region.end += 1.5; - // end += 1.5; - // } STATE.selection = {}; STATE['selection']['start'] = start.toFixed(3); STATE['selection']['end'] = end.toFixed(3); @@ -817,15 +814,6 @@ const getSelectionResults = (fromDB) => { }); } -// const navbarAnalysis = document.getElementById('navbarAnalysis'); -// navbarAnalysis.addEventListener('click', async () => { -// // Switch to Analyse mode -// if (STATE.mode !== 'analyse'){ -// worker.postMessage({ action: 'change-mode', mode: 'analyse' }); -// worker.postMessage({ action: 'filter' }); -// } -// }); - const analyseLink = document.getElementById('analyse'); analyseLink.addEventListener('click', async () => { postAnalyseMessage({ filesInScope: [currentFile] }); @@ -1015,7 +1003,7 @@ const handleLocationFilterChange = (e) => { worker.postMessage({ action: 'update-state', locationID: location }); // Update the seen species list worker.postMessage({ action: 'get-detected-species-list' }) - if (STATE.mode === 'explore') worker.postMessage({ action: 'filter', species: isSpeciesViewFiltered(true), explore: true }); + if (STATE.mode === 'explore') worker.postMessage({ action: 'filter', species: isSpeciesViewFiltered(true), updateSummary: true }); } const exploreLink = document.getElementById('explore'); @@ -1029,7 +1017,7 @@ exploreLink.addEventListener('click', async () => { showElement(['exploreWrapper', 'spectrogramWrapper'], false); enableMenuItem(['saveCSV']); adjustSpecDims(true) - worker.postMessage({ action: 'filter', species: undefined, range: STATE.explore.range, explore: true }); // re-prepare + worker.postMessage({ action: 'filter', species: undefined, range: STATE.explore.range, updateSummary: true }); // re-prepare }); const datasetLink = document.getElementById('dataset'); @@ -1498,7 +1486,7 @@ const setUpWorkerMessaging = () => { onOpenFiles(args); break; case 'generate-alert': - if (args.render) { + if (args.updateFilenamePanel) { renderFilenamePanel(); window.electron.unsavedRecords(false); document.getElementById('unsaved-icon').classList.add('d-none'); @@ -1509,11 +1497,12 @@ const setUpWorkerMessaging = () => { if (confirm(message)) deleteFile(args.file) } else { if (args.filter){ - worker.postMessage({ - action: 'filter', - species: isSpeciesViewFiltered(true), - active: getActiveRowID(), - }); // no re-prepare + worker.postMessage({ + action: 'filter', + species: isSpeciesViewFiltered(true), + active: getActiveRowID(), + updateSummary: true + }); // no re-prepare } else { alert(args.message) } @@ -1535,9 +1524,6 @@ const setUpWorkerMessaging = () => { renderFilenamePanel(); console.log('Mode changed to: ' + args.mode); break; - case 'no-detections-remain': - detectionsModal.hide(); - break; case 'prediction-done': onPredictionDone(args); break; @@ -1551,7 +1537,7 @@ const setUpWorkerMessaging = () => { generateBirdList('seenSpecies', args.list); break; case 'show-spinner': - showLoadingSpinner(); + showLoadingSpinner(500); break; case 'spawning': displayWarmUpMessage(); @@ -1666,7 +1652,7 @@ $(document).on('change', '#bird-list-seen', function (e) { } else { action = 'filter'; } - worker.postMessage({ action: action, species: cname, range: STATE[context].range }) // no re-prepare + worker.postMessage({ action: action, species: cname, range: STATE[context].range, updateSummary: true }) // no re-prepare }) @@ -2571,6 +2557,11 @@ function onProgress(args) { } } +function updatePagination(total, offset) { + //Pagination + total > config.limit ? addPagination(total, offset) : pagination.forEach(item => item.classList.add('d-none')); + +} const updateSummary = ({ summary = [], filterSpecies = '' }) => { let total, summaryHTML = ` @@ -2651,6 +2642,7 @@ async function onPredictionDone({ } //Pagination total > config.limit ? addPagination(total, offset) : pagination.forEach(item => item.classList.add('d-none')); + if (action !== 'filter') { if (! isEmptyObject(AUDACITY_LABELS)) { enableMenuItem(['saveLabels', 'saveCSV']); @@ -2712,7 +2704,6 @@ pagination.forEach(item => { const limit = config.limit; const offset = (clicked - 1) * limit; const species = isSpeciesViewFiltered(true); - showLoadingSpinner(); worker.postMessage({ action: 'filter', species: species, @@ -2758,7 +2749,14 @@ function speciesFilter(e) { clearActive(); let species, range; // Am I trying to unfilter? - if (!e.target.closest('tr').classList.contains('text-warning')) { + if (e.target.closest('tr').classList.contains('text-warning')) { + e.target.closest('tr').classList.remove('text-warning'); + } else { + //Clear any highlighted rows + const tableRows = summary.querySelectorAll('tr'); + tableRows.forEach(row => {row.classList.remove('text-warning');}) + // Add a highlight to the current row + e.target.closest('tr').classList.add('text-warning'); // Clicked on unfiltered species species = getSpecies(e.target) } @@ -2773,7 +2771,6 @@ function speciesFilter(e) { }); // no re-prepare seenTheDarkness = false; shownDaylightBanner = false; - showLoadingSpinner(); document.getElementById('results').scrollTop = 0; } @@ -2986,11 +2983,13 @@ const deleteSpecies = (target) => { // Clear the record in the UI const row = target.closest('tr'); const table = document.getElementById('resultSummary') - table.deleteRow(row.rowIndex); + const rowClicked = row.rowIndex; + table.deleteRow(rowClicked); const resultTable = document.getElementById('resultTableBody'); resultTable.innerHTML = ''; // Highlight the next row - if (row.rowIndex) row.rowIndex.click() + const newRow = table.rows[rowClicked] + newRow?.click(); } const getSelectionRange = () => { @@ -3412,6 +3411,7 @@ $(function () { action: 'filter', species: STATE.explore.species, range: STATE.explore.range, + updateSummary: true }); // re-prepare } // Update the seen species list @@ -3437,6 +3437,7 @@ $(function () { worker.postMessage({ action: 'filter', species: isSpeciesViewFiltered(true), + updateSummary: true }); // re-prepare } // Update the seen species list @@ -3554,6 +3555,7 @@ const handleThresholdChange = (e) => { worker.postMessage({ action: 'filter', species: isSpeciesViewFiltered(true), + updateSummary: true }); // no re-prepare } } @@ -3847,13 +3849,7 @@ $('#spectrogramWrapper, #resultTableContainer, #selectionResultTableBody').on('c const recordEntryModalDiv = document.getElementById('record-entry-modal') const recordEntryModal = new bootstrap.Modal(recordEntryModalDiv, { backdrop: 'static' }); -const recordEntryHandler = () => { - worker.postMessage({ - action: 'filter', - species: isSpeciesViewFiltered(true), - active: getActiveRowID(), - }); // no re-prepare -} + const recordEntryForm = document.getElementById('record-entry-form'); let focusBirdList; diff --git a/js/worker.js b/js/worker.js index 46644332..8dbd95e9 100644 --- a/js/worker.js +++ b/js/worker.js @@ -240,7 +240,7 @@ async function handleMessage(e) { break; case 'clear-cache': CACHE_LOCATION = p.join(TEMP, 'chirpity'); - if (!fs.existsSync(CACHE_LOCATION)) fs.mkdirSync(CACHE_LOCATION); + fs.existsSync(CACHE_LOCATION) || fs.mkdirSync(CACHE_LOCATION); await clearCache(CACHE_LOCATION, 0); // belt and braces - in dev mode, ctrl-c in console will prevent cache clear on exit break; case 'convert-dataset': @@ -260,7 +260,7 @@ async function handleMessage(e) { break; case 'file-load-request': index = 0; - if (filesBeingProcessed.length) onAbort(args); + filesBeingProcessed.length && onAbort(args); console.log('Worker received audio ' + args.file); await loadAudioFile(args); metadata[args.file].isSaved ? onChangeMode('archive') : onChangeMode('analyse'); @@ -269,8 +269,10 @@ async function handleMessage(e) { if (STATE.db) { t0 = Date.now() UI.postMessage({event: 'show-spinner'}); - await Promise.all([getSummary(args), getResults(args)]); - if (DEBUG) console.log('Filter took ', (Date.now() - t0) / 1000, 'seconds') + await getResults(args); + args.updateSummary && await getSummary(args); + //await Promise.all([getResults(args), getSummary(args)]); + DEBUG && console.log('Filter took ', (Date.now() - t0) / 1000, 'seconds') } break; @@ -308,7 +310,7 @@ async function handleMessage(e) { memoryDB = null; BACKEND = args.backend; STATE.update({ model: args.model }); - if (predictWorkers.length) terminateWorkers(); + predictWorkers.length && terminateWorkers(); spawnWorkers(args.model, args.list, BATCH_SIZE, args.threads); break; case 'post': @@ -359,7 +361,7 @@ ipcRenderer.on('new-client', (event) => { }) async function onChangeMode(mode) { - if (!memoryDB) await createDB(); + memoryDB || await createDB(); UI.postMessage({ event: 'mode-changed', mode: mode }) STATE.changeMode({ mode: mode, @@ -431,9 +433,9 @@ const getSummaryParams = (species) => { } else if (useRange) params.push(range.start, range.end); extraParams.push(...blocked); - if (STATE.locationID) extraParams.push(STATE.locationID); + STATE.locationID && extraParams.push(STATE.locationID); params.push(...extraParams); - if (species) params.push(species); + species && params.push(species); return params } @@ -443,9 +445,10 @@ const prepSummaryStatement = (species) => { const useRange = range?.start; let summaryStatement = ` WITH ranked_records AS ( - SELECT records.dateTime, records.confidence, cname, sname, COALESCE(callCount, 1) as callCount, speciesID, COALESCE(isDaylight, 0) as isDaylight, + SELECT records.dateTime, records.confidence, files.name, cname, sname, COALESCE(callCount, 1) as callCount, speciesID, COALESCE(isDaylight, 0) as isDaylight, RANK() OVER (PARTITION BY records.dateTime ORDER BY records.confidence DESC) AS rank FROM records + JOIN files ON files.id = records.fileID JOIN species ON species.id = records.speciesID WHERE confidence >= ? `; let extraClause = ''; @@ -494,15 +497,12 @@ const prepSummaryStatement = (species) => { const getResultsParams = (species, confidence, offset, limit, topRankin) => { const params = []; params.push(confidence); - if (species) params.push(species); + species && params.push(species); const blocked = (! species && STATE.blocked.length && !STATE.selection) ? STATE.blocked : []; - if (blocked.length) params.push(...blocked); - if (['analyse', 'archive'].includes(STATE.mode) && !STATE.selection) { - params.push(...STATE.filesToAnalyse); - } - - if (limit !== Infinity) params.push(limit, offset); + blocked.length && params.push(...blocked); + ['analyse', 'archive'].includes(STATE.mode) && !STATE.selection && params.push(...STATE.filesToAnalyse); + limit !== Infinity && params.push(limit, offset); params.push(topRankin); return params } @@ -649,7 +649,7 @@ async function onAnalyse({ } else { onChangeMode('archive'); FILE_QUEUE.forEach(file => UI.postMessage({ event: 'update-audio-duration', value: metadata[file].duration })) - await Promise.all([getSummary(), getResults()]); + await Promise.all([getResults(), getSummary()] ); } return; @@ -657,7 +657,7 @@ async function onAnalyse({ } console.log("FILE_QUEUE has ", FILE_QUEUE.length, 'files', count, 'files ignored') - if (!STATE.selection) onChangeMode('analyse'); + STATE.selection || onChangeMode('analyse'); filesBeingProcessed = [...FILE_QUEUE]; @@ -1190,7 +1190,7 @@ const getPredictBuffers = async ({ }) readStream.on('error', err => { console.log(`readstream error: ${err}, start: ${start}, , end: ${end}, duration: ${metadata[file].duration}`); - if (err.code === 'ENOENT') notifyMissingFile(file); + err.code === 'ENOENT' && notifyMissingFile(file); }) } @@ -1239,7 +1239,7 @@ const fetchAudioBuffer = async ({ }) readStream.on('error', err => { console.log(`readstream error: ${err}, start: ${start}, , end: ${end}, duration: ${metadata[file].duration}`); - if (err.code === 'ENOENT') notifyMissingFile(file); + err.code === 'ENOENT' && notifyMissingFile(file); }) }); } @@ -1301,7 +1301,7 @@ const speciesMatch = (path, sname) => { } const convertSpecsFromExistingSpecs = async (path) => { - if (!path) path = '/mnt/608E21D98E21A88C/Users/simpo/PycharmProjects/Data/New_Dataset'; + path ??= '/mnt/608E21D98E21A88C/Users/simpo/PycharmProjects/Data/New_Dataset'; const file_list = await getFiles([path], true); for (let i = 0; i < file_list.length; i++) { const parts = p.parse(file_list[i]); @@ -1469,7 +1469,7 @@ const bufferToAudio = ({ let bitrate = STATE.audio.bitrate; let quality = parseInt(STATE.audio.quality); let downmix = STATE.audio.downmix; - if (!format) format = STATE.audio.format; + format ??= STATE.audio.format; const bitrateMap = { 24000: '24k', 16000: '16k', 12000: '12k', 8000: '8k', 44100: '44k', 22050: '22k', 11025: '11k' }; if (format === 'mp3') { audioCodec = 'libmp3lame'; @@ -1558,7 +1558,7 @@ const bufferToAudio = ({ // ) // } ffmpgCommand.on('start', function (commandLine) { - if (DEBUG) console.log('FFmpeg command: ' + commandLine); + DEBUG && console.log('FFmpeg command: ' + commandLine); }) ffmpgCommand.on('error', (err) => { console.log('An error occurred: ' + err.message); @@ -1737,7 +1737,7 @@ const onInsertManualRecord = async ({ cname, start, end, comment, count, file, l confidence = confidence || 2000; // Delete an existing record if it exists const result = await db.getAsync(`SELECT id as originalSpeciesID FROM species WHERE cname = ?`, originalCname); - if (result?.originalSpeciesID) await db.runAsync('DELETE FROM records WHERE datetime = ? AND speciesID = ? AND fileID = ?', dateTime, result.originalSpeciesID, fileID) + result?.originalSpeciesID && await db.runAsync('DELETE FROM records WHERE datetime = ? AND speciesID = ? AND fileID = ?', dateTime, result.originalSpeciesID, fileID) const response = await db.runAsync('INSERT OR REPLACE INTO records VALUES ( ?,?,?,?,?,?,?,?,?,?)', dateTime, start, fileID, speciesID, confidence, label, comment, end, parseInt(count), isDaylight); @@ -1752,7 +1752,7 @@ const onInsertManualRecord = async ({ cname, start, end, comment, count, file, l const parsePredictions = async (response) => { let file = response.file, batchInProgress = filesBeingProcessed.length; const latestResult = response.result, db = STATE.db; - if (DEBUG) console.log('worker being used:', response.worker); + DEBUG && console.log('worker being used:', response.worker); let [keysArray, speciesIDBatch, confidenceBatch] = latestResult; @@ -1770,7 +1770,7 @@ const parsePredictions = async (response) => { updateUI = (confidence > STATE.detect.confidence && STATE.blocked.indexOf(speciesID) === -1); //save all results to db, regardless of confidence - if (!STATE.selection) await insertRecord(timestamp, key, speciesID, confidence, file); + STATE.selection || await insertRecord(timestamp, key, speciesID, confidence, file); if (STATE.selection || updateUI) { let end, confidenceRequired; if (STATE.selection) { @@ -1826,7 +1826,7 @@ const parsePredictions = async (response) => { console.log(`Prediction done ${filesBeingProcessed.length} files to go`); console.log('Analysis took ' + (new Date() - predictionStart) / 1000 + ' seconds.'); } - if (!STATE.selection && (!DATASET || STATE.increment() === 0) ) getSummary({ interim: true }); + !STATE.selection && (!DATASET || STATE.increment() === 0) && getSummary({ interim: true }); return response.worker } @@ -1842,7 +1842,7 @@ async function parseMessage(e) { if (!SEEN_LABELS) { SEEN_LABELS = true; await loadDB(appPath); - if (!memoryDB) await createDB(); + memoryDB || await createDB(); } break; case 'model-ready': @@ -1885,7 +1885,7 @@ async function parseMessage(e) { if (response['updateResults'] && STATE.db) { // update-results called after setting migrants list, so DB may not be initialized - await Promise.all([getSummary(), getResults()]); + await Promise.all([getResults(), getSummary()] ); if (['explore', 'chart'].includes(STATE.mode)) { // Update the seen species list getSpecies(); @@ -2024,7 +2024,7 @@ function calculateNighttimeBoundaries(fileStart, fileEnd, latitude, longitude) { activeIntervals.push({ start: Math.max(offset, 0) }); // and the others are a stop trigger } else { - if (!activeIntervals.length) activeIntervals.push({ start: 0 }) + activeIntervals.length || activeIntervals.push({ start: 0 }) activeIntervals[activeIntervals.length - 1].end = Math.min(offset, maxFileOffset); } } @@ -2056,7 +2056,6 @@ const getSummary = async ({ active = undefined, interim = false, action = undefined, - topRankin = STATE.topRankin } = {}) => { const db = STATE.db; prepSummaryStatement(species); @@ -2073,7 +2072,7 @@ const getSummary = async ({ const params = getSummaryParams(species); const summary = await STATE.GET_SUMMARY_SQL.allAsync(...params); - if (DEBUG) console.log("Get Summary took", (Date.now() - t0) / 1000, " seconds"); + DEBUG && console.log("Get Summary took", (Date.now() - t0) / 1000, " seconds"); const event = interim ? 'update-summary' : 'prediction-done'; UI.postMessage({ event: event, @@ -2153,7 +2152,7 @@ const getResults = async ({ const filename = `${r.cname}-${dateString}.${STATE.audio.format}` console.log(`Exporting from ${r.file}, position ${r.position}, into folder ${exportTo}`) saveAudio(r.file, r.position, r.position + 3, filename, metadata, exportTo) - if (i === result.length - 1) UI.postMessage({ event: 'generate-alert', message: `${result.length} files saved` }) + i === result.length - 1 && UI.postMessage({ event: 'generate-alert', message: `${result.length} files saved` }) } } else if (species && context !== 'explore') { @@ -2322,7 +2321,7 @@ const onSave2DiskDB = async (args) => { UI.postMessage({ event: 'generate-alert', message: `Database update complete, ${response.changes} records added to the archive in ${((Date.now() - t0) / 1000)} seconds`, - render: true + updateFilenamePanel: true }) } } @@ -2564,28 +2563,20 @@ async function onDeleteSpecies({ speciesFiltered }) { const db = STATE.db; - const speciesSQL = prepSQL(species); + const params = [species]; let SQL = `DELETE FROM records - WHERE speciesID = (SELECT id FROM species WHERE cname = '${speciesSQL}')`; + WHERE speciesID = (SELECT id FROM species WHERE cname = ?)`; if (STATE.mode === 'analyse') { - const filesSQL = STATE.filesToAnalyse.map(file => `'${prepSQL(file)}'`).join(','); - const rows = await db.allAsync(`SELECT id FROM files WHERE NAME IN (${filesSQL})`); + const rows = await db.allAsync(`SELECT id FROM files WHERE NAME IN (${prepParams(STATE.filesToAnalyse)})`, ...STATE.filesToAnalyse); const ids = rows.map(row => row.id).join(','); SQL += ` AND fileID in (${ids})`; } - if (STATE.mode === 'explore') { + else if (STATE.mode === 'explore') { const { start, end } = STATE.explore.range; if (start) SQL += ` AND dateTime BETWEEN ${start} AND ${end}` } - let { changes } = await db.runAsync(SQL); + let { changes } = await db.runAsync(SQL, ...params); if (changes) { - if (STATE.mode !== 'selection') { - // Update the summary table - if (speciesFiltered === false) { - delete arguments[0].species - } - await getSummary(arguments[0]); - } if (db === diskDB) { // Update the seen species list getSpecies(); @@ -2688,10 +2679,10 @@ const onFileDelete = async (fileName) => { event: 'generate-alert', message: `${fileName} and its associated records were deleted successfully`, - render: true + updateFilenamePanel: true }); - await Promise.all([getSummary(), getResults()]); + await Promise.all([getResults(), getSummary()] ); } else { UI.postMessage({ event: 'generate-alert', message: `${fileName}