diff --git a/meteor_packages/mats-common/imports/startup/server/matsMiddle_timeSeries.js b/meteor_packages/mats-common/imports/startup/server/matsMiddle_timeSeries.js index 7c42616c1..096d71164 100644 --- a/meteor_packages/mats-common/imports/startup/server/matsMiddle_timeSeries.js +++ b/meteor_packages/mats-common/imports/startup/server/matsMiddle_timeSeries.js @@ -1,7 +1,8 @@ import { matsMiddleCommon } from "meteor/randyp:mats-common"; import { Meteor } from "meteor/meteor"; -class MatsMiddleTimeSeries { +class MatsMiddleTimeSeries +{ logToFile = false; logMemUsage = false; @@ -40,7 +41,8 @@ class MatsMiddleTimeSeries { mmCommon = null; - constructor(cbPool) { + constructor(cbPool) + { this.cbPool = cbPool; this.mmCommon = new matsMiddleCommon.MatsMiddleCommon(cbPool); } @@ -55,12 +57,14 @@ class MatsMiddleTimeSeries { fromSecs, toSecs, validTimes - ) => { + ) => + { const Future = require("fibers/future"); let rv = []; const dFuture = new Future(); - (async () => { + (async () => + { rv = await this.processStationQuery_int( varName, stationNames, @@ -88,12 +92,12 @@ class MatsMiddleTimeSeries { fromSecs, toSecs, validTimes - ) => { + ) => + { const fs = require("fs"); console.log( - `processStationQuery(${varName},${ - stationNames.length + `processStationQuery(${varName},${stationNames.length },${model},${fcstLen},${threshold},${average},${fromSecs},${toSecs},${JSON.stringify( validTimes )})` @@ -109,9 +113,12 @@ class MatsMiddleTimeSeries { this.toSecs = toSecs; this.average = this.average.replace(/m0./g, ""); - if (validTimes && validTimes.length > 0) { - for (let i = 0; i < validTimes.length; i++) { - if (validTimes[i] != null && Number(validTimes[i]) > 0) { + if (validTimes && validTimes.length > 0) + { + for (let i = 0; i < validTimes.length; i++) + { + if (validTimes[i] != null && Number(validTimes[i]) > 0) + { this.validTimes.push(Number(validTimes[i])); } } @@ -128,8 +135,7 @@ class MatsMiddleTimeSeries { let endTime = new Date().valueOf(); console.log( - `\tfcstValidEpoch_Array:${this.fcstValidEpoch_Array.length} in ${ - endTime - startTime + `\tfcstValidEpoch_Array:${this.fcstValidEpoch_Array.length} in ${endTime - startTime } ms.` ); @@ -138,11 +144,13 @@ class MatsMiddleTimeSeries { await Promise.all([prObs, prModel]); this.generateCtc(threshold); - for (let i = 0; i < this.ctc.length; i++) { + for (let i = 0; i < this.ctc.length; i++) + { this.ctc[i] = this.mmCommon.sumUpCtc(this.ctc[i]); } - if (this.logToFile === true) { + if (this.logToFile === true) + { this.mmCommon.writeToLocalFile( "/scratch/matsMiddle/output/fveObs.json", JSON.stringify(this.fveObs, null, 2) @@ -163,7 +171,8 @@ class MatsMiddleTimeSeries { return this.ctc; }; - createObsData = async () => { + createObsData = async () => + { console.log("createObsData()"); const fs = require("fs"); @@ -175,10 +184,13 @@ class MatsMiddleTimeSeries { ); let stationNames_obs = ""; - for (let i = 0; i < this.stationNames.length; i++) { - if (i === 0) { + for (let i = 0; i < this.stationNames.length; i++) + { + if (i === 0) + { stationNames_obs = `obs.data.${this.stationNames[i]}.${this.varName} ${this.stationNames[i]}`; - } else { + } else + { stationNames_obs += `,obs.data.${this.stationNames[i]}.${this.varName} ${this.stationNames[i]}`; } } @@ -194,21 +206,29 @@ class MatsMiddleTimeSeries { console.log(`\tobs query:${stationNames_obs.length} in ${endTime - startTime} ms.`); const promises = []; - for (let iofve = 0; iofve < this.fcstValidEpoch_Array.length; iofve += 100) { + for (let iofve = 0; iofve < this.fcstValidEpoch_Array.length; iofve += 100) + { const fveArraySlice = this.fcstValidEpoch_Array.slice(iofve, iofve + 100); const sql = tmplWithStationNames_obs.replace( /{{fcstValidEpoch}}/g, JSON.stringify(fveArraySlice) ); + if (this.logToFile === true && iofve === 0) + { + this.mmCommon.writeToLocalFile("/scratch/matsMiddle/output/obs.sql", sql); + } const prSlice = this.conn.cluster.query(sql); promises.push(prSlice); - prSlice.then((qr) => { + prSlice.then((qr) => + { console.log(`qr:\n${qr.rows.length}`); - for (let jmfve = 0; jmfve < qr.rows.length; jmfve++) { + for (let jmfve = 0; jmfve < qr.rows.length; jmfve++) + { const fveDataSingleEpoch = qr.rows[jmfve]; const dataSingleEpoch = {}; const stationsSingleEpoch = {}; - for (let i = 0; i < this.stationNames.length; i++) { + for (let i = 0; i < this.stationNames.length; i++) + { const varValStation = fveDataSingleEpoch[this.stationNames[i]]; stationsSingleEpoch[this.stationNames[i]] = varValStation; } @@ -216,11 +236,11 @@ class MatsMiddleTimeSeries { dataSingleEpoch.stations = stationsSingleEpoch; this.fveObs[fveDataSingleEpoch.fve] = dataSingleEpoch; } - if (iofve % 100 == 0) { + if (iofve % 100 == 0) + { endTime = new Date().valueOf(); console.log( - `iofve:${iofve}/${this.fcstValidEpoch_Array.length} in ${ - endTime - startTime + `iofve:${iofve}/${this.fcstValidEpoch_Array.length} in ${endTime - startTime } ms.` ); } @@ -232,7 +252,8 @@ class MatsMiddleTimeSeries { console.log(`fveObs:` + ` in ${endTime - startTime} ms.`); }; - createModelData = async () => { + createModelData = async () => + { console.log("createModelData()"); const fs = require("fs"); @@ -264,10 +285,13 @@ class MatsMiddleTimeSeries { ); let stationNames_models = ""; - for (let i = 0; i < this.stationNames.length; i++) { - if (i === 0) { + for (let i = 0; i < this.stationNames.length; i++) + { + if (i === 0) + { stationNames_models = `models.data.${this.stationNames[i]}.${this.varName} ${this.stationNames[i]}`; - } else { + } else + { stationNames_models += `,models.data.${this.stationNames[i]}.${this.varName} ${this.stationNames[i]}`; } } @@ -282,21 +306,29 @@ class MatsMiddleTimeSeries { ); const promises = []; - for (let imfve = 0; imfve < this.fcstValidEpoch_Array.length; imfve += 100) { + for (let imfve = 0; imfve < this.fcstValidEpoch_Array.length; imfve += 100) + { const fveArraySlice = this.fcstValidEpoch_Array.slice(imfve, imfve + 100); const sql = tmplWithStationNames_models.replace( /{{fcstValidEpoch}}/g, JSON.stringify(fveArraySlice) ); + if (this.logToFile === true && imfve === 0) + { + this.mmCommon.writeToLocalFile("/scratch/matsMiddle/output/model.sql", sql); + } const prSlice = this.conn.cluster.query(sql); promises.push(prSlice); - prSlice.then((qr) => { - for (let jmfve = 0; jmfve < qr.rows.length; jmfve++) { + prSlice.then((qr) => + { + for (let jmfve = 0; jmfve < qr.rows.length; jmfve++) + { const fveDataSingleEpoch = qr.rows[jmfve]; const dataSingleEpoch = {}; const stationsSingleEpoch = {}; - for (let i = 0; i < this.stationNames.length; i++) { + for (let i = 0; i < this.stationNames.length; i++) + { const varValStation = fveDataSingleEpoch[this.stationNames[i]]; stationsSingleEpoch[this.stationNames[i]] = varValStation; } @@ -304,11 +336,11 @@ class MatsMiddleTimeSeries { dataSingleEpoch.stations = stationsSingleEpoch; this.fveModels[fveDataSingleEpoch.fve] = dataSingleEpoch; } - if (imfve % 100 == 0) { + if (imfve % 100 == 0) + { endTime = new Date().valueOf(); console.log( - `imfve:${imfve}/${this.fcstValidEpoch_Array.length} in ${ - endTime - startTime + `imfve:${imfve}/${this.fcstValidEpoch_Array.length} in ${endTime - startTime } ms.` ); } @@ -319,22 +351,27 @@ class MatsMiddleTimeSeries { console.log(`fveModel:` + ` in ${endTime - startTime} ms.`); }; - generateCtc = async (threshold) => { + generateCtc = async (threshold) => + { console.log(`generateCtc(${threshold})`); const startTime = new Date().valueOf(); - for (let imfve = 0; imfve < this.fcstValidEpoch_Array.length; imfve++) { + for (let imfve = 0; imfve < this.fcstValidEpoch_Array.length; imfve++) + { const fve = this.fcstValidEpoch_Array[imfve]; const obsSingleFve = this.fveObs[fve]; const modelSingleFve = this.fveModels[fve]; - if (!obsSingleFve || !modelSingleFve) { + if (!obsSingleFve || !modelSingleFve) + { continue; } - if (this.validTimes && this.validTimes.length > 0) { - if (this.validTimes.includes((fve % (24 * 3600)) / 3600) == false) { + if (this.validTimes && this.validTimes.length > 0) + { + if (this.validTimes.includes((fve % (24 * 3600)) / 3600) == false) + { continue; } } @@ -349,39 +386,49 @@ class MatsMiddleTimeSeries { stats_fve.N_times = 1; stats_fve.sub_data = []; - for (let i = 0; i < this.stationNames.length; i++) { + for (let i = 0; i < this.stationNames.length; i++) + { const station = this.stationNames[i]; const varVal_o = obsSingleFve.stations[station]; const varVal_m = modelSingleFve.stations[station]; - if (varVal_o && varVal_m) { + if (varVal_o && varVal_m) + { stats_fve.N0 += 1; let sub = `${fve};`; - if (varVal_o < threshold && varVal_m < threshold) { + if (varVal_o < threshold && varVal_m < threshold) + { stats_fve.hit += 1; sub += "1;"; - } else { + } else + { sub += "0;"; } - if (varVal_o >= threshold && varVal_m < threshold) { + if (varVal_o >= threshold && varVal_m < threshold) + { stats_fve.fa += 1; sub += "1;"; - } else { + } else + { sub += "0;"; } - if (varVal_o < threshold && varVal_m >= threshold) { + if (varVal_o < threshold && varVal_m >= threshold) + { stats_fve.miss += 1; sub += "1;"; - } else { + } else + { sub += "0;"; } - if (varVal_o >= threshold && varVal_m >= threshold) { + if (varVal_o >= threshold && varVal_m >= threshold) + { stats_fve.cn += 1; sub += "1"; - } else { + } else + { sub += "0"; } stats_fve.sub_data.push(sub); diff --git a/meteor_packages/mats-common/imports/startup/server/matsMiddle_validTime.js b/meteor_packages/mats-common/imports/startup/server/matsMiddle_validTime.js index fdb9b345b..b655f6c5c 100644 --- a/meteor_packages/mats-common/imports/startup/server/matsMiddle_validTime.js +++ b/meteor_packages/mats-common/imports/startup/server/matsMiddle_validTime.js @@ -41,6 +41,8 @@ class MatsMiddleValidTime mmCommon = null; + ctcTime = 0; + constructor(cbPool) { this.cbPool = cbPool; @@ -154,6 +156,8 @@ class MatsMiddleValidTime } endTime = new Date().valueOf(); + + console.log(`\tctcTime: ${this.ctcTime} ms.`); console.log(`\tprocessStationQuery in ${endTime - startTime} ms.`); return this.ctc; @@ -260,20 +264,25 @@ class MatsMiddleValidTime ); tmpl_get_N_stations_mfve_model = this.cbPool.trfmSQLRemoveClause( tmpl_get_N_stations_mfve_model, - "{{vxFCST_LEN}}" + "{{vxFCST_LEN_ARRAY}}" ); tmpl_get_N_stations_mfve_model = this.cbPool.trfmSQLRemoveClause( tmpl_get_N_stations_mfve_model, - "{{vxFCST_LEN_ARRAY}}" + "{{vxAVERAGE}}" ); tmpl_get_N_stations_mfve_model = this.cbPool.trfmSQLRemoveClause( tmpl_get_N_stations_mfve_model, - "{{vxAVERAGE}}" + "fcstLen fcst_lead" ); + tmpl_get_N_stations_mfve_model = tmpl_get_N_stations_mfve_model.replace( /{{vxMODEL}}/g, `"${this.model}"` ); + tmpl_get_N_stations_mfve_model = tmpl_get_N_stations_mfve_model.replace( + /{{vxFCST_LEN}}/g, + this.fcstLen + ); let stationNames_models = ""; for (let i = 0; i < this.stationNames.length; i++) { @@ -367,7 +376,7 @@ class MatsMiddleValidTime }); } await Promise.all(promises); - this.generateCtc(); + this.ctcTime += this.generateCtc(); endTime = new Date().valueOf(); console.log(`fveModel:` + ` in ${endTime - startTime} ms.`); }; @@ -474,6 +483,7 @@ class MatsMiddleValidTime const endTime = new Date().valueOf(); console.log(`generateCtc:` + ` in ${endTime - startTime} ms.`); + return (endTime - startTime); }; }