From 0d57248e09dd29731c3fe6fe14c4bba985919dcd Mon Sep 17 00:00:00 2001 From: gopa-noaa Date: Wed, 9 Aug 2023 03:55:30 +0000 Subject: [PATCH] valid time --- .../startup/server/matsMiddle_dieoff.js | 192 ++++--- .../startup/server/matsMiddle_validTime.js | 514 ++++++++++++++++++ meteor_packages/mats-common/package.js | 2 + 3 files changed, 639 insertions(+), 69 deletions(-) create mode 100644 meteor_packages/mats-common/imports/startup/server/matsMiddle_validTime.js diff --git a/meteor_packages/mats-common/imports/startup/server/matsMiddle_dieoff.js b/meteor_packages/mats-common/imports/startup/server/matsMiddle_dieoff.js index 79e1fc911..fb5a1df7d 100644 --- a/meteor_packages/mats-common/imports/startup/server/matsMiddle_dieoff.js +++ b/meteor_packages/mats-common/imports/startup/server/matsMiddle_dieoff.js @@ -3,7 +3,8 @@ import { matsMiddleCommon } from "meteor/randyp:mats-common"; import { Meteor } from "meteor/meteor"; import { memoryUsage } from "node:process"; -class MatsMiddleDieoff { +class MatsMiddleDieoff +{ logToFile = false; logMemUsage = false; @@ -46,7 +47,8 @@ class MatsMiddleDieoff { mmCommon = null; - constructor(cbPool) { + constructor(cbPool) + { this.cbPool = cbPool; this.mmCommon = new matsMiddleCommon.MatsMiddleCommon(cbPool); } @@ -62,12 +64,14 @@ class MatsMiddleDieoff { validTimes, utcCycleStart, singleCycle - ) => { + ) => + { const Future = require("fibers/future"); let rv = []; const dFuture = new Future(); - (async () => { + (async () => + { rv = await this.processStationQuery_int( varName, stationNames, @@ -97,10 +101,10 @@ class MatsMiddleDieoff { validTimes, utcCycleStart, singleCycle - ) => { + ) => + { console.log( - `processStationQuery(${varName},${ - stationNames.length + `processStationQuery(${varName},${stationNames.length },${model},${fcstLen},${threshold},${fromSecs},${toSecs},${JSON.stringify( validTimes )})` @@ -115,18 +119,24 @@ class MatsMiddleDieoff { this.toSecs = toSecs; const fs = require("fs"); - if (validTimes && validTimes.length > 0) { - for (let i = 0; i < validTimes.length; i++) { - if (validTimes[i] != null && Number(validTimes[i]) > 0) { + if (validTimes && validTimes.length > 0) + { + for (let i = 0; i < validTimes.length; i++) + { + if (validTimes[i] != null && Number(validTimes[i]) > 0) + { this.validTimes.push(Number(validTimes[i])); } } console.log(`validTimes:${JSON.stringify(this.validTimes)}`); } - if (utcCycleStart && utcCycleStart.length > 0) { - for (let i = 0; i < utcCycleStart.length; i++) { - if (utcCycleStart[i] != null && Number(utcCycleStart[i]) > 0) { + if (utcCycleStart && utcCycleStart.length > 0) + { + for (let i = 0; i < utcCycleStart.length; i++) + { + if (utcCycleStart[i] != null && Number(utcCycleStart[i]) > 0) + { this.utcCycleStart.push(Number(utcCycleStart[i])); } } @@ -146,8 +156,7 @@ class MatsMiddleDieoff { let endTime = new Date().valueOf(); console.log( - `\tfcstValidEpoch_Array:${this.fcstValidEpoch_Array.length} in ${ - endTime - startTime + `\tfcstValidEpoch_Array:${this.fcstValidEpoch_Array.length} in ${endTime - startTime } ms.` ); @@ -160,7 +169,8 @@ class MatsMiddleDieoff { await this.createObsData(); await this.createModelData(); - if (this.logToFile === true) { + if (this.logToFile === true) + { this.mmCommon.writeToLocalFile( "/scratch/matsMiddle/output/fveObs.json", JSON.stringify(this.fveObs, null, 2) @@ -181,7 +191,8 @@ class MatsMiddleDieoff { return this.ctc; }; - createObsData = async () => { + createObsData = async () => + { console.log("createObsData()"); const fs = require("fs"); @@ -193,10 +204,13 @@ class MatsMiddleDieoff { ); let stationNames_obs = ""; - for (let i = 0; i < this.stationNames.length; i++) { - if (i === 0) { + for (let i = 0; i < this.stationNames.length; i++) + { + if (i === 0) + { stationNames_obs = `obs.data.${this.stationNames[i]}.${this.varName} ${this.stationNames[i]}`; - } else { + } else + { stationNames_obs += `,obs.data.${this.stationNames[i]}.${this.varName} ${this.stationNames[i]}`; } } @@ -212,24 +226,29 @@ class MatsMiddleDieoff { console.log(`\tobs query:${stationNames_obs.length} in ${endTime - startTime} ms.`); const promises = []; - for (let iofve = 0; iofve < this.fcstValidEpoch_Array.length; iofve += 100) { + for (let iofve = 0; iofve < this.fcstValidEpoch_Array.length; iofve += 100) + { const fveArraySlice = this.fcstValidEpoch_Array.slice(iofve, iofve + 100); const sql = tmplWithStationNames_obs.replace( /{{fcstValidEpoch}}/g, JSON.stringify(fveArraySlice) ); - if (this.logToFile === true && iofve === 0) { + if (this.logToFile === true && iofve === 0) + { this.mmCommon.writeToLocalFile("/scratch/matsMiddle/output/obs.sql", sql); } const prSlice = this.conn.cluster.query(sql); promises.push(prSlice); - prSlice.then((qr) => { + prSlice.then((qr) => + { console.log(`qr:\n${qr.rows.length}`); - for (let jmfve = 0; jmfve < qr.rows.length; jmfve++) { + for (let jmfve = 0; jmfve < qr.rows.length; jmfve++) + { const fveDataSingleEpoch = qr.rows[jmfve]; const dataSingleEpoch = {}; const stationsSingleEpoch = {}; - for (let i = 0; i < this.stationNames.length; i++) { + for (let i = 0; i < this.stationNames.length; i++) + { const varValStation = fveDataSingleEpoch[this.stationNames[i]]; stationsSingleEpoch[this.stationNames[i]] = varValStation; } @@ -237,11 +256,11 @@ class MatsMiddleDieoff { dataSingleEpoch.stations = stationsSingleEpoch; this.fveObs[fveDataSingleEpoch.fve] = dataSingleEpoch; } - if (iofve % 100 == 0) { + if (iofve % 100 == 0) + { endTime = new Date().valueOf(); console.log( - `iofve:${iofve}/${this.fcstValidEpoch_Array.length} in ${ - endTime - startTime + `iofve:${iofve}/${this.fcstValidEpoch_Array.length} in ${endTime - startTime } ms.` ); } @@ -253,7 +272,8 @@ class MatsMiddleDieoff { console.log(`fveObs:` + ` in ${endTime - startTime} ms.`); }; - createModelData = async () => { + createModelData = async () => + { console.log("createModelData()"); const fs = require("fs"); @@ -276,10 +296,13 @@ class MatsMiddleDieoff { `"${this.model}"` ); let stationNames_models = ""; - for (let i = 0; i < this.stationNames.length; i++) { - if (i === 0) { + for (let i = 0; i < this.stationNames.length; i++) + { + if (i === 0) + { stationNames_models = `models.data.${this.stationNames[i]}.${this.varName} ${this.stationNames[i]}`; - } else { + } else + { stationNames_models += `,models.data.${this.stationNames[i]}.${this.varName} ${this.stationNames[i]}`; } } @@ -294,7 +317,8 @@ class MatsMiddleDieoff { ); const flaIncr = 3; - for (let flai = 0; flai < this.fcstLenArray.length; flai += flaIncr) { + for (let flai = 0; flai < this.fcstLenArray.length; flai += flaIncr) + { this.fveModels = {}; const flaSlice = this.fcstLenArray.slice(flai, flai + flaIncr); const tmplWithStationNames_models_fcst_array = @@ -303,7 +327,8 @@ class MatsMiddleDieoff { JSON.stringify(flaSlice) ); const promises = []; - for (let imfve = 0; imfve < this.fcstValidEpoch_Array.length; imfve += 100) { + for (let imfve = 0; imfve < this.fcstValidEpoch_Array.length; imfve += 100) + { const fveArraySlice = this.fcstValidEpoch_Array.slice(imfve, imfve + 100); const sql = tmplWithStationNames_models_fcst_array.replace( /{{fcstValidEpoch}}/g, @@ -311,28 +336,32 @@ class MatsMiddleDieoff { ); // console.log(sql); console.log( - `flaSlice:${JSON.stringify(flaSlice)},fveArraySlice:${fveArraySlice[0]} => ${ - fveArraySlice[fveArraySlice.length - 1] + `flaSlice:${JSON.stringify(flaSlice)},fveArraySlice:${fveArraySlice[0]} => ${fveArraySlice[fveArraySlice.length - 1] }` ); - if (this.logToFile === true && imfve === 0) { + if (this.logToFile === true && imfve === 0) + { this.mmCommon.writeToLocalFile("/scratch/matsMiddle/output/model.sql", sql); } const prSlice = this.conn.cluster.query(sql); promises.push(prSlice); - prSlice.then((qr) => { + prSlice.then((qr) => + { const idx = imfve / 100; - for (let jmfve = 0; jmfve < qr.rows.length; jmfve++) { + for (let jmfve = 0; jmfve < qr.rows.length; jmfve++) + { const fveDataSingleEpoch = qr.rows[jmfve]; const dataSingleEpoch = {}; const stationsSingleEpoch = {}; - for (let i = 0; i < this.stationNames.length; i++) { + for (let i = 0; i < this.stationNames.length; i++) + { const varValStation = fveDataSingleEpoch[this.stationNames[i]]; stationsSingleEpoch[this.stationNames[i]] = varValStation; } dataSingleEpoch.stations = stationsSingleEpoch; - if (!this.fveModels[fveDataSingleEpoch.fcst_lead]) { + if (!this.fveModels[fveDataSingleEpoch.fcst_lead]) + { this.fveModels[fveDataSingleEpoch.fcst_lead] = {}; } this.fveModels[fveDataSingleEpoch.fcst_lead][fveDataSingleEpoch.fve] = @@ -341,13 +370,14 @@ class MatsMiddleDieoff { endTime = new Date().valueOf(); console.log( - `imfve:${imfve}/${this.fcstValidEpoch_Array.length} idx: ${idx} in ${ - endTime - startTime + `imfve:${imfve}/${this.fcstValidEpoch_Array.length} idx: ${idx} in ${endTime - startTime } ms.` ); - if (this.logMemUsage === true) { - try { + if (this.logMemUsage === true) + { + try + { console.log(memoryUsage()); const obsSize = new TextEncoder().encode(JSON.stringify(this.fveObs)).length / @@ -361,7 +391,8 @@ class MatsMiddleDieoff { console.log( `sizes (MB), obs:${obsSize},model:${modelsSize},ctc:${ctcSize}` ); - } catch (ex) { + } catch (ex) + { console.log(`exception getting sizes:${ex}`); } } @@ -374,7 +405,8 @@ class MatsMiddleDieoff { console.log(`fveModel:` + ` in ${endTime - startTime} ms.`); }; - generateCtc = () => { + generateCtc = () => + { console.log("generateCtc()"); const { threshold } = this; @@ -382,10 +414,12 @@ class MatsMiddleDieoff { const startTime = new Date().valueOf(); const fcst_lead_array = Object.keys(this.fveModels); - fcst_lead_array.sort(function (a, b) { + fcst_lead_array.sort(function (a, b) + { return a - b; }); - for (let flai = 0; flai < fcst_lead_array.length; flai++) { + for (let flai = 0; flai < fcst_lead_array.length; flai++) + { const stats_fcst_lead = {}; const fcst_lead = Number(fcst_lead_array[flai]); @@ -405,83 +439,103 @@ class MatsMiddleDieoff { stats_fcst_lead.min_secs = fve_array[0]; stats_fcst_lead.max_secs = fve_array[fve_array.length - 1]; - for (let imfve = 0; imfve < fve_array.length; imfve++) { + for (let imfve = 0; imfve < fve_array.length; imfve++) + { const fve = fve_array[imfve]; const obsSingleFve = this.fveObs[fve]; const modelSingleFve = fcst_lead_single[fve]; - if (!obsSingleFve || !modelSingleFve) { + if (!obsSingleFve || !modelSingleFve) + { continue; } - if (this.validTimes && this.validTimes.length > 0) { + if (this.validTimes && this.validTimes.length > 0) + { // m0.fcstValidEpoch%(24*3600)/3600 IN[vxVALID_TIMES] - if (this.validTimes.includes((fve % (24 * 3600)) / 3600) == false) { + if (this.validTimes.includes((fve % (24 * 3600)) / 3600) == false) + { continue; } } - if (this.utcCycleStart && this.utcCycleStart.length > 0) { + if (this.utcCycleStart && this.utcCycleStart.length > 0) + { // (obs.fcstValidEpoch - obs.fcstLen*3600)%(24*3600)/3600 IN[vxUTC_CYCLE_START]) if ( this.utcCycleStart.includes( ((fve - fcst_lead * 3600) % (24 * 3600)) / 3600 ) == false - ) { + ) + { continue; } } - if (this.singleCycle !== null) { + if (this.singleCycle !== null) + { // obs.fcstValidEpoch-obs.fcstLen*3600 = vxFROM_SECS - if (fve - fcst_lead * 3600 == this.singleCycle) { + if (fve - fcst_lead * 3600 == this.singleCycle) + { continue; } } - for (let i = 0; i < this.stationNames.length; i++) { + for (let i = 0; i < this.stationNames.length; i++) + { const station = this.stationNames[i]; const varVal_o = obsSingleFve.stations[station]; const varVal_m = modelSingleFve.stations[station]; - if (varVal_o && varVal_m) { + if (varVal_o && varVal_m) + { stats_fcst_lead.N0 += 1; let sub = `${fve};`; - if (varVal_o < threshold && varVal_m < threshold) { + if (varVal_o < threshold && varVal_m < threshold) + { stats_fcst_lead.hit += 1; sub += "1;"; - } else { + } else + { sub += "0;"; } - if (varVal_o >= threshold && varVal_m < threshold) { + if (varVal_o >= threshold && varVal_m < threshold) + { stats_fcst_lead.fa += 1; sub += "1;"; - } else { + } else + { sub += "0;"; } - if (varVal_o < threshold && varVal_m >= threshold) { + if (varVal_o < threshold && varVal_m >= threshold) + { stats_fcst_lead.miss += 1; sub += "1;"; - } else { + } else + { sub += "0;"; } - if (varVal_o >= threshold && varVal_m >= threshold) { + if (varVal_o >= threshold && varVal_m >= threshold) + { stats_fcst_lead.cn += 1; sub += "1"; - } else { + } else + { sub += "0"; } stats_fcst_lead.sub_data.push(sub); } } } - try { + try + { const stats_fcst_lead_summed = this.mmCommon.sumUpCtc(stats_fcst_lead); this.ctc.push(stats_fcst_lead_summed); - } catch (ex) { + } catch (ex) + { console.log(ex); } } diff --git a/meteor_packages/mats-common/imports/startup/server/matsMiddle_validTime.js b/meteor_packages/mats-common/imports/startup/server/matsMiddle_validTime.js new file mode 100644 index 000000000..b27b50872 --- /dev/null +++ b/meteor_packages/mats-common/imports/startup/server/matsMiddle_validTime.js @@ -0,0 +1,514 @@ +import { matsMiddleCommon } from "meteor/randyp:mats-common"; + +import { Meteor } from "meteor/meteor"; +import { memoryUsage } from "node:process"; + +class MatsMiddleValidTime +{ + logToFile = false; + + logMemUsage = false; + + hrOfDay_Array = []; + + cbPool = null; + + conn = null; + + fcstLenArray = []; + + fveObs = {}; + + fveModels = {}; + + ctc = []; + + varName = null; + + stationNames = null; + + model = null; + + fcstLen = null; + + threshold = null; + + fromSecs = null; + + toSecs = null; + + writeOutput = false; + + mmCommon = null; + + constructor(cbPool) + { + this.cbPool = cbPool; + this.mmCommon = new matsMiddleCommon.MatsMiddleCommon(cbPool); + } + + processStationQuery = ( + varName, + stationNames, + model, + fcstLen, + threshold, + fromSecs, + toSecs + ) => + { + const Future = require("fibers/future"); + + let rv = []; + const dFuture = new Future(); + (async () => + { + rv = await this.processStationQuery_int( + varName, + stationNames, + model, + fcstLen, + threshold, + fromSecs, + toSecs + ); + dFuture.return(); + })(); + dFuture.wait(); + return rv; + }; + + processStationQuery_int = async ( + varName, + stationNames, + model, + fcstLen, + threshold, + fromSecs, + toSecs + ) => + { + console.log( + `processStationQuery(${varName},${stationNames.length + },${model},${fcstLen},${threshold},${fromSecs},${toSecs} + )})` + ); + + this.varName = varName; + this.stationNames = stationNames; + this.model = model; + this.fcstLen = fcstLen; + this.threshold = threshold; + this.fromSecs = fromSecs; + this.toSecs = toSecs; + const fs = require("fs"); + + this.conn = await cbPool.getConnection(); + + const startTime = new Date().valueOf(); + + this.fcstValidEpoch_Array = await this.mmCommon.get_fcstValidEpoch_Array( + fromSecs, + toSecs + ); + + // create distinct hour of day array + hrOfDay_prev = null; + for (let iofve = 0; iofve < this.fcstValidEpoch_Array.length; iofve += 100) + { + if (!hrOfDay_prev || hrOfDay_prev !== this.fcstValidEpoch_Array[iofve]) + { + this.hrOfDay_Array.push(this.fcstValidEpoch_Array[iofve]); + hrOfDay_prev = this.fcstValidEpoch_Array[iofve]; + } + } + if (this.logToFile === true) + { + console.log("fcstValidEpoch_Array:\n" + JSON.stringify(this.fcstValidEpoch_Array)); + } + + let endTime = new Date().valueOf(); + console.log( + `\tfcstValidEpoch_Array:${this.fcstValidEpoch_Array.length} in ${endTime - startTime + } ms.` + ); + + await this.createObsData(); + await this.createModelData(); + + if (this.logToFile === true) + { + this.mmCommon.writeToLocalFile( + "/scratch/matsMiddle/output/fveObs.json", + JSON.stringify(this.fveObs, null, 2) + ); + this.mmCommon.writeToLocalFile( + "/scratch/matsMiddle/output/fveModels.json", + JSON.stringify(this.fveModels, null, 2) + ); + this.mmCommon.writeToLocalFile( + "/scratch/matsMiddle/output/ctc.json", + JSON.stringify(this.ctc, null, 2) + ); + } + + endTime = new Date().valueOf(); + console.log(`\tprocessStationQuery in ${endTime - startTime} ms.`); + + return this.ctc; + }; + + createObsData = async () => + { + console.log("createObsData()"); + const fs = require("fs"); + + const startTime = new Date().valueOf(); + + const tmpl_get_N_stations_mfve_obs = fs.readFileSync( + "assets/app/matsMiddle/sqlTemplates/tmpl_get_N_stations_mfve_IN_obs.sql", + "utf-8" + ); + + let stationNames_obs = ""; + for (let i = 0; i < this.stationNames.length; i++) + { + if (i === 0) + { + stationNames_obs = `obs.data.${this.stationNames[i]}.${this.varName} ${this.stationNames[i]}`; + } else + { + stationNames_obs += `,obs.data.${this.stationNames[i]}.${this.varName} ${this.stationNames[i]}`; + } + } + let tmplWithStationNames_obs = cbPool.trfmSQLRemoveClause( + tmpl_get_N_stations_mfve_obs, + "{{vxAVERAGE}}" + ); + tmplWithStationNames_obs = tmplWithStationNames_obs.replace( + /{{stationNamesList}}/g, + stationNames_obs + ); + let endTime = new Date().valueOf(); + console.log(`\tobs query:${stationNames_obs.length} in ${endTime - startTime} ms.`); + + const promises = []; + for (let iofve = 0; iofve < this.fcstValidEpoch_Array.length; iofve += 100) + { + const fveArraySlice = this.fcstValidEpoch_Array.slice(iofve, iofve + 100); + const sql = tmplWithStationNames_obs.replace( + /{{fcstValidEpoch}}/g, + JSON.stringify(fveArraySlice) + ); + if (this.logToFile === true && iofve === 0) + { + this.mmCommon.writeToLocalFile("/scratch/matsMiddle/output/obs.sql", sql); + } + const prSlice = this.conn.cluster.query(sql); + promises.push(prSlice); + prSlice.then((qr) => + { + console.log(`qr:\n${qr.rows.length}`); + for (let jmfve = 0; jmfve < qr.rows.length; jmfve++) + { + + const fveDataSingleEpoch = qr.rows[jmfve]; + let hod = fveDataSingleEpoch.fcst % (24 * 3600) / 3600; + if (!this.fveObs.hod) + { + this.fveObs[hod] = {}; + } + const dataSingleEpoch = {}; + const stationsSingleEpoch = {}; + for (let i = 0; i < this.stationNames.length; i++) + { + const varValStation = fveDataSingleEpoch[this.stationNames[i]]; + stationsSingleEpoch[this.stationNames[i]] = varValStation; + } + dataSingleEpoch.fcst = fveDataSingleEpoch.fcst; + dataSingleEpoch.stations = stationsSingleEpoch; + this.fveObs[hod][fveDataSingleEpoch.fve] = dataSingleEpoch; + } + if (iofve % 100 == 0) + { + endTime = new Date().valueOf(); + console.log( + `iofve:${iofve}/${this.fcstValidEpoch_Array.length} in ${endTime - startTime + } ms.` + ); + } + }); + } + + await Promise.all(promises); + endTime = new Date().valueOf(); + console.log(`fveObs:` + ` in ${endTime - startTime} ms.`); + }; + + createModelData = async () => + { + console.log("createModelData()"); + const fs = require("fs"); + + const startTime = new Date().valueOf(); + + let tmpl_get_N_stations_mfve_model = fs.readFileSync( + "assets/app/matsMiddle/sqlTemplates/tmpl_get_N_stations_mfve_IN_model.sql", + "utf-8" + ); + tmpl_get_N_stations_mfve_model = this.cbPool.trfmSQLRemoveClause( + tmpl_get_N_stations_mfve_model, + "{{vxFCST_LEN}}" + ); + tmpl_get_N_stations_mfve_model = this.cbPool.trfmSQLRemoveClause( + tmpl_get_N_stations_mfve_model, + "{{vxFCST_LEN_ARRAY}}" + ); + tmpl_get_N_stations_mfve_model = this.cbPool.trfmSQLRemoveClause( + tmpl_get_N_stations_mfve_model, + "{{vxAVERAGE}}" + ); + tmpl_get_N_stations_mfve_model = tmpl_get_N_stations_mfve_model.replace( + /{{vxMODEL}}/g, + `"${this.model}"` + ); + let stationNames_models = ""; + for (let i = 0; i < this.stationNames.length; i++) + { + if (i === 0) + { + stationNames_models = `models.data.${this.stationNames[i]}.${this.varName} ${this.stationNames[i]}`; + } else + { + stationNames_models += `,models.data.${this.stationNames[i]}.${this.varName} ${this.stationNames[i]}`; + } + } + + const tmplWithStationNames_models = tmpl_get_N_stations_mfve_model.replace( + /{{stationNamesList}}/g, + stationNames_models + ); + let endTime = new Date().valueOf(); + console.log( + `\tmodel query:${stationNames_models.length} in ${endTime - startTime} ms.` + ); + + const flaIncr = 3; + this.fveModels = {}; + const promises = []; + for (let imfve = 0; imfve < this.fcstValidEpoch_Array.length; imfve += 100) + { + const fveArraySlice = this.fcstValidEpoch_Array.slice(imfve, imfve + 100); + const sql = tmplWithStationNames_models.replace( + /{{fcstValidEpoch}}/g, + JSON.stringify(fveArraySlice) + ); + if (this.logToFile === true && imfve === 0) + { + this.mmCommon.writeToLocalFile("/scratch/matsMiddle/output/model.sql", sql); + } + const prSlice = this.conn.cluster.query(sql); + + promises.push(prSlice); + prSlice.then((qr) => + { + const idx = imfve / 100; + for (let jmfve = 0; jmfve < qr.rows.length; jmfve++) + { + const fveDataSingleEpoch = qr.rows[jmfve]; + let hod = fveDataSingleEpoch.fcst % (24 * 3600) / 3600; + if (!this.fveModels.hod) + { + this.fveModels[hod] = {}; + } + const dataSingleEpoch = {}; + const stationsSingleEpoch = {}; + for (let i = 0; i < this.stationNames.length; i++) + { + const varValStation = fveDataSingleEpoch[this.stationNames[i]]; + stationsSingleEpoch[this.stationNames[i]] = varValStation; + } + dataSingleEpoch.stations = stationsSingleEpoch; + this.fveModels[hod][fveDataSingleEpoch.fve] = + dataSingleEpoch; + } + + endTime = new Date().valueOf(); + console.log( + `imfve:${imfve}/${this.fcstValidEpoch_Array.length} idx: ${idx} in ${endTime - startTime + } ms.` + ); + + if (this.logMemUsage === true) + { + try + { + console.log(memoryUsage()); + const obsSize = + new TextEncoder().encode(JSON.stringify(this.fveObs)).length / + (1024 * 1024); + const modelsSize = + new TextEncoder().encode(JSON.stringify(this.fveModels)).length / + (1024 * 1024); + const ctcSize = + new TextEncoder().encode(JSON.stringify(this.ctc)).length / + (1024 * 1024); + console.log( + `sizes (MB), obs:${obsSize},model:${modelsSize},ctc:${ctcSize}` + ); + } catch (ex) + { + console.log(`exception getting sizes:${ex}`); + } + } + }); + } + await Promise.all(promises); + this.generateCtc(); + endTime = new Date().valueOf(); + console.log(`fveModel:` + ` in ${endTime - startTime} ms.`); + }; + + generateCtc = () => + { + console.log("generateCtc()"); + + const { threshold } = this; + + const startTime = new Date().valueOf(); + + const fcst_lead_array = Object.keys(this.fveModels); + fcst_lead_array.sort(function (a, b) + { + return a - b; + }); + for (let flai = 0; flai < fcst_lead_array.length; flai++) + { + const stats_fcst_lead = {}; + + const fcst_lead = Number(fcst_lead_array[flai]); + stats_fcst_lead.fcst_lead = fcst_lead; + stats_fcst_lead.hit = 0; + stats_fcst_lead.miss = 0; + stats_fcst_lead.fa = 0; + stats_fcst_lead.cn = 0; + stats_fcst_lead.N0 = 0; + stats_fcst_lead.N_times = new Set(fcst_lead_array).size; + stats_fcst_lead.sub_data = []; + + // get all the fve for this fcst_lead + const fcst_lead_single = this.fveModels[fcst_lead_array[flai]]; + const fve_array = Object.keys(fcst_lead_single); + fve_array.sort(); + + stats_fcst_lead.min_secs = fve_array[0]; + stats_fcst_lead.max_secs = fve_array[fve_array.length - 1]; + for (let imfve = 0; imfve < fve_array.length; imfve++) + { + const fve = fve_array[imfve]; + const obsSingleFve = this.fveObs[fve]; + const modelSingleFve = fcst_lead_single[fve]; + + if (!obsSingleFve || !modelSingleFve) + { + continue; + } + + if (this.validTimes && this.validTimes.length > 0) + { + // m0.fcstValidEpoch%(24*3600)/3600 IN[vxVALID_TIMES] + if (this.validTimes.includes((fve % (24 * 3600)) / 3600) == false) + { + continue; + } + } + + if (this.utcCycleStart && this.utcCycleStart.length > 0) + { + // (obs.fcstValidEpoch - obs.fcstLen*3600)%(24*3600)/3600 IN[vxUTC_CYCLE_START]) + if ( + this.utcCycleStart.includes( + ((fve - fcst_lead * 3600) % (24 * 3600)) / 3600 + ) == false + ) + { + continue; + } + } + + if (this.singleCycle !== null) + { + // obs.fcstValidEpoch-obs.fcstLen*3600 = vxFROM_SECS + if (fve - fcst_lead * 3600 == this.singleCycle) + { + continue; + } + } + + for (let i = 0; i < this.stationNames.length; i++) + { + const station = this.stationNames[i]; + const varVal_o = obsSingleFve.stations[station]; + const varVal_m = modelSingleFve.stations[station]; + + if (varVal_o && varVal_m) + { + stats_fcst_lead.N0 += 1; + let sub = `${fve};`; + if (varVal_o < threshold && varVal_m < threshold) + { + stats_fcst_lead.hit += 1; + sub += "1;"; + } else + { + sub += "0;"; + } + + if (varVal_o >= threshold && varVal_m < threshold) + { + stats_fcst_lead.fa += 1; + sub += "1;"; + } else + { + sub += "0;"; + } + + if (varVal_o < threshold && varVal_m >= threshold) + { + stats_fcst_lead.miss += 1; + sub += "1;"; + } else + { + sub += "0;"; + } + + if (varVal_o >= threshold && varVal_m >= threshold) + { + stats_fcst_lead.cn += 1; + sub += "1"; + } else + { + sub += "0"; + } + stats_fcst_lead.sub_data.push(sub); + } + } + } + try + { + const stats_fcst_lead_summed = this.mmCommon.sumUpCtc(stats_fcst_lead); + this.ctc.push(stats_fcst_lead_summed); + } catch (ex) + { + console.log(ex); + } + } + + const endTime = new Date().valueOf(); + console.log(`generateCtc:` + ` in ${endTime - startTime} ms.`); + }; +} + +export default matsMiddleValidTime = { + MatsMiddleValidTime, +}; diff --git a/meteor_packages/mats-common/package.js b/meteor_packages/mats-common/package.js index 3e16e39b5..de2cd3825 100644 --- a/meteor_packages/mats-common/package.js +++ b/meteor_packages/mats-common/package.js @@ -92,6 +92,7 @@ Package.onUse(function (api) { api.export("matsMiddleCommon", ["server"]); api.export("matsMiddleDieoff", ["server"]); api.export("matsMiddleTimeSeries", ["server"]); + api.export("matsMiddleValidTime", ["server"]); // add imports // both @@ -132,6 +133,7 @@ Package.onUse(function (api) { api.addFiles("imports/startup/server/matsMiddle_common.js"); api.addFiles("imports/startup/server/matsMiddle_timeSeries.js"); api.addFiles("imports/startup/server/matsMiddle_dieoff.js"); + api.addFiles("imports/startup/server/matsMiddle_validTime.js"); // files outside of imports // client