From afe7ffff3dede5f14aafe23d06e14a6c25e7d598 Mon Sep 17 00:00:00 2001 From: gopa-noaa Date: Tue, 29 Aug 2023 15:22:55 +0000 Subject: [PATCH] Map - all stations for 1 month in 2 minutes --- .../imports/startup/server/data_query_util.js | 124 ++-- .../server/matsMiddle_dailyModelCycle.js | 693 ++++++++---------- ...dailyModelCycle_0.js => matsMiddle_map.js} | 304 ++++---- meteor_packages/mats-common/package.js | 2 + 4 files changed, 521 insertions(+), 602 deletions(-) rename meteor_packages/mats-common/imports/startup/server/{matsMiddle_dailyModelCycle_0.js => matsMiddle_map.js} (55%) diff --git a/meteor_packages/mats-common/imports/startup/server/data_query_util.js b/meteor_packages/mats-common/imports/startup/server/data_query_util.js index fc2acb162..07f2dc8c4 100644 --- a/meteor_packages/mats-common/imports/startup/server/data_query_util.js +++ b/meteor_packages/mats-common/imports/startup/server/data_query_util.js @@ -339,15 +339,6 @@ const queryDBTimeSeries = function ( } else { (async () => { rows = await pool.queryCB(statementOrMwRows); - // TODO - remove log to file - { - const fs = require("fs"); - const homedir = require("os").homedir(); - fs.writeFileSync( - `${homedir}/scratch/matsMiddle/output/rowsOrg_dmc.json`, - JSON.stringify(rows, null, 2) - ); - } // done waiting - have results dFuture.return(); })(); @@ -472,19 +463,6 @@ const queryDBSpecialtyCurve = function ( } else { (async () => { rows = await pool.queryCB(statementOrMwRows); - // TODO - remove log to file - { - const fs = require("fs"); - const homedir = require("os").homedir(); - fs.writeFileSync( - `${homedir}/scratch/matsMiddle/output/statementOrMwRows.sql`, - statementOrMwRows - ); - fs.writeFileSync( - `${homedir}/scratch/matsMiddle/output/rowsOrg_dmc.json`, - JSON.stringify(rows, null, 2) - ); - } dFuture.return(); })(); } @@ -963,7 +941,7 @@ const queryDBMapScalar = function ( // this method queries the database for map plots in CTC apps const queryDBMapCTC = function ( pool, - statement, + statementOrMwRows, dataSource, statistic, siteMap, @@ -1085,56 +1063,64 @@ const queryDBMapCTC = function ( let error = ""; let parsedData; const Future = require("fibers/future"); - const pFuture = new Future(); + const dFuture = new Future(); + if (matsCollections.Settings.findOne().dbType === matsTypes.DbTypes.couchbase) { /* we have to call the couchbase utilities as async functions but this - routine 'queryDBMapCTC' cannot itself be async because the graph page needs to wait + routine 'queryDBSpecialtyCurve' cannot itself be async because the graph page needs to wait for its result, so we use an anonymous async() function here to wrap the queryCB call - */ - (async () => { - const rows = await pool.queryCB(statement); - if (rows === undefined || rows === null || rows.length === 0) { - error = matsTypes.Messages.NO_DATA_FOUND; - } else if (rows.includes("queryCB ERROR: ")) { - error = rows; - } else { - parsedData = parseQueryDataMapCTC( - rows, - d, - dPurple, - dPurpleBlue, - dBlue, - dBlueGreen, - dGreen, - dGreenYellow, - dYellow, - dOrange, - dOrangeRed, - dRed, - dataSource, - siteMap, - statistic, - appParams - ); - d = parsedData.d; - dPurple = parsedData.dPurple; - dPurpleBlue = parsedData.dPurpleBlue; - dBlue = parsedData.dBlue; - dBlueGreen = parsedData.dBlueGreen; - dGreen = parsedData.dGreen; - dGreenYellow = parsedData.dGreenYellow; - dYellow = parsedData.dYellow; - dOrange = parsedData.dOrange; - dOrangeRed = parsedData.dOrangeRed; - dRed = parsedData.dRed; - valueLimits = parsedData.valueLimits; - } - pFuture.return(); - })(); + */ + let rows = null; + if (Array.isArray(statementOrMwRows)) { + rows = statementOrMwRows; + dFuture.return(); + } else { + (async () => { + rows = await pool.queryCB(statementOrMwRows); + dFuture.return(); + })(); + } + dFuture.wait(); + if (rows === undefined || rows === null || rows.length === 0) { + error = matsTypes.Messages.NO_DATA_FOUND; + } else if (rows.includes("queryCB ERROR: ")) { + error = rows; + } else { + parsedData = parseQueryDataMapCTC( + rows, + d, + dPurple, + dPurpleBlue, + dBlue, + dBlueGreen, + dGreen, + dGreenYellow, + dYellow, + dOrange, + dOrangeRed, + dRed, + dataSource, + siteMap, + statistic, + appParams + ); + d = parsedData.d; + dPurple = parsedData.dPurple; + dPurpleBlue = parsedData.dPurpleBlue; + dBlue = parsedData.dBlue; + dBlueGreen = parsedData.dBlueGreen; + dGreen = parsedData.dGreen; + dGreenYellow = parsedData.dGreenYellow; + dYellow = parsedData.dYellow; + dOrange = parsedData.dOrange; + dOrangeRed = parsedData.dOrangeRed; + dRed = parsedData.dRed; + valueLimits = parsedData.valueLimits; + } } else { // if this app isn't couchbase, use mysql - pool.query(statement, function (err, rows) { + pool.query(statementOrMwRows, function (err, rows) { // query callback - build the curve data from the results - or set an error if (err !== undefined && err !== null) { error = err.message; @@ -1173,12 +1159,12 @@ const queryDBMapCTC = function ( valueLimits = parsedData.valueLimits; } // done waiting - have results - pFuture.return(); + dFuture.return(); }); } // wait for future to finish - pFuture.wait(); + dFuture.wait(); return { data: d, dataPurple: dPurple, diff --git a/meteor_packages/mats-common/imports/startup/server/matsMiddle_dailyModelCycle.js b/meteor_packages/mats-common/imports/startup/server/matsMiddle_dailyModelCycle.js index e886ed1e2..7e883181c 100644 --- a/meteor_packages/mats-common/imports/startup/server/matsMiddle_dailyModelCycle.js +++ b/meteor_packages/mats-common/imports/startup/server/matsMiddle_dailyModelCycle.js @@ -1,82 +1,63 @@ import { matsMiddleCommon } from "meteor/randyp:mats-common"; import { Meteor } from "meteor/meteor"; -class MatsMiddleDailyModelCycle -{ - logToFile = false; +class MatsMiddleDailyModelCycle { + logToFile = false; - logMemUsage = false; + logMemUsage = false; - fcstValidEpoch_Array = []; + fcstValidEpoch_Array = []; - cbPool = null; + cbPool = null; - conn = null; + conn = null; - fveObs = {}; + fveObs = {}; - fveModels = {}; + fveModels = {}; - ctc = []; + ctc = []; - varName = null; + varName = null; - stationNames = null; + stationNames = null; - model = null; + model = null; - threshold = null; + threshold = null; - average = null; + average = null; - fromSecs = null; + fromSecs = null; - toSecs = null; + toSecs = null; - utcCycleStart = null; + utcCycleStart = null; - writeOutput = false; + writeOutput = false; - mmCommon = null; + mmCommon = null; - constructor(cbPool) - { - this.cbPool = cbPool; - this.mmCommon = new matsMiddleCommon.MatsMiddleCommon(cbPool); - } + constructor(cbPool) { + this.cbPool = cbPool; + this.mmCommon = new matsMiddleCommon.MatsMiddleCommon(cbPool); + } - processStationQuery = ( - varName, - stationNames, - model, - threshold, - fromSecs, - toSecs, - utcCycleStart - ) => - { - const Future = require("fibers/future"); - - let rv = []; - const dFuture = new Future(); - (async () => - { - rv = await this.processStationQuery_int( - varName, - stationNames, - model, - threshold, - fromSecs, - toSecs, - utcCycleStart - ); - dFuture.return(); - })(); - dFuture.wait(); - return rv; - }; - - processStationQuery_int = async ( + processStationQuery = ( + varName, + stationNames, + model, + threshold, + fromSecs, + toSecs, + utcCycleStart + ) => { + const Future = require("fibers/future"); + + let rv = []; + const dFuture = new Future(); + (async () => { + rv = await this.processStationQuery_int( varName, stationNames, model, @@ -84,279 +65,265 @@ class MatsMiddleDailyModelCycle fromSecs, toSecs, utcCycleStart - ) => - { - const fs = require("fs"); - - console.log( - `processStationQuery(${varName},${stationNames.length - },${model},${threshold},${fromSecs},${toSecs},${JSON.stringify(utcCycleStart)})` - ); - - // TODO - remove debug overrides - /* - threshold = 3000.0; - fromSecs = 1662249600; - toSecs = 1664841600; - */ - - this.varName = varName; - this.stationNames = stationNames; - this.model = model; - this.threshold = threshold; - this.fromSecs = fromSecs; - this.toSecs = toSecs; - this.utcCycleStart = utcCycleStart; - - this.conn = await cbPool.getConnection(); - - const startTime = new Date().valueOf(); - this.fcstValidEpoch_Array = await this.mmCommon.get_fcstValidEpoch_Array( - this.fromSecs, - this.toSecs - ); - - let endTime = new Date().valueOf(); - console.log( - `\tfcstValidEpoch_Array:${this.fcstValidEpoch_Array.length} in ${endTime - startTime - } ms.` - ); - - const prObs = this.createObsData(); - const prModel = this.createModelData(); - await Promise.all([prObs, prModel]); - this.generateCtc(threshold); - - for (let i = 0; i < this.ctc.length; i++) - { - this.ctc[i] = this.mmCommon.sumUpCtc(this.ctc[i]); - } + ); + dFuture.return(); + })(); + dFuture.wait(); + return rv; + }; + + processStationQuery_int = async ( + varName, + stationNames, + model, + threshold, + fromSecs, + toSecs, + utcCycleStart + ) => { + const fs = require("fs"); + + console.log( + `processStationQuery(${varName},${ + stationNames.length + },${model},${threshold},${fromSecs},${toSecs},${JSON.stringify(utcCycleStart)})` + ); + + this.varName = varName; + this.stationNames = stationNames; + this.model = model; + this.threshold = threshold; + this.fromSecs = fromSecs; + this.toSecs = toSecs; + this.utcCycleStart = utcCycleStart; + + this.conn = await cbPool.getConnection(); + + const startTime = new Date().valueOf(); + this.fcstValidEpoch_Array = await this.mmCommon.get_fcstValidEpoch_Array( + this.fromSecs, + this.toSecs + ); + + let endTime = new Date().valueOf(); + console.log( + `\tfcstValidEpoch_Array:${this.fcstValidEpoch_Array.length} in ${ + endTime - startTime + } ms.` + ); + + const prObs = this.createObsData(); + const prModel = this.createModelData(); + await Promise.all([prObs, prModel]); + this.generateCtc(threshold); + + for (let i = 0; i < this.ctc.length; i++) { + this.ctc[i] = this.mmCommon.sumUpCtc(this.ctc[i]); + } - if (this.logToFile === true) - { - this.mmCommon.writeToLocalFile( - "/scratch/matsMiddle/output/fveObs.json", - JSON.stringify(this.fveObs, null, 2) - ); - this.mmCommon.writeToLocalFile( - "/scratch/matsMiddle/output/fveModels.json", - JSON.stringify(this.fveModels, null, 2) - ); - this.mmCommon.writeToLocalFile( - "/scratch/matsMiddle/output/ctc.json", - JSON.stringify(this.ctc, null, 2) - ); - } + if (this.logToFile === true) { + this.mmCommon.writeToLocalFile( + "/scratch/matsMiddle/output/fveObs.json", + JSON.stringify(this.fveObs, null, 2) + ); + this.mmCommon.writeToLocalFile( + "/scratch/matsMiddle/output/fveModels.json", + JSON.stringify(this.fveModels, null, 2) + ); + this.mmCommon.writeToLocalFile( + "/scratch/matsMiddle/output/ctc.json", + JSON.stringify(this.ctc, null, 2) + ); + } - endTime = new Date().valueOf(); - console.log(`\tprocessStationQuery in ${endTime - startTime} ms.`); + endTime = new Date().valueOf(); + console.log(`\tprocessStationQuery in ${endTime - startTime} ms.`); - return this.ctc; - }; + return this.ctc; + }; - createObsData = async () => - { - console.log("createObsData()"); - const fs = require("fs"); + createObsData = async () => { + console.log("createObsData()"); + const fs = require("fs"); - const startTime = new Date().valueOf(); + const startTime = new Date().valueOf(); - const tmpl_get_N_stations_mfve_obs = fs.readFileSync( - "assets/app/matsMiddle/sqlTemplates/tmpl_get_N_stations_mfve_IN_obs.sql", - "utf-8" - ); + const tmpl_get_N_stations_mfve_obs = fs.readFileSync( + "assets/app/matsMiddle/sqlTemplates/tmpl_get_N_stations_mfve_IN_obs.sql", + "utf-8" + ); - let stationNames_obs = ""; - for (let i = 0; i < this.stationNames.length; i++) - { - if (i === 0) - { - stationNames_obs = `obs.data.${this.stationNames[i]}.${this.varName} ${this.stationNames[i]}`; - } else - { - stationNames_obs += `,obs.data.${this.stationNames[i]}.${this.varName} ${this.stationNames[i]}`; - } + let stationNames_obs = ""; + for (let i = 0; i < this.stationNames.length; i++) { + if (i === 0) { + stationNames_obs = `obs.data.${this.stationNames[i]}.${this.varName} ${this.stationNames[i]}`; + } else { + stationNames_obs += `,obs.data.${this.stationNames[i]}.${this.varName} ${this.stationNames[i]}`; + } + } + let tmplWithStationNames_obs = this.cbPool.trfmSQLRemoveClause( + tmpl_get_N_stations_mfve_obs, + "{{vxAVERAGE}}" + ); + tmplWithStationNames_obs = tmplWithStationNames_obs.replace( + /{{stationNamesList}}/g, + stationNames_obs + ); + let endTime = new Date().valueOf(); + console.log(`\tobs query:${stationNames_obs.length} in ${endTime - startTime} ms.`); + + const promises = []; + for (let iofve = 0; iofve < this.fcstValidEpoch_Array.length; iofve += 100) { + const fveArraySlice = this.fcstValidEpoch_Array.slice(iofve, iofve + 100); + const sql = tmplWithStationNames_obs.replace( + /{{fcstValidEpoch}}/g, + JSON.stringify(fveArraySlice) + ); + if (this.logToFile === true && iofve === 0) { + this.mmCommon.writeToLocalFile("/scratch/matsMiddle/output/obs.sql", sql); + } + const prSlice = this.conn.cluster.query(sql); + promises.push(prSlice); + prSlice.then((qr) => { + console.log(`qr:\n${qr.rows.length}`); + for (let jmfve = 0; jmfve < qr.rows.length; jmfve++) { + const fveDataSingleEpoch = qr.rows[jmfve]; + const dataSingleEpoch = {}; + const stationsSingleEpoch = {}; + for (let i = 0; i < this.stationNames.length; i++) { + const varValStation = fveDataSingleEpoch[this.stationNames[i]]; + stationsSingleEpoch[this.stationNames[i]] = varValStation; + } + dataSingleEpoch.avtime = fveDataSingleEpoch.fve; + dataSingleEpoch.stations = stationsSingleEpoch; + this.fveObs[fveDataSingleEpoch.fve] = dataSingleEpoch; } - let tmplWithStationNames_obs = this.cbPool.trfmSQLRemoveClause( - tmpl_get_N_stations_mfve_obs, - "{{vxAVERAGE}}" - ); - tmplWithStationNames_obs = tmplWithStationNames_obs.replace( - /{{stationNamesList}}/g, - stationNames_obs - ); - let endTime = new Date().valueOf(); - console.log(`\tobs query:${stationNames_obs.length} in ${endTime - startTime} ms.`); - - const promises = []; - for (let iofve = 0; iofve < this.fcstValidEpoch_Array.length; iofve += 100) - { - const fveArraySlice = this.fcstValidEpoch_Array.slice(iofve, iofve + 100); - const sql = tmplWithStationNames_obs.replace( - /{{fcstValidEpoch}}/g, - JSON.stringify(fveArraySlice) - ); - if (this.logToFile === true && iofve === 0) - { - this.mmCommon.writeToLocalFile("/scratch/matsMiddle/output/obs.sql", sql); - } - const prSlice = this.conn.cluster.query(sql); - promises.push(prSlice); - prSlice.then((qr) => - { - console.log(`qr:\n${qr.rows.length}`); - for (let jmfve = 0; jmfve < qr.rows.length; jmfve++) - { - const fveDataSingleEpoch = qr.rows[jmfve]; - const dataSingleEpoch = {}; - const stationsSingleEpoch = {}; - for (let i = 0; i < this.stationNames.length; i++) - { - const varValStation = fveDataSingleEpoch[this.stationNames[i]]; - stationsSingleEpoch[this.stationNames[i]] = varValStation; - } - dataSingleEpoch.avtime = fveDataSingleEpoch.fve; - dataSingleEpoch.stations = stationsSingleEpoch; - this.fveObs[fveDataSingleEpoch.fve] = dataSingleEpoch; - } - if (iofve % 100 == 0) - { - endTime = new Date().valueOf(); - console.log( - `iofve:${iofve}/${this.fcstValidEpoch_Array.length} in ${endTime - startTime - } ms.` - ); - } - }); + if (iofve % 100 == 0) { + endTime = new Date().valueOf(); + console.log( + `iofve:${iofve}/${this.fcstValidEpoch_Array.length} in ${ + endTime - startTime + } ms.` + ); } + }); + } - await Promise.all(promises); - endTime = new Date().valueOf(); - console.log(`fveObs:` + ` in ${endTime - startTime} ms.`); - }; + await Promise.all(promises); + endTime = new Date().valueOf(); + console.log(`fveObs:` + ` in ${endTime - startTime} ms.`); + }; - createModelData = async () => - { - console.log("createModelData()"); - const fs = require("fs"); + createModelData = async () => { + console.log("createModelData()"); + const fs = require("fs"); - const startTime = new Date().valueOf(); + const startTime = new Date().valueOf(); - let tmpl_get_N_stations_mfve_model = fs.readFileSync( - "assets/app/matsMiddle/sqlTemplates/tmpl_get_N_stations_mfve_IN_model.sql", - "utf-8" - ); - /* + let tmpl_get_N_stations_mfve_model = fs.readFileSync( + "assets/app/matsMiddle/sqlTemplates/tmpl_get_N_stations_mfve_IN_model.sql", + "utf-8" + ); + /* tmpl_get_N_stations_mfve_model = this.cbPool.trfmSQLRemoveClause( tmpl_get_N_stations_mfve_model, "fcstLen fcst_lead" ); */ - tmpl_get_N_stations_mfve_model = this.cbPool.trfmSQLRemoveClause( - tmpl_get_N_stations_mfve_model, - "{{vxFCST_LEN_ARRAY}}" - ); - tmpl_get_N_stations_mfve_model = this.cbPool.trfmSQLRemoveClause( - tmpl_get_N_stations_mfve_model, - "{{vxAVERAGE}}" - ); - tmpl_get_N_stations_mfve_model = tmpl_get_N_stations_mfve_model.replace( - /{{vxMODEL}}/g, - `"${this.model}"` - ); - tmpl_get_N_stations_mfve_model = tmpl_get_N_stations_mfve_model.replace( - /fcstLen = {{vxFCST_LEN}}/g, - "fcstLen < 24 AND (models.fcstValidEpoch - models.fcstLen*3600)%(24*3600)/3600 IN [" + this.utcCycleStart + "]" - ); + tmpl_get_N_stations_mfve_model = this.cbPool.trfmSQLRemoveClause( + tmpl_get_N_stations_mfve_model, + "{{vxFCST_LEN_ARRAY}}" + ); + tmpl_get_N_stations_mfve_model = this.cbPool.trfmSQLRemoveClause( + tmpl_get_N_stations_mfve_model, + "{{vxAVERAGE}}" + ); + tmpl_get_N_stations_mfve_model = tmpl_get_N_stations_mfve_model.replace( + /{{vxMODEL}}/g, + `"${this.model}"` + ); + tmpl_get_N_stations_mfve_model = tmpl_get_N_stations_mfve_model.replace( + /fcstLen = {{vxFCST_LEN}}/g, + `fcstLen < 24 AND (models.fcstValidEpoch - models.fcstLen*3600)%(24*3600)/3600 IN [${this.utcCycleStart}]` + ); + + let stationNames_models = ""; + for (let i = 0; i < this.stationNames.length; i++) { + if (i === 0) { + stationNames_models = `models.data.${this.stationNames[i]}.${this.varName} ${this.stationNames[i]}`; + } else { + stationNames_models += `,models.data.${this.stationNames[i]}.${this.varName} ${this.stationNames[i]}`; + } + } - let stationNames_models = ""; - for (let i = 0; i < this.stationNames.length; i++) - { - if (i === 0) - { - stationNames_models = `models.data.${this.stationNames[i]}.${this.varName} ${this.stationNames[i]}`; - } else - { - stationNames_models += `,models.data.${this.stationNames[i]}.${this.varName} ${this.stationNames[i]}`; - } + const tmplWithStationNames_models = tmpl_get_N_stations_mfve_model.replace( + /{{stationNamesList}}/g, + stationNames_models + ); + let endTime = new Date().valueOf(); + console.log( + `\tmodel query:${stationNames_models.length} in ${endTime - startTime} ms.` + ); + + const promises = []; + for (let imfve = 0; imfve < this.fcstValidEpoch_Array.length; imfve += 100) { + const fveArraySlice = this.fcstValidEpoch_Array.slice(imfve, imfve + 100); + const sql = tmplWithStationNames_models.replace( + /{{fcstValidEpoch}}/g, + JSON.stringify(fveArraySlice) + ); + if (this.logToFile === true && imfve === 0) { + this.mmCommon.writeToLocalFile("/scratch/matsMiddle/output/model.sql", sql); + } + const prSlice = this.conn.cluster.query(sql); + + promises.push(prSlice); + prSlice.then((qr) => { + for (let jmfve = 0; jmfve < qr.rows.length; jmfve++) { + const fveDataSingleEpoch = qr.rows[jmfve]; + const dataSingleEpoch = {}; + const stationsSingleEpoch = {}; + for (let i = 0; i < this.stationNames.length; i++) { + const varValStation = fveDataSingleEpoch[this.stationNames[i]]; + stationsSingleEpoch[this.stationNames[i]] = varValStation; + } + dataSingleEpoch.avtime = fveDataSingleEpoch.fve; + dataSingleEpoch.fcst_lead = fveDataSingleEpoch.fcst_lead; + dataSingleEpoch.stations = stationsSingleEpoch; + this.fveModels[fveDataSingleEpoch.fve] = dataSingleEpoch; } - - const tmplWithStationNames_models = tmpl_get_N_stations_mfve_model.replace( - /{{stationNamesList}}/g, - stationNames_models - ); - let endTime = new Date().valueOf(); - console.log( - `\tmodel query:${stationNames_models.length} in ${endTime - startTime} ms.` - ); - - const promises = []; - for (let imfve = 0; imfve < this.fcstValidEpoch_Array.length; imfve += 100) - { - const fveArraySlice = this.fcstValidEpoch_Array.slice(imfve, imfve + 100); - const sql = tmplWithStationNames_models.replace( - /{{fcstValidEpoch}}/g, - JSON.stringify(fveArraySlice) - ); - if (this.logToFile === true && imfve === 0) - { - this.mmCommon.writeToLocalFile("/scratch/matsMiddle/output/model.sql", sql); - } - const prSlice = this.conn.cluster.query(sql); - - promises.push(prSlice); - prSlice.then((qr) => - { - for (let jmfve = 0; jmfve < qr.rows.length; jmfve++) - { - const fveDataSingleEpoch = qr.rows[jmfve]; - const dataSingleEpoch = {}; - const stationsSingleEpoch = {}; - for (let i = 0; i < this.stationNames.length; i++) - { - const varValStation = fveDataSingleEpoch[this.stationNames[i]]; - stationsSingleEpoch[this.stationNames[i]] = varValStation; - } - dataSingleEpoch.avtime = fveDataSingleEpoch.fve; - dataSingleEpoch.fcst_lead = fveDataSingleEpoch.fcst_lead; - dataSingleEpoch.stations = stationsSingleEpoch; - this.fveModels[fveDataSingleEpoch.fve] = dataSingleEpoch; - } - if (imfve % 100 == 0) - { - endTime = new Date().valueOf(); - console.log( - `imfve:${imfve}/${this.fcstValidEpoch_Array.length} in ${endTime - startTime - } ms.` - ); - } - }); + if (imfve % 100 == 0) { + endTime = new Date().valueOf(); + console.log( + `imfve:${imfve}/${this.fcstValidEpoch_Array.length} in ${ + endTime - startTime + } ms.` + ); } - await Promise.all(promises); - endTime = new Date().valueOf(); - console.log(`fveModel:` + ` in ${endTime - startTime} ms.`); - }; + }); + } + await Promise.all(promises); + endTime = new Date().valueOf(); + console.log(`fveModel:` + ` in ${endTime - startTime} ms.`); + }; - generateCtc = async (threshold) => - { - console.log(`generateCtc(${threshold})`); + generateCtc = async (threshold) => { + console.log(`generateCtc(${threshold})`); - const startTime = new Date().valueOf(); + const startTime = new Date().valueOf(); - for (let imfve = 0; imfve < this.fcstValidEpoch_Array.length; imfve++) - { - const fve = this.fcstValidEpoch_Array[imfve]; - const obsSingleFve = this.fveObs[fve]; - const modelSingleFve = this.fveModels[fve]; + for (let imfve = 0; imfve < this.fcstValidEpoch_Array.length; imfve++) { + const fve = this.fcstValidEpoch_Array[imfve]; + const obsSingleFve = this.fveObs[fve]; + const modelSingleFve = this.fveModels[fve]; - if (!obsSingleFve || !modelSingleFve) - { - continue; - } + if (!obsSingleFve || !modelSingleFve) { + continue; + } - const fcst_lead = modelSingleFve.fcst_lead; + const { fcst_lead } = modelSingleFve; - /* + /* if (this.utcCycleStart) { // (obs.fcstValidEpoch - obs.fcstLen*3600)%(24*3600)/3600 IN[vxUTC_CYCLE_START]) @@ -369,74 +336,64 @@ class MatsMiddleDailyModelCycle } */ - const stats_fve = {}; - stats_fve.avtime = fve; - stats_fve.min_secs = fve; - stats_fve.max_secs = fve; - stats_fve.hit = 0; - stats_fve.miss = 0; - stats_fve.fa = 0; - stats_fve.cn = 0; - stats_fve.N0 = 0; - stats_fve.N_times = 1; - stats_fve.sub_data = []; - - for (let i = 0; i < this.stationNames.length; i++) - { - const station = this.stationNames[i]; - const varVal_o = obsSingleFve.stations[station]; - const varVal_m = modelSingleFve.stations[station]; - - if (varVal_o && varVal_m) - { - stats_fve.N0 += 1; - let sub = `${fve};`; - if (varVal_o < threshold && varVal_m < threshold) - { - stats_fve.hit += 1; - sub += "1;"; - } else - { - sub += "0;"; - } - - if (varVal_o >= threshold && varVal_m < threshold) - { - stats_fve.fa += 1; - sub += "1;"; - } else - { - sub += "0;"; - } - - if (varVal_o < threshold && varVal_m >= threshold) - { - stats_fve.miss += 1; - sub += "1;"; - } else - { - sub += "0;"; - } - - if (varVal_o >= threshold && varVal_m >= threshold) - { - stats_fve.cn += 1; - sub += "1"; - } else - { - sub += "0"; - } - stats_fve.sub_data.push(sub); - } - } - this.ctc.push(stats_fve); + const stats_fve = {}; + stats_fve.avtime = fve; + stats_fve.min_secs = fve; + stats_fve.max_secs = fve; + stats_fve.hit = 0; + stats_fve.miss = 0; + stats_fve.fa = 0; + stats_fve.cn = 0; + stats_fve.N0 = 0; + stats_fve.N_times = 1; + stats_fve.sub_data = []; + + for (let i = 0; i < this.stationNames.length; i++) { + const station = this.stationNames[i]; + const varVal_o = obsSingleFve.stations[station]; + const varVal_m = modelSingleFve.stations[station]; + + if (varVal_o && varVal_m) { + stats_fve.N0 += 1; + let sub = `${fve};`; + if (varVal_o < threshold && varVal_m < threshold) { + stats_fve.hit += 1; + sub += "1;"; + } else { + sub += "0;"; + } + + if (varVal_o >= threshold && varVal_m < threshold) { + stats_fve.fa += 1; + sub += "1;"; + } else { + sub += "0;"; + } + + if (varVal_o < threshold && varVal_m >= threshold) { + stats_fve.miss += 1; + sub += "1;"; + } else { + sub += "0;"; + } + + if (varVal_o >= threshold && varVal_m >= threshold) { + stats_fve.cn += 1; + sub += "1"; + } else { + sub += "0"; + } + stats_fve.sub_data.push(sub); } + } + this.ctc.push(stats_fve); + } - const endTime = new Date().valueOf(); - console.log(`generateCtc:` + ` in ${endTime - startTime} ms.`); - }; + const endTime = new Date().valueOf(); + console.log(`generateCtc:` + ` in ${endTime - startTime} ms.`); + }; } export default matsMiddleDailyModelCycle = { - MatsMiddleDailyModelCycle, + MatsMiddleDailyModelCycle, }; diff --git a/meteor_packages/mats-common/imports/startup/server/matsMiddle_dailyModelCycle_0.js b/meteor_packages/mats-common/imports/startup/server/matsMiddle_map.js similarity index 55% rename from meteor_packages/mats-common/imports/startup/server/matsMiddle_dailyModelCycle_0.js rename to meteor_packages/mats-common/imports/startup/server/matsMiddle_map.js index 0be05a0c9..fa6f1b3bc 100644 --- a/meteor_packages/mats-common/imports/startup/server/matsMiddle_dailyModelCycle_0.js +++ b/meteor_packages/mats-common/imports/startup/server/matsMiddle_map.js @@ -1,21 +1,17 @@ import { matsMiddleCommon } from "meteor/randyp:mats-common"; - import { Meteor } from "meteor/meteor"; -import { memoryUsage } from "node:process"; -class MatsMiddleDailyModelCycle_0 { +class MatsMiddleMap { logToFile = false; logMemUsage = false; - hrOfDay_Array = []; + fcstValidEpoch_Array = []; cbPool = null; conn = null; - fcstLenArray = []; - fveObs = {}; fveModels = {}; @@ -24,7 +20,7 @@ class MatsMiddleDailyModelCycle_0 { varName = null; - stationNames = null; + stationNamesFull = null; model = null; @@ -36,12 +32,12 @@ class MatsMiddleDailyModelCycle_0 { toSecs = null; + validTimes = []; + writeOutput = false; mmCommon = null; - ctcTime = 0; - constructor(cbPool) { this.cbPool = cbPool; this.mmCommon = new matsMiddleCommon.MatsMiddleCommon(cbPool); @@ -88,6 +84,8 @@ class MatsMiddleDailyModelCycle_0 { toSecs, validTimes ) => { + const fs = require("fs"); + console.log( `processStationQuery(${varName},${ stationNames.length @@ -97,7 +95,7 @@ class MatsMiddleDailyModelCycle_0 { ); this.varName = varName; - this.stationNames = stationNames; + this.stationNamesFull = stationNames; this.model = model; this.fcstLen = fcstLen; this.threshold = threshold; @@ -116,25 +114,11 @@ class MatsMiddleDailyModelCycle_0 { this.conn = await cbPool.getConnection(); const startTime = new Date().valueOf(); - this.fcstValidEpoch_Array = await this.mmCommon.get_fcstValidEpoch_Array( fromSecs, toSecs ); - // create distinct hour of day array - for (let iofve = 0; iofve < this.fcstValidEpoch_Array.length; ++iofve) { - const ofve = this.fcstValidEpoch_Array[iofve]; - const hod = (ofve % (24 * 3600)) / 3600; - if (!this.hrOfDay_Array.includes(hod)) { - this.hrOfDay_Array.push(hod); - } - } - this.hrOfDay_Array.sort((a, b) => a - b); - if (this.logToFile === true) { - console.log(`hrOfDay_Array:\n${JSON.stringify(this.hrOfDay_Array)}`); - } - let endTime = new Date().valueOf(); console.log( `\tfcstValidEpoch_Array:${this.fcstValidEpoch_Array.length} in ${ @@ -142,8 +126,29 @@ class MatsMiddleDailyModelCycle_0 { } ms.` ); - await this.createObsData(); - await this.createModelData(); + for (let iofve = 0; iofve < this.stationNamesFull.length; iofve += 100) { + const stationNamesSlice = this.stationNamesFull.slice(iofve, iofve + 100); + const prObs = this.createObsData(stationNamesSlice); + const prModel = this.createModelData(stationNamesSlice); + await Promise.all([prObs, prModel]); + this.generateCtc(threshold, stationNamesSlice); + endTime = new Date().valueOf(); + console.log( + `stations:${iofve + stationNamesSlice.length}/${ + this.stationNamesFull.length + } in ${endTime - startTime} ms` + ); + } + + this.fveObs = {}; + this.fveModels = {}; + + /* + for (let i = 0; i < this.ctc.length; i++) + { + this.ctc[i] = this.mmCommon.sumUpCtc(this.ctc[i]); + } + */ if (this.logToFile === true) { this.mmCommon.writeToLocalFile( @@ -161,14 +166,12 @@ class MatsMiddleDailyModelCycle_0 { } endTime = new Date().valueOf(); - - console.log(`\tctcTime: ${this.ctcTime} ms.`); console.log(`\tprocessStationQuery in ${endTime - startTime} ms.`); return this.ctc; }; - createObsData = async () => { + createObsData = async (stationNamesSlice) => { console.log("createObsData()"); const fs = require("fs"); @@ -179,15 +182,17 @@ class MatsMiddleDailyModelCycle_0 { "utf-8" ); + this.fveObs = {}; + let stationNames_obs = ""; - for (let i = 0; i < this.stationNames.length; i++) { + for (let i = 0; i < stationNamesSlice.length; i++) { if (i === 0) { - stationNames_obs = `obs.data.${this.stationNames[i]}.${this.varName} ${this.stationNames[i]}`; + stationNames_obs = `obs.data.${stationNamesSlice[i]}.${this.varName} ${stationNamesSlice[i]}`; } else { - stationNames_obs += `,obs.data.${this.stationNames[i]}.${this.varName} ${this.stationNames[i]}`; + stationNames_obs += `,obs.data.${stationNamesSlice[i]}.${this.varName} ${stationNamesSlice[i]}`; } } - let tmplWithStationNames_obs = cbPool.trfmSQLRemoveClause( + let tmplWithStationNames_obs = this.cbPool.trfmSQLRemoveClause( tmpl_get_N_stations_mfve_obs, "{{vxAVERAGE}}" ); @@ -214,20 +219,16 @@ class MatsMiddleDailyModelCycle_0 { console.log(`qr:\n${qr.rows.length}`); for (let jmfve = 0; jmfve < qr.rows.length; jmfve++) { const fveDataSingleEpoch = qr.rows[jmfve]; - const hod = (fveDataSingleEpoch.fve % (24 * 3600)) / 3600; - const hodKey = hod.toString(); - if (!this.fveObs[hodKey]) { - this.fveObs[hodKey] = {}; - } const dataSingleEpoch = {}; const stationsSingleEpoch = {}; - for (let i = 0; i < this.stationNames.length; i++) { - const varValStation = fveDataSingleEpoch[this.stationNames[i]]; - stationsSingleEpoch[this.stationNames[i]] = varValStation; + for (let i = 0; i < stationNamesSlice.length; i++) { + if (!this.fveObs[stationNamesSlice[i]]) { + this.fveObs[stationNamesSlice[i]] = {}; + this.fveObs[stationNamesSlice[i]][fveDataSingleEpoch.fve] = {}; + } + const varValStation = fveDataSingleEpoch[stationNamesSlice[i]]; + this.fveObs[stationNamesSlice[i]][fveDataSingleEpoch.fve] = varValStation; } - dataSingleEpoch.fcst = fveDataSingleEpoch.fcst; - dataSingleEpoch.stations = stationsSingleEpoch; - this.fveObs[hodKey][fveDataSingleEpoch.fve] = dataSingleEpoch; } if (iofve % 100 == 0) { endTime = new Date().valueOf(); @@ -245,29 +246,30 @@ class MatsMiddleDailyModelCycle_0 { console.log(`fveObs:` + ` in ${endTime - startTime} ms.`); }; - createModelData = async () => { + createModelData = async (stationNamesSlice) => { console.log("createModelData()"); const fs = require("fs"); const startTime = new Date().valueOf(); + this.fveModels = {}; + let tmpl_get_N_stations_mfve_model = fs.readFileSync( "assets/app/matsMiddle/sqlTemplates/tmpl_get_N_stations_mfve_IN_model.sql", "utf-8" ); tmpl_get_N_stations_mfve_model = this.cbPool.trfmSQLRemoveClause( tmpl_get_N_stations_mfve_model, - "{{vxFCST_LEN_ARRAY}}" + "fcstLen fcst_lead" ); tmpl_get_N_stations_mfve_model = this.cbPool.trfmSQLRemoveClause( tmpl_get_N_stations_mfve_model, - "{{vxAVERAGE}}" + "{{vxFCST_LEN_ARRAY}}" ); tmpl_get_N_stations_mfve_model = this.cbPool.trfmSQLRemoveClause( tmpl_get_N_stations_mfve_model, - "fcstLen fcst_lead" + "{{vxAVERAGE}}" ); - tmpl_get_N_stations_mfve_model = tmpl_get_N_stations_mfve_model.replace( /{{vxMODEL}}/g, `"${this.model}"` @@ -276,12 +278,17 @@ class MatsMiddleDailyModelCycle_0 { /{{vxFCST_LEN}}/g, this.fcstLen ); + tmpl_get_N_stations_mfve_model = tmpl_get_N_stations_mfve_model.replace( + /{{vxAVERAGE}}/g, + this.average + ); + let stationNames_models = ""; - for (let i = 0; i < this.stationNames.length; i++) { + for (let i = 0; i < stationNamesSlice.length; i++) { if (i === 0) { - stationNames_models = `models.data.${this.stationNames[i]}.${this.varName} ${this.stationNames[i]}`; + stationNames_models = `models.data.${stationNamesSlice[i]}.${this.varName} ${stationNamesSlice[i]}`; } else { - stationNames_models += `,models.data.${this.stationNames[i]}.${this.varName} ${this.stationNames[i]}`; + stationNames_models += `,models.data.${stationNamesSlice[i]}.${this.varName} ${stationNamesSlice[i]}`; } } @@ -294,8 +301,6 @@ class MatsMiddleDailyModelCycle_0 { `\tmodel query:${stationNames_models.length} in ${endTime - startTime} ms.` ); - const flaIncr = 3; - this.fveModels = {}; const promises = []; for (let imfve = 0; imfve < this.fcstValidEpoch_Array.length; imfve += 100) { const fveArraySlice = this.fcstValidEpoch_Array.slice(imfve, imfve + 100); @@ -310,91 +315,69 @@ class MatsMiddleDailyModelCycle_0 { promises.push(prSlice); prSlice.then((qr) => { - const idx = imfve / 100; for (let jmfve = 0; jmfve < qr.rows.length; jmfve++) { const fveDataSingleEpoch = qr.rows[jmfve]; - const hod = (fveDataSingleEpoch.fve % (24 * 3600)) / 3600; - const hodKey = hod.toString(); - if (!this.fveModels[hodKey]) { - this.fveModels[hodKey] = {}; - } const dataSingleEpoch = {}; const stationsSingleEpoch = {}; - for (let i = 0; i < this.stationNames.length; i++) { - const varValStation = fveDataSingleEpoch[this.stationNames[i]]; - stationsSingleEpoch[this.stationNames[i]] = varValStation; + for (let i = 0; i < stationNamesSlice.length; i++) { + if (!this.fveModels[stationNamesSlice[i]]) { + this.fveModels[stationNamesSlice[i]] = {}; + this.fveModels[stationNamesSlice[i]][fveDataSingleEpoch.fve] = {}; + } + const varValStation = fveDataSingleEpoch[stationNamesSlice[i]]; + this.fveModels[stationNamesSlice[i]][fveDataSingleEpoch.fve] = + varValStation; } - dataSingleEpoch.stations = stationsSingleEpoch; - this.fveModels[hodKey][fveDataSingleEpoch.fve] = dataSingleEpoch; } - - endTime = new Date().valueOf(); - console.log( - `imfve:${imfve}/${this.fcstValidEpoch_Array.length} idx: ${idx} in ${ - endTime - startTime - } ms.` - ); - - if (this.logMemUsage === true) { - try { - console.log(memoryUsage()); - const obsSize = - new TextEncoder().encode(JSON.stringify(this.fveObs)).length / - (1024 * 1024); - const modelsSize = - new TextEncoder().encode(JSON.stringify(this.fveModels)).length / - (1024 * 1024); - const ctcSize = - new TextEncoder().encode(JSON.stringify(this.ctc)).length / (1024 * 1024); - console.log( - `sizes (MB), obs:${obsSize},model:${modelsSize},ctc:${ctcSize}` - ); - } catch (ex) { - console.log(`exception getting sizes:${ex}`); - } + if (imfve % 100 == 0) { + endTime = new Date().valueOf(); + console.log( + `imfve:${imfve}/${this.fcstValidEpoch_Array.length} in ${ + endTime - startTime + } ms.` + ); } }); } await Promise.all(promises); - this.ctcTime += this.generateCtc(); endTime = new Date().valueOf(); console.log(`fveModel:` + ` in ${endTime - startTime} ms.`); }; - generateCtc = () => { - console.log("generateCtc()"); - - const { threshold } = this; + generateCtc = async (threshold, stationNamesSlice) => { + console.log(`generateCtc(${threshold})`); const startTime = new Date().valueOf(); - for (let ihod = 0; ihod < this.hrOfDay_Array.length; ihod++) { - const stats_hod = {}; - - const hod = this.hrOfDay_Array[ihod]; - const hodKey = hod.toString(); - stats_hod.hr_of_day = hod; - stats_hod.hit = 0; - stats_hod.miss = 0; - stats_hod.fa = 0; - stats_hod.cn = 0; - stats_hod.N0 = 0; - stats_hod.N_times = new Set(this.hrOfDay_Array).size; - stats_hod.sub_data = []; - - // get all the fve for this hod - const hod_single = this.fveModels[hodKey]; - const fve_array = Object.keys(hod_single); - fve_array.sort(); - - stats_hod.min_secs = fve_array[0]; - stats_hod.max_secs = fve_array[fve_array.length - 1]; - for (let imfve = 0; imfve < fve_array.length; imfve++) { - const fve = fve_array[imfve]; - const obsSingleFve = this.fveObs[hodKey][fve]; - const modelSingleFve = hod_single[fve]; - - if (!obsSingleFve || !modelSingleFve) { + for (let stni = 0; stni < stationNamesSlice.length; stni++) { + const stn = stationNamesSlice[stni]; + stnObs = this.fveObs[stn]; + stnModel = this.fveModels[stn]; + + if (!stnObs || !stnModel) { + continue; + } + + const stats_fve = {}; + stats_fve.sta_id = stn; + stats_fve.hit = 0; + stats_fve.miss = 0; + stats_fve.fa = 0; + stats_fve.cn = 0; + stats_fve.N0 = 0; + stats_fve.N_times = 0; + stats_fve.sub_data = []; + stats_fve.min_secs = this.fcstValidEpoch_Array[0]; + stats_fve.max_secs = + this.fcstValidEpoch_Array[this.fcstValidEpoch_Array.length - 1]; + + for (let imfve = 0; imfve < this.fcstValidEpoch_Array.length; imfve++) { + const fve = this.fcstValidEpoch_Array[imfve]; + + const varVal_o = stnObs[fve]; + const varVal_m = stnModel[fve]; + + if (!varVal_o || !varVal_m) { continue; } @@ -404,59 +387,50 @@ class MatsMiddleDailyModelCycle_0 { } } - for (let i = 0; i < this.stationNames.length; i++) { - const station = this.stationNames[i]; - const varVal_o = obsSingleFve.stations[station]; - const varVal_m = modelSingleFve.stations[station]; - - if (varVal_o && varVal_m) { - stats_hod.N0 += 1; - let sub = `${fve};`; - if (varVal_o < threshold && varVal_m < threshold) { - stats_hod.hit += 1; - sub += "1;"; - } else { - sub += "0;"; - } + stats_fve.N0 += 1; + stats_fve.N_times += 1; - if (varVal_o >= threshold && varVal_m < threshold) { - stats_hod.fa += 1; - sub += "1;"; - } else { - sub += "0;"; - } + let sub = `${fve};`; + if (varVal_o < threshold && varVal_m < threshold) { + stats_fve.hit += 1; + sub += "1;"; + } else { + sub += "0;"; + } - if (varVal_o < threshold && varVal_m >= threshold) { - stats_hod.miss += 1; - sub += "1;"; - } else { - sub += "0;"; - } + if (varVal_o >= threshold && varVal_m < threshold) { + stats_fve.fa += 1; + sub += "1;"; + } else { + sub += "0;"; + } - if (varVal_o >= threshold && varVal_m >= threshold) { - stats_hod.cn += 1; - sub += "1"; - } else { - sub += "0"; - } - stats_hod.sub_data.push(sub); - } + if (varVal_o < threshold && varVal_m >= threshold) { + stats_fve.miss += 1; + sub += "1;"; + } else { + sub += "0;"; } + + if (varVal_o >= threshold && varVal_m >= threshold) { + stats_fve.cn += 1; + varVal_o; + sub += "1"; + } else { + sub += "0"; + } + // stats_fve.sub_data.push(sub); } - try { - const stats_fcst_lead_summed = this.mmCommon.sumUpCtc(stats_hod); - this.ctc.push(stats_fcst_lead_summed); - } catch (ex) { - console.log(ex); - } + const sub = `${this.fcstValidEpoch_Array[0]};${stats_fve.hit};${stats_fve.fa};${stats_fve.miss};${stats_fve.cn}`; + stats_fve.sub_data.push(sub); + this.ctc.push(stats_fve); } const endTime = new Date().valueOf(); console.log(`generateCtc:` + ` in ${endTime - startTime} ms.`); - return endTime - startTime; }; } -export default matsMiddleDailyModelCycle_0 = { - MatsMiddleDailyModelCycle_0, +export default matsMiddleMap = { + MatsMiddleMap, }; diff --git a/meteor_packages/mats-common/package.js b/meteor_packages/mats-common/package.js index b75f0b6aa..1298b8e2c 100644 --- a/meteor_packages/mats-common/package.js +++ b/meteor_packages/mats-common/package.js @@ -94,6 +94,7 @@ Package.onUse(function (api) { api.export("matsMiddleTimeSeries", ["server"]); api.export("matsMiddleValidTime", ["server"]); api.export("matsMiddleDailyModelCycle", ["server"]); + api.export("matsMiddleMap", ["server"]); // add imports // both @@ -136,6 +137,7 @@ Package.onUse(function (api) { api.addFiles("imports/startup/server/matsMiddle_dieoff.js"); api.addFiles("imports/startup/server/matsMiddle_validTime.js"); api.addFiles("imports/startup/server/matsMiddle_dailyModelCycle.js"); + api.addFiles("imports/startup/server/matsMiddle_map.js"); // files outside of imports // client