Skip to content

Commit

Permalink
ValidTime all stations for 1 month in 33 seconds
Browse files Browse the repository at this point in the history
  • Loading branch information
gopa-noaa committed Aug 10, 2023
1 parent 25e911b commit f7ea0ed
Show file tree
Hide file tree
Showing 2 changed files with 112 additions and 55 deletions.
149 changes: 98 additions & 51 deletions meteor_packages/mats-common/imports/startup/server/matsMiddle_timeSeries.js
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
import { matsMiddleCommon } from "meteor/randyp:mats-common";
import { Meteor } from "meteor/meteor";

class MatsMiddleTimeSeries {
class MatsMiddleTimeSeries
{
logToFile = false;

logMemUsage = false;
Expand Down Expand Up @@ -40,7 +41,8 @@ class MatsMiddleTimeSeries {

mmCommon = null;

constructor(cbPool) {
constructor(cbPool)
{
this.cbPool = cbPool;
this.mmCommon = new matsMiddleCommon.MatsMiddleCommon(cbPool);
}
Expand All @@ -55,12 +57,14 @@ class MatsMiddleTimeSeries {
fromSecs,
toSecs,
validTimes
) => {
) =>
{
const Future = require("fibers/future");

let rv = [];
const dFuture = new Future();
(async () => {
(async () =>
{
rv = await this.processStationQuery_int(
varName,
stationNames,
Expand Down Expand Up @@ -88,12 +92,12 @@ class MatsMiddleTimeSeries {
fromSecs,
toSecs,
validTimes
) => {
) =>
{
const fs = require("fs");

console.log(
`processStationQuery(${varName},${
stationNames.length
`processStationQuery(${varName},${stationNames.length
},${model},${fcstLen},${threshold},${average},${fromSecs},${toSecs},${JSON.stringify(
validTimes
)})`
Expand All @@ -109,9 +113,12 @@ class MatsMiddleTimeSeries {
this.toSecs = toSecs;

this.average = this.average.replace(/m0./g, "");
if (validTimes && validTimes.length > 0) {
for (let i = 0; i < validTimes.length; i++) {
if (validTimes[i] != null && Number(validTimes[i]) > 0) {
if (validTimes && validTimes.length > 0)
{
for (let i = 0; i < validTimes.length; i++)
{
if (validTimes[i] != null && Number(validTimes[i]) > 0)
{
this.validTimes.push(Number(validTimes[i]));
}
}
Expand All @@ -128,8 +135,7 @@ class MatsMiddleTimeSeries {

let endTime = new Date().valueOf();
console.log(
`\tfcstValidEpoch_Array:${this.fcstValidEpoch_Array.length} in ${
endTime - startTime
`\tfcstValidEpoch_Array:${this.fcstValidEpoch_Array.length} in ${endTime - startTime
} ms.`
);

Expand All @@ -138,11 +144,13 @@ class MatsMiddleTimeSeries {
await Promise.all([prObs, prModel]);
this.generateCtc(threshold);

for (let i = 0; i < this.ctc.length; i++) {
for (let i = 0; i < this.ctc.length; i++)
{
this.ctc[i] = this.mmCommon.sumUpCtc(this.ctc[i]);
}

if (this.logToFile === true) {
if (this.logToFile === true)
{
this.mmCommon.writeToLocalFile(
"/scratch/matsMiddle/output/fveObs.json",
JSON.stringify(this.fveObs, null, 2)
Expand All @@ -163,7 +171,8 @@ class MatsMiddleTimeSeries {
return this.ctc;
};

createObsData = async () => {
createObsData = async () =>
{
console.log("createObsData()");
const fs = require("fs");

Expand All @@ -175,10 +184,13 @@ class MatsMiddleTimeSeries {
);

let stationNames_obs = "";
for (let i = 0; i < this.stationNames.length; i++) {
if (i === 0) {
for (let i = 0; i < this.stationNames.length; i++)
{
if (i === 0)
{
stationNames_obs = `obs.data.${this.stationNames[i]}.${this.varName} ${this.stationNames[i]}`;
} else {
} else
{
stationNames_obs += `,obs.data.${this.stationNames[i]}.${this.varName} ${this.stationNames[i]}`;
}
}
Expand All @@ -194,33 +206,41 @@ class MatsMiddleTimeSeries {
console.log(`\tobs query:${stationNames_obs.length} in ${endTime - startTime} ms.`);

const promises = [];
for (let iofve = 0; iofve < this.fcstValidEpoch_Array.length; iofve += 100) {
for (let iofve = 0; iofve < this.fcstValidEpoch_Array.length; iofve += 100)
{
const fveArraySlice = this.fcstValidEpoch_Array.slice(iofve, iofve + 100);
const sql = tmplWithStationNames_obs.replace(
/{{fcstValidEpoch}}/g,
JSON.stringify(fveArraySlice)
);
if (this.logToFile === true && iofve === 0)
{
this.mmCommon.writeToLocalFile("/scratch/matsMiddle/output/obs.sql", sql);
}
const prSlice = this.conn.cluster.query(sql);
promises.push(prSlice);
prSlice.then((qr) => {
prSlice.then((qr) =>
{
console.log(`qr:\n${qr.rows.length}`);
for (let jmfve = 0; jmfve < qr.rows.length; jmfve++) {
for (let jmfve = 0; jmfve < qr.rows.length; jmfve++)
{
const fveDataSingleEpoch = qr.rows[jmfve];
const dataSingleEpoch = {};
const stationsSingleEpoch = {};
for (let i = 0; i < this.stationNames.length; i++) {
for (let i = 0; i < this.stationNames.length; i++)
{
const varValStation = fveDataSingleEpoch[this.stationNames[i]];
stationsSingleEpoch[this.stationNames[i]] = varValStation;
}
dataSingleEpoch.avtime = fveDataSingleEpoch.avtime;
dataSingleEpoch.stations = stationsSingleEpoch;
this.fveObs[fveDataSingleEpoch.fve] = dataSingleEpoch;
}
if (iofve % 100 == 0) {
if (iofve % 100 == 0)
{
endTime = new Date().valueOf();
console.log(
`iofve:${iofve}/${this.fcstValidEpoch_Array.length} in ${
endTime - startTime
`iofve:${iofve}/${this.fcstValidEpoch_Array.length} in ${endTime - startTime
} ms.`
);
}
Expand All @@ -232,7 +252,8 @@ class MatsMiddleTimeSeries {
console.log(`fveObs:` + ` in ${endTime - startTime} ms.`);
};

createModelData = async () => {
createModelData = async () =>
{
console.log("createModelData()");
const fs = require("fs");

Expand Down Expand Up @@ -264,10 +285,13 @@ class MatsMiddleTimeSeries {
);

let stationNames_models = "";
for (let i = 0; i < this.stationNames.length; i++) {
if (i === 0) {
for (let i = 0; i < this.stationNames.length; i++)
{
if (i === 0)
{
stationNames_models = `models.data.${this.stationNames[i]}.${this.varName} ${this.stationNames[i]}`;
} else {
} else
{
stationNames_models += `,models.data.${this.stationNames[i]}.${this.varName} ${this.stationNames[i]}`;
}
}
Expand All @@ -282,33 +306,41 @@ class MatsMiddleTimeSeries {
);

const promises = [];
for (let imfve = 0; imfve < this.fcstValidEpoch_Array.length; imfve += 100) {
for (let imfve = 0; imfve < this.fcstValidEpoch_Array.length; imfve += 100)
{
const fveArraySlice = this.fcstValidEpoch_Array.slice(imfve, imfve + 100);
const sql = tmplWithStationNames_models.replace(
/{{fcstValidEpoch}}/g,
JSON.stringify(fveArraySlice)
);
if (this.logToFile === true && imfve === 0)
{
this.mmCommon.writeToLocalFile("/scratch/matsMiddle/output/model.sql", sql);
}
const prSlice = this.conn.cluster.query(sql);

promises.push(prSlice);
prSlice.then((qr) => {
for (let jmfve = 0; jmfve < qr.rows.length; jmfve++) {
prSlice.then((qr) =>
{
for (let jmfve = 0; jmfve < qr.rows.length; jmfve++)
{
const fveDataSingleEpoch = qr.rows[jmfve];
const dataSingleEpoch = {};
const stationsSingleEpoch = {};
for (let i = 0; i < this.stationNames.length; i++) {
for (let i = 0; i < this.stationNames.length; i++)
{
const varValStation = fveDataSingleEpoch[this.stationNames[i]];
stationsSingleEpoch[this.stationNames[i]] = varValStation;
}
dataSingleEpoch.avtime = fveDataSingleEpoch.avtime;
dataSingleEpoch.stations = stationsSingleEpoch;
this.fveModels[fveDataSingleEpoch.fve] = dataSingleEpoch;
}
if (imfve % 100 == 0) {
if (imfve % 100 == 0)
{
endTime = new Date().valueOf();
console.log(
`imfve:${imfve}/${this.fcstValidEpoch_Array.length} in ${
endTime - startTime
`imfve:${imfve}/${this.fcstValidEpoch_Array.length} in ${endTime - startTime
} ms.`
);
}
Expand All @@ -319,22 +351,27 @@ class MatsMiddleTimeSeries {
console.log(`fveModel:` + ` in ${endTime - startTime} ms.`);
};

generateCtc = async (threshold) => {
generateCtc = async (threshold) =>
{
console.log(`generateCtc(${threshold})`);

const startTime = new Date().valueOf();

for (let imfve = 0; imfve < this.fcstValidEpoch_Array.length; imfve++) {
for (let imfve = 0; imfve < this.fcstValidEpoch_Array.length; imfve++)
{
const fve = this.fcstValidEpoch_Array[imfve];
const obsSingleFve = this.fveObs[fve];
const modelSingleFve = this.fveModels[fve];

if (!obsSingleFve || !modelSingleFve) {
if (!obsSingleFve || !modelSingleFve)
{
continue;
}

if (this.validTimes && this.validTimes.length > 0) {
if (this.validTimes.includes((fve % (24 * 3600)) / 3600) == false) {
if (this.validTimes && this.validTimes.length > 0)
{
if (this.validTimes.includes((fve % (24 * 3600)) / 3600) == false)
{
continue;
}
}
Expand All @@ -349,39 +386,49 @@ class MatsMiddleTimeSeries {
stats_fve.N_times = 1;
stats_fve.sub_data = [];

for (let i = 0; i < this.stationNames.length; i++) {
for (let i = 0; i < this.stationNames.length; i++)
{
const station = this.stationNames[i];
const varVal_o = obsSingleFve.stations[station];
const varVal_m = modelSingleFve.stations[station];

if (varVal_o && varVal_m) {
if (varVal_o && varVal_m)
{
stats_fve.N0 += 1;
let sub = `${fve};`;
if (varVal_o < threshold && varVal_m < threshold) {
if (varVal_o < threshold && varVal_m < threshold)
{
stats_fve.hit += 1;
sub += "1;";
} else {
} else
{
sub += "0;";
}

if (varVal_o >= threshold && varVal_m < threshold) {
if (varVal_o >= threshold && varVal_m < threshold)
{
stats_fve.fa += 1;
sub += "1;";
} else {
} else
{
sub += "0;";
}

if (varVal_o < threshold && varVal_m >= threshold) {
if (varVal_o < threshold && varVal_m >= threshold)
{
stats_fve.miss += 1;
sub += "1;";
} else {
} else
{
sub += "0;";
}

if (varVal_o >= threshold && varVal_m >= threshold) {
if (varVal_o >= threshold && varVal_m >= threshold)
{
stats_fve.cn += 1;
sub += "1";
} else {
} else
{
sub += "0";
}
stats_fve.sub_data.push(sub);
Expand Down
Loading

0 comments on commit f7ea0ed

Please sign in to comment.