From db7e4095b6378d593914ac5f57cef1008b84460b Mon Sep 17 00:00:00 2001 From: Ivo von Putzer Reibegg Date: Sat, 14 Oct 2023 07:23:40 +0200 Subject: [PATCH] cleanup --- lib/date.mjs | 22 +++++------ lib/jsonl.mjs | 96 +++++++++++++++++++--------------------------- test/lib/date.mjs | 91 ++++++++++++++++++++++++++++++++----------- test/lib/jsonl.mjs | 56 +++++++++++++++------------ 4 files changed, 150 insertions(+), 115 deletions(-) diff --git a/lib/date.mjs b/lib/date.mjs index d1a4a36..67faf75 100644 --- a/lib/date.mjs +++ b/lib/date.mjs @@ -64,20 +64,18 @@ export function jsonDateReviver (_, value, iso8601Regex = /^\d{4}-\d{2}-\d{2}T\d : value } -export function csvDateReplacer (_, value) { - return Array.isArray(value) - ? value.map(replaceIfDate) - : replaceIfDate(value) +export function noopDateReplacer (_, value) { + return value +} - function replaceIfDate (date) { - return date instanceof Date - ? date.toISOString().slice(0, 16).replace('T', ' ') - : date - } +export function isoDateReplacer (_, value) { + return !isNaN(Date.parse(value)) && value === new Date(value).toISOString() + ? value?.substring(0, 10) + : value } -export function dateReplacerFor (interval) { +export function shortDateFor (interval) { return /\d+[dwmy]/.test(interval) - ? (_, value) => value === new Date(value).toJSON() ? new Date(value).toJSON().substring(0, 10) : value - : (_, value) => value + ? isoDateReplacer + : noopDateReplacer } diff --git a/lib/jsonl.mjs b/lib/jsonl.mjs index 97da77d..fc4693f 100644 --- a/lib/jsonl.mjs +++ b/lib/jsonl.mjs @@ -1,23 +1,19 @@ import os from 'node:os' import readline from 'node:readline' -import stream, { PassThrough } from 'node:stream' +import stream from 'node:stream' -import { jsonDateReviver, csvDateReplacer } from './date.mjs' +import { jsonDateReviver, noopDateReplacer } from './date.mjs' -/* - createReadSteam(file) - .pipe(parseJsonl()) - -> - .pipe( - groupBy(line => line[0].getFullYear()) - -> un readable stream per ogni anno: 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023 - ) - .pipe( - flatMap(groupReadableStream => groupReadableStream.pipe(writeJsonl())) - ) +export function parseJsonl ({ objectMode, reviver } = { objectMode: true, reviver: jsonDateReviver }, { createInterface } = readline, { Duplex } = stream) { + return new Duplex({ objectMode, read () {}, write (_0, _1, next) { next(null) } }) + .on('pipe', function (input) { + createInterface({ input, terminal: false }) + .on('line', (line) => this.push(objectMode ? JSON.parse(line, reviver) : line)) + .on('close', () => this.push(null)) + }) +} -*/ -export function groupBy (groupFn, options = { objectMode: true }, { Transform } = stream) { +export function groupBy (groupFn, options = { objectMode: true }, { Transform, PassThrough } = stream) { const groups = new Map() return new Transform({ ...options, @@ -36,16 +32,7 @@ export function groupBy (groupFn, options = { objectMode: true }, { Transform } }) } -export function parseJsonl ({ objectMode, reviver } = { objectMode: true, reviver: jsonDateReviver }, { createInterface } = readline, { Duplex } = stream) { - return new Duplex({ objectMode, read () {}, write (_0, _1, next) { next(null) } }) - .on('pipe', function (input) { - createInterface({ input, terminal: false }) - .on('line', (line) => this.push(objectMode ? JSON.parse(line, reviver) : line)) - .on('close', () => this.push(null)) - }) -} - -export function writeJsonl ({ objectMode, replacer } = { objectMode: true, replacer: (key, value) => value }, { createInterface } = readline, { Transform } = stream, { EOL } = os) { +export function writeJsonl ({ objectMode, replacer } = { objectMode: true, replacer: noopDateReplacer }, { createInterface } = readline, { Transform } = stream, { EOL } = os) { return new Transform({ objectMode, transform (line, _, next) { @@ -54,7 +41,7 @@ export function writeJsonl ({ objectMode, replacer } = { objectMode: true, repla }) } -export function writeJson ({ objectMode, replacer } = { objectMode: true, replacer: (_, value) => value }, { Transform } = stream, { EOL } = os) { +export function writeJson ({ objectMode, replacer } = { objectMode: true, replacer: noopDateReplacer }, { Transform } = stream, { EOL } = os) { let isFirstLine = true return new Transform({ objectMode: true, @@ -71,8 +58,8 @@ export function writeJson ({ objectMode, replacer } = { objectMode: true, replac } }) } -// .pipe( writeCsv(['date', 'open, 'high', 'low, 'close', 'volume', 'metadata']) ) -export function writeCsv (headers = null, { objectMode, replacer } = { objectMode: true, replacer: csvDateReplacer }, { Transform } = stream, { EOL } = os) { + +export function writeCsv (headers = null, { objectMode, replacer } = { objectMode: true, replacer: noopDateReplacer }, { Transform } = stream, { EOL } = os) { let isFirstLine = true return new Transform({ objectMode: true, @@ -94,32 +81,29 @@ export function writeCsv (headers = null, { objectMode, replacer } = { objectMod }) } -// export function writeXml (headers, elementName = 'candle', { objectMode, replacer } = { objectMode: true, replacer: csvDateReplacer }, { Transform } = stream, { EOL } = os) { -// const isFirstLine = true -// return new Transform({ -// objectMode: true, -// transform (line, _, next) { -// if (isFirstLine) { -// isFirstLine = false -// this.push(`${EOL}`) -// this.push(`${EOL}`) -// this.push(`<${elementName}s>${EOL}`) -// const [date, open, high, low, close, volume] = line -// return `\t${EOL}\t\t${date.toJSON().substr(0, 10)}${EOL}\t\t${open}${EOL}\t\t${high}${EOL}\t\t${low}${EOL}\t\t${close}${EOL}\t\t${volume}${EOL}\t${EOL}` - -// // ${(headers ?? Object.keys(line)).join(',')}${EOL}`) -// // // return next(null) -// } else { -// // // this.push(`${objectMode ? JSON.stringify(Object.values(line), replacer).slice(1, -1).replaceAll('"', '') : line}${EOL}`) -// // // next(null) -// } -// }, -// flush (next) { -// next(null, `${EOL}`) -// } -// }) +export function writeXml (documentRoot = 'candle', { objectMode, replacer } = { objectMode: true, replacer: noopDateReplacer }, { Stream } = stream, { EOL } = os) { + throw new Error('not implemented') + // const isFirstLine = true + // return new Transform({ + // objectMode: true, + // transform (line, _, next) { + // if (isFirstLine) { + // isFirstLine = false + // this.push(`${EOL}`) + // this.push(`${EOL}`) + // this.push(`<${elementName}s>${EOL}`) + // const [date, open, high, low, close, volume] = line + // return `\t${EOL}\t\t${date.toJSON().substr(0, 10)}${EOL}\t\t${open}${EOL}\t\t${high}${EOL}\t\t${low}${EOL}\t\t${close}${EOL}\t\t${volume}${EOL}\t${EOL}` -// function toXML () { - -// } -// } + // // ${(headers ?? Object.keys(line)).join(',')}${EOL}`) + // // // return next(null) + // } else { + // // // this.push(`${objectMode ? JSON.stringify(Object.values(line), replacer).slice(1, -1).replaceAll('"', '') : line}${EOL}`) + // // // next(null) + // } + // }, + // flush (next) { + // next(null, `${EOL}`) + // } + // }) +} diff --git a/test/lib/date.mjs b/test/lib/date.mjs index 1c2b61c..d1e0613 100644 --- a/test/lib/date.mjs +++ b/test/lib/date.mjs @@ -1,7 +1,13 @@ import { describe, it } from 'node:test' -import { ok, equal } from 'node:assert/strict' +import { ok, equal, deepEqual } from 'node:assert/strict' -import { INTERVALS, parseInterval, daysBetween, jsonDateReviver, utcDate } from '../../lib/date.mjs' +import { + INTERVALS, + parseInterval, + daysBetween, utcDate, + jsonDateReviver, + noopDateReplacer, isoDateReplacer, shortDateFor +} from '../../lib/date.mjs' describe('lib/date', () => { describe('.INTERVALS', () => { @@ -24,20 +30,6 @@ describe('lib/date', () => { ok(INTERVALS.has('1')) }) }) - describe('.daysBetween', () => { - it('is callable', () => { - equal(typeof daysBetween, 'function') - }) - it('returns a number', () => { - equal(typeof daysBetween(new Date(), new Date()), 'number') - }) - it('returns the number of days between two dates', () => { - equal(daysBetween(new Date('2021-09-26'), new Date('2021-09-27')), 1) - }) - it('returns the number of days between two dates with iso8601 format', () => { - equal(daysBetween(new Date('2021-09-26T10:30:00.000Z'), new Date('2021-09-27T10:30:00.000Z')), 1) - }) - }) describe('.parseInterval', () => { it('is callable', () => { equal(typeof parseInterval, 'function') @@ -76,6 +68,32 @@ describe('lib/date', () => { equal(parseInterval('2'), INTERVALS.get('2')) }) }) + describe('.daysBetween', () => { + it('is callable', () => { + equal(typeof daysBetween, 'function') + }) + it('returns a number', () => { + equal(typeof daysBetween(new Date(), new Date()), 'number') + }) + it('returns the number of days between two dates', () => { + equal(daysBetween(new Date('2021-09-26'), new Date('2021-09-27')), 1) + }) + it('returns the number of days between two dates with iso8601 format', () => { + equal(daysBetween(new Date('2021-09-26T10:30:00.000Z'), new Date('2021-09-27T10:30:00.000Z')), 1) + }) + }) + describe('.utcDate', () => { + it('is callable', () => { + equal(typeof utcDate, 'function') + }) + it('returns a Date', () => { + ok(utcDate() instanceof Date) + }) + it('returns a Date that is adjusted to .getTimezoneOffset ', () => { + const now = new Date() + equal(utcDate(now).toJSON(), new Date(now.getTime() + (now.getTimezoneOffset() * 6e4)).toJSON()) + }) + }) describe('.jsonDateReviver', () => { it('is callable', () => { equal(typeof jsonDateReviver, 'function') @@ -96,16 +114,43 @@ describe('lib/date', () => { ok(jsonDateReviver(null, '2023/09/26 10:30:00', /^\d{4}\/\d{2}\/\d{2} \d{2}:\d{2}:\d{2}$/) instanceof Date) // Custom format regex }) }) - describe('.utcDate', () => { + describe('.noopDateReplacer', () => { it('is callable', () => { - equal(typeof utcDate, 'function') + ok(noopDateReplacer instanceof Function) }) - it('returns a Date', () => { - ok(utcDate() instanceof Date) + it('returns without applying any replacement', () => { + deepEqual(noopDateReplacer('key', 'value'), 'value') }) - it('returns a Date that is adjusted to .getTimezoneOffset ', () => { - const now = new Date() - equal(utcDate(now).toJSON(), new Date(now.getTime() + (now.getTimezoneOffset() * 6e4)).toJSON()) + }) + describe('.isoDateReplacer', () => { + it('is callable', () => { + ok(isoDateReplacer instanceof Function) + }) + it('shortens iso dates down to its date value', () => { + const aJsonDateString = new Date().toISOString() + equal(isoDateReplacer(null, aJsonDateString), aJsonDateString.slice(0, 10)) + }) + it('returns the original value otherwise', () => { + equal(isoDateReplacer(null, 42), 42) + }) + it('is compatible with JSON.stringify', () => { + const aDate = new Date() // note: to my understanding .toJSON get's invoked prior calling the replacer function + equal(JSON.stringify(aDate, isoDateReplacer), JSON.stringify(aDate.toISOString().slice(0, 10))) + }) + }) + describe('.shortDateFor', () => { + it('is callable', () => { + ok(shortDateFor instanceof Function) + }) + it('returns noopDateReplacer intervals that require hours minutes and seconds', () => { + equal(shortDateFor('1'), noopDateReplacer) + equal(shortDateFor('2h'), noopDateReplacer) + }) + it('returns isoDateReplacer for date intervals that are daily timeframe and above', () => { + equal(shortDateFor('1d'), isoDateReplacer) + equal(shortDateFor('2w'), isoDateReplacer) + equal(shortDateFor('3m'), isoDateReplacer) + equal(shortDateFor('4y'), isoDateReplacer) }) }) }) diff --git a/test/lib/jsonl.mjs b/test/lib/jsonl.mjs index a4049dc..cfd89fb 100644 --- a/test/lib/jsonl.mjs +++ b/test/lib/jsonl.mjs @@ -1,8 +1,8 @@ import { describe, it } from 'node:test' -import { ok, equal, deepEqual } from 'node:assert/strict' +import { throws, ok, equal, deepEqual } from 'node:assert/strict' import { Duplex, Readable, Transform } from 'node:stream' -import { groupBy, parseJsonl, writeJsonl, writeJson, writeCsv } from '../../lib/jsonl.mjs' +import { parseJsonl, groupBy, writeJsonl, writeJson, writeCsv, writeXml } from '../../lib/jsonl.mjs' import { EOL } from 'node:os' import { fromAsync } from '../../lib/async.mjs' @@ -70,6 +70,31 @@ describe('lib/jsonl', () => { it.todo('handles object mode true -> and converts to objects') it.todo('handles object mode false -> splits by line but keeps buffers|strings and does not parse nor rewrite dates') }) + describe('.groupBy', () => { + it('is callable', () => { + equal(typeof groupBy, 'function') + }) + it('returns a stream.Transform', () => { + ok(groupBy() instanceof Transform) + }) + it('groups by a given function', async () => { + const aByYearPredicate = ([date]) => date.getUTCFullYear() + const groups = groupBy(aByYearPredicate) + + createReadableObjectsWith( + [new Date(Date.UTC(2022, 1, 1)), 'foo'], + [new Date(Date.UTC(2023, 1, 1)), 'bar'], + [new Date(Date.UTC(2023, 2, 1)), 'baz'] + ).pipe(groups) + + const groupStreams = [] + for await (const group of groups) { + groupStreams.push(group) + } + + equal(groupStreams.length, 2) + }) + }) describe('.writeJsonl', () => { it('is callable', () => { equal(typeof writeJsonl, 'function') @@ -157,30 +182,13 @@ describe('lib/jsonl', () => { deepEqual(await fromAsync(await csvLines), ['xyz' + EOL, 'bar' + EOL, 'baz' + EOL]) }) }) - describe.todo('.writeXml') - describe('.groupBy', () => { + + describe('.writeXml', () => { it('is callable', () => { - equal(typeof groupBy, 'function') - }) - it('returns a stream.Transform', () => { - ok(groupBy() instanceof Transform) + ok(writeXml instanceof Function) }) - it('groups by a given function', async () => { - const aByYearPredicate = ([date]) => date.getUTCFullYear() - const groups = groupBy(aByYearPredicate) - - createReadableObjectsWith( - [new Date(Date.UTC(2022, 1, 1)), 'foo'], - [new Date(Date.UTC(2023, 1, 1)), 'bar'], - [new Date(Date.UTC(2023, 2, 1)), 'baz'] - ).pipe(groups) - - const groupStreams = [] - for await (const group of groups) { - groupStreams.push(group) - } - - equal(groupStreams.length, 2) + it('returns a stream', () => { + throws(() => writeXml()) }) }) })