diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index ce74036..30cef85 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -14,6 +14,9 @@ permissions: concurrency: group: "pages" cancel-in-progress: false +env: + TZ: UTC + LANG: en_US.UTF-8 jobs: build: runs-on: ubuntu-latest diff --git a/bin/sync-coinbase.mjs b/bin/sync-coinbase.mjs index cd7b4ce..9a8a4a9 100644 --- a/bin/sync-coinbase.mjs +++ b/bin/sync-coinbase.mjs @@ -10,13 +10,13 @@ import { EOL } from 'node:os' import { readCacheBy, readLastCachedJsonLineOf } from '../lib/cache.mjs' import { fetchCandlesSince, coinbaseIntervalFor, coinbaseIdFor } from '../lib/coinbase.mjs' import { dateReviver } from '../lib/json.mjs' -import { daysBetween, INTERVALS } from '../lib/date.mjs' +import { utcDate, daysBetween, INTERVALS } from '../lib/date.mjs' -// move this -export function intervalFor(file) { - const [,,interval] = file.replace('.', ',').split(',') - return interval -} +console.log('[bin/sync-coinbase] toLocaleTimeString', new Date().toLocaleTimeString()) +console.log('[bin/sync-coinbase] toJSON', new Date().toJSON()) +console.log('[bin/sync-coinbase] toISOString', new Date().toISOString()) +console.log('[bin/sync-coinbase] getTimezoneOffset', new Date().getTimezoneOffset()) +console.log('[bin/sync-coinbase] utcDate(new Date())', utcDate(new Date())) for (const filePath of await readCacheBy(name => name.startsWith('coinbase,'))) { console.log('[bin/sync-coinbase] Loading:%s', filePath) @@ -25,14 +25,19 @@ for (const filePath of await readCacheBy(name => name.startsWith('coinbase,'))) console.log('↳ interval:', intervalFor(filePath)) console.log('↳ coinbaseId:', coinbaseIdFor(filePath)) console.log('↳ coinbaseInterval:', coinbaseIntervalFor(filePath)) - const nextUncachedCandleDate = new Date(lastCachedCandleDate.getTime() + INTERVALS.get(intervalFor(filePath))) - const numberOfCandlesToSync = daysBetween(nextUncachedCandleDate, new Date(Date.now() - INTERVALS.get(intervalFor(filePath)))) + const lastCachableCandleDate = new Date(utcDate(new Date()).getTime() - INTERVALS.get(intervalFor(filePath))) + const numberOfCandlesToSync = daysBetween(nextUncachedCandleDate, lastCachableCandleDate) console.log('↳ numberOfCandlesToSync:', numberOfCandlesToSync) if (numberOfCandlesToSync === 0) continue - const stream = createWriteStream(filePath, { flags: 'a' }) + const stream = createWriteStream(filePath, { flags: 'a' }) // append for await (const [date, open, high, low, close, volume] of fetchCandlesSince(nextUncachedCandleDate, coinbaseIdFor(filePath), coinbaseIntervalFor(filePath))) { stream.write(JSON.stringify([date, open, high, low, close, volume]) + EOL) } stream.close() } + +function intervalFor(file) { + const [,,interval] = file.replace('.', ',').split(',') + return interval +} diff --git a/lib/coinbase.mjs b/lib/coinbase.mjs index 4fe43a4..0535f22 100644 --- a/lib/coinbase.mjs +++ b/lib/coinbase.mjs @@ -4,7 +4,9 @@ import { env } from 'node:process' import fs from 'node:fs/promises' import path from 'node:path' import readline from 'node:readline' +import {utcDate} from './date.mjs' +// @deprecated this is a duplicate of lib/json.mjs export function iso8601DateReviver (_, value, iso8601Regex = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/) { return typeof value === 'string' && iso8601Regex.test(value) ? new Date(value) @@ -26,7 +28,7 @@ export function withJsonBody (data) { export function fetchCoinbase (input,init = {method: 'GET',headers: {'Content-Type': 'application/json'}},{env: {npm_config_coinbase_api_key: key,npm_config_coinbase_api_secret: secret,npm_config_coinbase_api_base: base}} = process) { const url = new URL(input, base) // base is set right at https://api.coinbase.com/api/v3/ for easy calls ie. coinbase.fetch('brokerage/products') - init.headers['CB-ACCESS-TIMESTAMP'] = Math.floor(1e-3 * Date.now()) + init.headers['CB-ACCESS-TIMESTAMP'] = Math.floor(1e-3 * Date.now()) // todo: refactor to toUnixTimestamp(Date.now()) init.headers['CB-ACCESS-KEY'] = key init.headers['CB-ACCESS-SIGN'] = createHmac('sha256', secret).update(init.headers['CB-ACCESS-TIMESTAMP'] + init.method.toUpperCase() + url.pathname + (init.body || String.prototype)).digest('hex') @@ -55,7 +57,7 @@ export async function * fetchCandlesSince (start, id, size = 'ONE_DAY') { ['SIX_HOUR', 21600000], ['ONE_DAY', 86400000] ]) - const yesterday = new Date(new Date().setTime(new Date().getTime() - granularity.get(size))) + const yesterday = new Date(utcDate(new Date()).getTime() - granularity.get(size)) do { const end = new Date( Math.min( diff --git a/lib/date.mjs b/lib/date.mjs index 2a570cb..9c47c64 100644 --- a/lib/date.mjs +++ b/lib/date.mjs @@ -54,6 +54,11 @@ export function daysBetween (date1, date2, oneDay = INTERVALS.get('1d')) { ) } +export function utcDate(now = new Date()) { + return new Date(now.getTime() + (now.getTimezoneOffset() * 6e4)) +} + +// @deprecated this is a duplicate of lib/json.mjs export function jsonDateReviver (_, value, iso8601Regex = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/) { return typeof value === 'string' && iso8601Regex.test(value) ? new Date(value) diff --git a/test/lib/cache.mjs b/test/lib/cache.mjs new file mode 100644 index 0000000..6d54727 --- /dev/null +++ b/test/lib/cache.mjs @@ -0,0 +1,23 @@ +import { describe, it } from 'node:test' +import { ok, strictEqual } from 'assert' + +describe('lib/cache', () => {}) + +// async function readCoinbaseCache(startsWith = 'coinbase,', directory = 'cache', { readdir } = fs, { join } = path) { +// return ( await readdir(directory) ) +// .filter(file => file.startsWith(startsWith)) +// .map(file => { +// const [exchange, id, rest] = file.split(',') +// const [interval, extension] = rest.split('.') +// return { exchange, id, interval, extension, path: join(directory, file) } +// }) +// } + + + +// describe('lib/cache', () => { +// it('returns an array of objects', async () => { +// const cache = await readCoinbaseCache('coinbase,', '../cache') +// console.log(cache) +// }) +// }) diff --git a/test/lib/date.mjs b/test/lib/date.mjs index 18d8854..082b015 100644 --- a/test/lib/date.mjs +++ b/test/lib/date.mjs @@ -1,17 +1,86 @@ import { describe, it } from 'node:test' import { ok, strictEqual } from 'node:assert' -import { INTERVALS, parseInterval, daysBetween, jsonDateReviver } from '../../lib/date.mjs' +import { INTERVALS, parseInterval, daysBetween, jsonDateReviver, utcDate } from '../../lib/date.mjs' import { dateReviver } from '../../lib/json.mjs' describe('lib/date', () => { describe('.INTERVALS', () => { + it('is a Map', () => { + ok(INTERVALS instanceof Map) + }) + it('has a key of "1y" for years', () => { + ok(INTERVALS.has('1y')) + }) + it('has a key of "1m" for months', () => { + ok(INTERVALS.has('1m')) + }) + it('has a key of "1d" for days', () => { + ok(INTERVALS.has('1d')) + }) + it('has a key of "1h" for hours', () => { + ok(INTERVALS.has('1h')) + }) + it('has a key of "1" for minutes', () => { + ok(INTERVALS.has('1')) + }) + }) + describe('.daysBetween', () => { + it('is callable', () => { + strictEqual(typeof daysBetween, 'function') + }) + it('returns a number', () => { + strictEqual(typeof daysBetween(new Date(), new Date()), 'number') + }) + it('returns the number of days between two dates', () => { + strictEqual(daysBetween(new Date('2021-09-26'), new Date('2021-09-27')), 1) + }) + it('returns the number of days between two dates with iso8601 format', () => { + strictEqual(daysBetween(new Date('2021-09-26T10:30:00.000Z'), new Date('2021-09-27T10:30:00.000Z')), 1) + }) + }) + describe.todo('.parseInterval', () => { + it('is callable', () => { + strictEqual(typeof parseInterval, 'function') + }) + it('returns a number', () => { + strictEqual(typeof parseInterval('1y'), 'number') + }) + it('handles years|y, months|m, weeks|w, days|d, hours|h, and minutes (without any label)', () => { + strictEqual(parseInterval('1y'), 31536000000) + strictEqual(parseInterval('1m'), 2592000000) + strictEqual(parseInterval('1w'), 604800000) + strictEqual(parseInterval('1d'), 86400000) + strictEqual(parseInterval('1h'), 3600000) + strictEqual(parseInterval('1'), 60000) + }) + it('handles multiples of any given interval', () => { + strictEqual(parseInterval('2y'), 63072000000) + strictEqual(parseInterval('3m'), 7776000000) + strictEqual(parseInterval('2w'), 1209600000) + strictEqual(parseInterval('3d'), 259200000) + strictEqual(parseInterval('2h'), 7200000) + strictEqual(parseInterval('2'), 120000) + }) }) - describe('.parseInterval', () => {}) - describe('.daysBetween', () => {}) describe('.jsonDateReviver (deprecated)', () => { it('is duplicate of ~/lib/json.mjs:dateReviver', () => { strictEqual(jsonDateReviver.toString(), dateReviver.toString().replace(/^function\s+\w+\s*\(/, `function ${jsonDateReviver.name} (`)) }) + it.skip('is not callable anymore', () => { + strictEqual(typeof jsonDateReviver, 'undefined') + }) + }) + describe('.utcDate', () => { + it('is callable', () => { + strictEqual(typeof utcDate, 'function') + }) + it('returns a Date', () => { + ok(utcDate() instanceof Date) + }) + it('returns a Date that is adjusted to .getTimezoneOffset ', () => { + const now = new Date() + strictEqual(utcDate(now).toJSON(), new Date(now.getTime() + (now.getTimezoneOffset() * 6e4)).toJSON()) + }) }) })