Skip to content

Commit

Permalink
feat: pino logs
Browse files Browse the repository at this point in the history
  • Loading branch information
@sanoel
sanoel committed Oct 3, 2024
1 parent 2958a51 commit 72912d9
Show file tree
Hide file tree
Showing 6 changed files with 116 additions and 192 deletions.
9 changes: 5 additions & 4 deletions package.json
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@qlever-llc/lf-sync",
"version": "3.1.3",
"version": "3.1.5",
"description": "Trellis LaserFiche sync service",
"main": "dist/index.js",
"type": "module",
Expand Down Expand Up @@ -57,11 +57,11 @@
"license": "Apache-2.0",
"dependencies": {
"@oada/client": "^5.1.0",
"@oada/jobs": "^4.6.1",
"@oada/jobs": "^4.7.0",
"@oada/lib-config": "^3.9.1",
"@oada/lib-prom": "^3.8.0",
"@oada/list-lib": "^6.0.2",
"@oada/pino-debug": "^3.10.0",
"@oada/pino-debug": "^4.0.1",
"@oada/types": "^3.5.3",
"bs58": "^6.0.0",
"convict": "^6.2.4",
Expand Down Expand Up @@ -130,7 +130,8 @@
"update-browserslist-db": "^1.1.0"
},
"resolutions": {
"xlsx": "https://cdn.sheetjs.com/xlsx-0.20.2/xlsx-0.20.2.tgz"
"xlsx": "https://cdn.sheetjs.com/xlsx-0.20.2/xlsx-0.20.2.tgz",
"@oada/list-lib": "portal:/home/sanoel/list-lib"
},
"packageManager": "[email protected]",
"volta": {
Expand Down
5 changes: 1 addition & 4 deletions src/cws/api.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,15 +75,13 @@ async function getToken() {
*/
export const cws = client.extend({
headers: { Authorization: authToken ?? (await getToken()) },
/*
hooks: {
beforeRequest: [
(options: any) => {
options.headers.Authorization = token
}
],
beforeError: [ "script:move-entry": "node dist/scripts/lf/move-entry.js",
beforeError: [
async (error: any) => {
const { response } = error;
if (response && response.statusCode === 401) {
Expand All @@ -101,7 +99,6 @@ export const cws = client.extend({
}
]
}
*/

});

Expand Down
44 changes: 21 additions & 23 deletions src/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import esMain from 'es-main';
import ksuid from 'ksuid';
import makeDebug from 'debug';
import pTimeout from 'p-timeout';
import { type Logger, pino } from '@oada/pino-debug'

import {
AssumeState,
Expand Down Expand Up @@ -57,10 +58,7 @@ import { browse, getEntryId, moveEntry, retrieveEntry } from './cws/index.js';
import { sync } from './sync.js';

const selfChange = new JsonPointer('/body/_meta/services/lf-sync');

const trace = makeDebug('lf-sync:trace');
const info = makeDebug('lf-sync:info');
const warn = makeDebug('lf-sync:warn');
const log = pino({base: {service: 'lf-sync'}});

// Stuff from config
const { token, domain } = config.get('oada');
Expand All @@ -79,8 +77,8 @@ const REPORT_PATH = '/bookmarks/services/lf-sync/jobs/reports/docs-synced';
let oada: OADAClient;

async function startService() {
info('Service: lf-sync');
info(`Version: ${process.env.npm_package_version}`);
log.info('Service: lf-sync');
log.info(`Version: ${process.env.npm_package_version}`);

// Connect to the OADA API
const conn = oada
Expand Down Expand Up @@ -127,19 +125,19 @@ export function watchLaserfiche(

for await (const [id, startTime] of workQueue.entries()) {
if (start.getTime() > startTime + SYNC_JOB_TIMEOUT) {
warn(`LF ${id} work never completed. moved to _NeedsReview`);
log.warn(`LF ${id} work never completed. moved to _NeedsReview`);
await moveEntry(id as EntryId, '/_NeedsReview');
workQueue.delete(id);
}
}

trace(`${start.toISOString()} Polling LaserFiche.`);
log.trace(`${start.toISOString()} Polling LaserFiche.`);

const files = await backOff(async () => browse(LF_AUTOMATION_FOLDER));
for await (const file of files) {
if (!workQueue.has(file.EntryId)) {
if (file.Type !== 'Document') {
info(`LF ${file.EntryId} not a document. Moved to _NeedsReview.`);
log.info(`LF ${file.EntryId} not a document. Moved to _NeedsReview.`);
await moveEntry(file, '/_NeedsReview');

continue;
Expand Down Expand Up @@ -204,7 +202,7 @@ const getLfEntry: WorkerFunction = async function (
return entriesFromMeta(data);
}

info('Missing LF Entries for vdocs, waiting for remainder');
log.info('Missing LF Entries for vdocs, waiting for remainder');
return waitForLfEntries(conn, doc, meta);
};

Expand All @@ -227,7 +225,7 @@ async function waitForLfEntries(
async function watchChanges() {
for await (const change of changes) {
if (selfChange.has(change)) {
info(
log.info(
`Got a change containing a meta entry for one of the vdocs: ${path}`,
selfChange.get(change),
);
Expand All @@ -236,7 +234,7 @@ async function waitForLfEntries(
...(selfChange.get(change) as Record<string, LfMetaEntry>),
};
if (Object.keys(meta.vdoc.pdf).every((key) => data[key])) {
info(
log.info(
`Got a meta entries for every vdoc of ${path}. Fetching entries`,
);
await unwatch();
Expand Down Expand Up @@ -336,7 +334,7 @@ function watchPartnerDocs(
conn: OADAClient,
callback: (item: Resource, tpKey: string) => void | PromiseLike<void>,
) {
info('Monitoring %s for new/current partners', TRADING_PARTNER_LIST);
log.info('Monitoring %s for new/current partners', TRADING_PARTNER_LIST);
// TODO: Update these to new ListWatch v4 API
const watch = new ListWatch({
conn,
Expand All @@ -349,7 +347,7 @@ function watchPartnerDocs(
ChangeType.ItemAdded,
async ({ pointer: tpKey }: { pointer: string }) => {
const documentPath = join(TRADING_PARTNER_LIST, tpKey, DOCS_LIST);
info('Monitoring %s for new/current document types', documentPath);
log.info('Monitoring %s for new/current document types', documentPath);
const docTypeWatch = new ListWatch({
conn,
resume: false, // No oada-list-lib entry in the _meta doc! May have previously, but is no more!
Expand All @@ -365,7 +363,7 @@ function watchPartnerDocs(

// If (!type.toLowerCase().includes('tickets')) return;
// FIXME: Remove this before production
info('Monitoring %s for new documents of type %s', path, type);
log.info('Monitoring %s for new documents of type %s', path, type);
const docWatch = new ListWatch({
conn,
name: `lf-sync:to-lf:${tpKey.replace('/', '')}:${type.replace('/', '')}`,
Expand All @@ -391,11 +389,11 @@ function watchPartnerDocs(
pointer: string;
}) => {
if (selfChange.has(change)) {
trace('Ignoring self made change to resource.');
log.trace('Ignoring self made change to resource.');
return;
}

trace(`Got work (change): ${join(path, documentKey)}`);
log.trace(`Got work (change): ${join(path, documentKey)}`);

// Fetch resource
const data = await oada.get({
Expand Down Expand Up @@ -429,7 +427,7 @@ function watchSelfDocs(
path: DOCS_LIST,
tree: selfDocsTree,
});
trace(`Monitoring ${DOCS_LIST} for new/current document types`);
log.trace(`Monitoring ${DOCS_LIST} for new/current document types`);
docTypeWatch.on(
ChangeType.ItemAdded,
async ({ pointer: type }: { pointer: string }) => {
Expand All @@ -444,7 +442,7 @@ function watchSelfDocs(
assertItem: assertResource,
tree,
});
trace(`Monitoring ${path} for new documents`);
log.trace(`Monitoring ${path} for new documents`);
docWatch.on(
ChangeType.ItemAdded,
async ({
Expand All @@ -454,7 +452,7 @@ function watchSelfDocs(
item: Promise<Resource>;
pointer: string;
}) => {
trace(`Got work (new): ${join(path, documentKey)}`);
log.trace(`Got work (new): ${join(path, documentKey)}`);
await callback(await item);
},
);
Expand All @@ -468,11 +466,11 @@ function watchSelfDocs(
pointer: string;
}) => {
if (selfChange.has(change)) {
trace('Ignoring self made change to resource.');
log.trace('Ignoring self made change to resource.');
return;
}

trace(`Got work (change): ${join(path, documentKey)}`);
log.trace(`Got work (change): ${join(path, documentKey)}`);

// Fetch resource
const data = await oada.get({
Expand Down Expand Up @@ -504,7 +502,7 @@ export async function reportItem(conn: OADAClient, item: ReportEntry) {
},
tree,
});
info(`Reported item to ${path}/${key}`);
log.info(`Reported item to ${path}/${key}`);
}

interface ReportEntry {
Expand Down
31 changes: 14 additions & 17 deletions src/sync.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,11 @@
import { config } from './config.js';

import '@oada/pino-debug';
import type { Logger } from '@oada/pino-debug';

import { extname, join } from 'node:path';

import equal from 'deep-equal';
import makeDebug from 'debug';

import { type Job, type Json, type WorkerFunction } from '@oada/jobs';
import { type OADAClient } from '@oada/client';
Expand All @@ -47,11 +47,6 @@ import {
import type { LfSyncMetaData, Metadata } from './utils.js';
import { transform } from './transformers/index.js';

const trace = makeDebug('lf-sync:trace');
const info = makeDebug('lf-sync:info');
// Const warn = makeDebug('lf-sync:warn');
const error = makeDebug('lf-sync:error');

// Stuff from config
const incomingFolder = join(
'/',
Expand All @@ -67,8 +62,10 @@ export const sync: WorkerFunction = async function (
job: Job,
{
oada: conn,
log,
}: {
oada: OADAClient;
log: Logger;
},
): Promise<Json> {
// Keeping deprecating tpKey
Expand All @@ -79,7 +76,7 @@ export const sync: WorkerFunction = async function (
})) as unknown as any;
const fieldList = await transform(document);

trace('Fetching vdocs for %s', document._id);
log.trace('Fetching vdocs for %s', document._id);
const vdocs = await getPdfVdocs(conn, document);
if (!(tradingPartner || tpKey))
throw new Error('No trading partner key or id provided');
Expand Down Expand Up @@ -117,7 +114,7 @@ export const sync: WorkerFunction = async function (

fieldList['Original Filename'] = await fetchVdocFilename(conn, value._id);

const syncMetadata = await fetchSyncMetadata(conn, document._id, key);
const syncMetadata = await fetchSyncMetadata(conn, document._id, key, log);
const syncMetaCopy = { ...syncMetadata };
let currentFields: LfSyncMetaData['fields'] = {};

Expand Down Expand Up @@ -148,7 +145,7 @@ export const sync: WorkerFunction = async function (

// Aka, an empty object
if (Object.keys(syncMetadata.fields).length === 0) {
trace(`Document vdoc ${key} has no data yet. Skipping.`);
log.trace(`Document vdoc ${key} has no data yet. Skipping.`);
continue;
}

Expand All @@ -157,7 +154,7 @@ export const sync: WorkerFunction = async function (

// Upsert into LF
if (syncMetadata.LaserficheEntryID) {
info(
log.info(
`LF Entry ${syncMetadata.LaserficheEntryID} (vdoc ${key}) already exists. Updating.`,
);
await setMetadata(
Expand All @@ -166,16 +163,16 @@ export const sync: WorkerFunction = async function (
syncMetadata.fields['Document Type'],
);

trace(`Moving the LF document to ${path}`);
log.trace(`Moving the LF document to ${path}`);
// Use our own filing workflow instead of incomingFolder
await moveEntry(syncMetadata.LaserficheEntryID, path as `/{string}`);

// New to LF
} else {
info(`Document (vdoc ${key}) is new to LF`);
log.info(`Document (vdoc ${key}) is new to LF`);

const { buffer, mimetype } = await getBuffer(conn, value);
trace('Uploading document to Laserfiche');
const { buffer, mimetype } = await getBuffer(conn, value, log);
log.trace('Uploading document to Laserfiche');
const lfDocument = await createDocument({
//name: `${document._id}-${key}.${extname(syncMetadata.fields['Original Filename'] ?? '').slice(1)}`,
name: filename,
Expand All @@ -186,7 +183,7 @@ export const sync: WorkerFunction = async function (
buffer,
});

info(
log.info(
`Created LF document ${lfDocument.LaserficheEntryID} (vdoc ${key})`,
);
syncMetadata.LaserficheEntryID = lfDocument.LaserficheEntryID;
Expand All @@ -208,7 +205,7 @@ export const sync: WorkerFunction = async function (
});
*/

trace('Recording lf-sync metadata to Trellis document');
log.trace('Recording lf-sync metadata to Trellis document');

// Update the sync metadata in Trellis only if it has actually changed
if (!equal(syncMetaCopy, syncMetadata)) {
Expand All @@ -221,7 +218,7 @@ export const sync: WorkerFunction = async function (

return docsSyncMetadata as unknown as Json;
} catch (error_: unknown) {
error(error_, `Could not sync document ${doc._id}.`);
log.error(error_, `Could not sync document ${doc._id}.`);
throw error_;
}
};
Loading

0 comments on commit 72912d9

Please sign in to comment.