diff --git a/package.json b/package.json index f84f4ea..e968432 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@qlever-llc/fl-sync", - "version": "1.4.8", + "version": "1.4.9", "description": "A Trellis microservice to sync Food Logiq data to a Trellis cloud", "main": "dist/index.ts", "scripts": { diff --git a/src/config.ts b/src/config.ts index 52599eb..e4711b3 100644 --- a/src/config.ts +++ b/src/config.ts @@ -148,36 +148,45 @@ const config = convict({ env: 'HANDLE_INCOMPLETE_INTERVAL', arg: 'handleIncompleteInterval', }, - reportInterval: { - doc: 'Time interval for reports to be generated.', - format: Number, - default: 86_400_000, - env: 'REPORT_INTERVAL', - arg: 'reportInterval', - }, - reportEmail: { - doc: 'Email addresses to send reports to', - format: String, - default: null, - env: 'REPORT_EMAIL', - arg: 'reportEmail', - }, - reportCcEmail: { - doc: 'Email address to cc reports to', - format: String, - default: null, - env: 'REPORT_CC_EMAIL', - arg: 'reportCcEmail', - }, - reportReplyToEmail: { - doc: 'reply to email listed for the report', - format: String, - default: null, - env: 'REPORT_REPLYTO_EMAIL', - arg: 'reportReplyToEmail', + reports: { + docFrequency: { + doc: 'CRON frequency for reports to be generated.', + format: String, + default: '0 0 0 * * 1', + env: 'DOC_REPORT_FREQ', + arg: 'docFrequency', + }, + vendorFrequency: { + doc: 'CRON frequency for reports to be generated.', + format: String, + default: '0 0 0 * * 2', + env: 'VENDOR_REPORT_FREQ', + arg: 'vendorFrequency', + }, + email: { + doc: 'Email addresses to send reports to', + format: String, + default: null, + env: 'REPORT_EMAIL', + arg: 'reportEmail', + }, + ccEmail: { + doc: 'Email address to cc reports to', + format: String, + default: null, + env: 'REPORT_CC_EMAIL', + arg: 'reportCcEmail', + }, + replyToEmail: { + doc: 'reply to email listed for the report', + format: String, + default: null, + env: 'REPORT_REPLYTO_EMAIL', + arg: 'reportReplyToEmail', + }, }, }, - foodlogiq: { + 'foodlogiq': { 'interval': { doc: 'polling interval', format: Number, diff --git a/src/index.ts b/src/index.ts index 64ed1f5..61c2cfc 100644 --- a/src/index.ts +++ b/src/index.ts @@ -55,15 +55,15 @@ const JUST_TPS = config.get('trellis.justTps'); const CO_ID = config.get('foodlogiq.community.owner.id'); const COMMUNITY_ID = config.get('foodlogiq.community.id'); const CONCURRENCY = config.get('trellis.concurrency'); -// Const HANDLE_INCOMPLETE_INTERVAL = config.get('trellis.handleIncompleteInterval'); -// const REPORT_INTERVAL = config.get('trellis.handleIncompleteInterval'); const INTERVAL_MS = config.get('foodlogiq.interval') * 1000; // FL polling interval const SERVICE_NAME = config.get('service.name'); const SERVICE_PATH = `/bookmarks/services/${SERVICE_NAME}`; const FL_FORCE_WRITE = config.get('foodlogiq.force_write'); -const REPORT_EMAIL = config.get('trellis.reportEmail'); -const REPORT_CC_EMAIL = config.get('trellis.reportCcEmail'); -const REPORT_REPLYTO_EMAIL = config.get('trellis.reportReplyToEmail'); +const DOC_FREQUENCY = config.get('trellis.reports.docFrequency'); +const VENDOR_FREQUENCY = config.get('trellis.reports.vendorFrequency'); +const REPORT_EMAIL = config.get('trellis.reports.email'); +const REPORT_CC_EMAIL = config.get('trellis.reports.ccEmail'); +const REPORT_REPLYTO_EMAIL = config.get('trellis.reports.replyToEmail'); const services = config.get('services'); const skipQueueOnStartup = config.get('skipQueueOnStartup'); @@ -552,14 +552,14 @@ export async function initialize({ svc.addReport({ name: 'fl-sync-report', reportConfig: docReportConfig, - frequency: `0 0 0 * * *`, + frequency: DOC_FREQUENCY, email: prepEmail, type: 'document-mirrored', }); svc.addReport({ name: 'businesses-report', reportConfig: tpReportConfig, - frequency: `0 0 0 * * 1`, + frequency: VENDOR_FREQUENCY, email: prepTpEmail, type: 'business-lookup', filter: tpReportFilter, diff --git a/src/mirrorWatch.ts b/src/mirrorWatch.ts index d3ab54e..0587969 100644 --- a/src/mirrorWatch.ts +++ b/src/mirrorWatch.ts @@ -192,7 +192,7 @@ async function handleTargetStatus( const docJobId = docJob.oadaId as string; const { bid, masterid, key } = docJob.config; if (docJob && docJob.config['allow-rejection'] === false) { - info(`Target finished with status ${status} on already-approved doc.`); + info(`[job ${docJobId}] Target finished with status ${status} on already-approved doc.`); if (status === 'success') { // If successful, skip the potential pitfalls of assessment creation and call finishDoc. await postUpdate( @@ -259,7 +259,7 @@ async function handleTargetStatus( } info( - `Target job ${targetJob._id} errored. reject: ${reject}; fl-sync job error ${jobError}` + `[job ${docJobId}] Target job ${targetJob._id} errored. reject: ${reject}; fl-sync job error ${jobError}` ); // @ts-expect-error @@ -279,7 +279,7 @@ async function handleTargetStatus( * @param {*} docId */ async function approveFlDocument(documentId: string, jobId: string) { - info(`Approving associated FL Doc ${documentId}`); + info(`[job ${jobId}] Approving associated FL Doc ${documentId}`); try { await axios({ method: 'put', @@ -310,9 +310,9 @@ export const handleAssessmentJob: WorkerFunction = async ( { oada, jobId } ) => { const jobKey = jobId.replace(/^resources\//, ''); - info(`Handling Assessment job ${jobKey}`); try { - const { bid, key } = job.config; + const { bid, key, flDocJobId: docJobId } = job.config; + info(`[job ${docJobId}] Handling incoming Assessment job ${jobKey}`); const itemData = await oada .get({ path: `${SERVICE_PATH}/businesses/${bid}/assessments/${key}`, @@ -346,7 +346,7 @@ export const handleAssessmentJob: WorkerFunction = async ( }); const aaa = getAutoApprove(); - info(`Autoapprove Assessments Configuration: [${aaa}]`); + info(`[job ${docJobId}] Autoapprove Assessments Configuration: [${aaa}]`); if (aaa) { const { failed, reasons }: { failed: boolean; reasons: string[] } = checkAssessment(item); @@ -357,7 +357,7 @@ export const handleAssessmentJob: WorkerFunction = async ( approval: !failed, }, }); - info(`Assessment Auto-${item.state}. [${item._id}]`); + info(`[job ${docJobId}] Assessment Auto-${item.state}. [${item._id}]`); await axios({ method: 'put', url: `${FL_DOMAIN}/v2/businesses/${CO_ID}/spawnedassessment/${ @@ -366,7 +366,7 @@ export const handleAssessmentJob: WorkerFunction = async ( headers: { Authorization: FL_TOKEN }, data: item, }); - info('WRITING REASONS', reasons.join(';')); + info(`[job ${docJobId}] Assessment Reasons: ${reasons.join(';')}`); const docJob = assessmentToFlId.get(item._id); if (docJob && failed) { @@ -522,6 +522,11 @@ export async function postTpDocument({ data: zdata, contentType: 'application/pdf', }); + await oada.put({ + path: `/${pdfId}/_meta`, + data: { filename: fKey }, + contentType: 'application/pdf', + }); trace(`Wrote file [${fKey}] to pdfId ${pdfId}.`); } catch (cError: unknown) { throw Buffer.byteLength(zdata) === 0 @@ -669,7 +674,7 @@ export async function postTpDocument({ [targetJobKey]: { _id: targetJobId }, }, }); - info(`Noted target job ${targetJobKey} in fl-sync job ${jobId}`); + info(`[job ${docJob.oadaId}] Noted target job ${targetJobKey} in fl-sync job ${jobId}`); return type; } @@ -863,25 +868,32 @@ async function finishDocument( } // Move approved docs to trading partner /bookmarks - info( - `Moving approved document to [/${masterid}/bookmarks/trellisfw/documents/${type}/${key}]` - ); - await CONNECTION.put({ - path: `/${masterid}/bookmarks/trellisfw/documents/${type}`, - data: {}, - tree, - }); - await CONNECTION.put({ - path: `/${masterid}/bookmarks/trellisfw/documents/${type}/${key}`, - data: { _id }, - }); - await CONNECTION.delete({ - path: `/${masterid}/shared/trellisfw/documents/${type}/${key}`, - }); + try { + info( + `[job ${docJobId}] Moving approved document to [/${masterid}/bookmarks/trellisfw/documents/${type}/${key}]` + ); + await CONNECTION.ensure({ + path: `/${masterid}/bookmarks/trellisfw/documents/${type}`, + data: {}, + tree, + }); + await CONNECTION.delete({ + path: `/${masterid}/bookmarks/trellisfw/documents/${type}/${key}`, + }); + await CONNECTION.put({ + path: `/${masterid}/bookmarks/trellisfw/documents/${type}/${key}`, + data: { _id, _rev: 0 }, + }); + await CONNECTION.delete({ + path: `/${masterid}/shared/trellisfw/documents/${type}/${key}`, + }); + } catch (err) { + error(`Error during move to bookmarks ${err}`) + } endJob(docJobId); } else { // Don't do anything; the job was already failed at the previous step and just marked in FL as Rejected. - info(`Document [${itemId}] with status [${status}]. finishDoc skipping.`); + info(`[job ${docJobId}] Document [${itemId}] with status [${status}]. finishDoc skipping.`); } } @@ -892,8 +904,8 @@ async function finishDocument( * @return */ function endJob(jobId: string, message?: string | Error | JobError) { - info('Removing %s from flSyncJobs Map', jobId); - trace(flSyncJobs, 'All flSyncJobs'); + info(`[job ${jobId}] Removing job from flSyncJobs Map`); + //trace(flSyncJobs, 'All flSyncJobs'); const prom = flSyncJobs.get(jobId); if (prom) { if (message) { @@ -1035,7 +1047,7 @@ async function handleScrapedResult( isObj(targetResultItem?.[key]) ) { const targetRes = targetResultItem?.[key] as JsonObject; - info(`Job result: [type: ${type}, key: ${key}, _id: ${targetRes?._id}]`); + info(`[job ${docJobId}] Job result: [type: ${type}, key: ${key}, _id: ${targetRes?._id}]`); } const { data: result } = (await CONNECTION.get({ @@ -1068,7 +1080,7 @@ async function handleScrapedResult( const validationResult = await validateResult(result, flMirror, type); info( - `Validation of pending document result:[${result._id}]: ${validationResult.status}` + `[job ${docJobId}] Validation of pending document result:[${result._id}]: ${validationResult.status}` ); await CONNECTION.put({ path: `/${docJobId}`, @@ -1094,7 +1106,7 @@ async function handleScrapedResult( //@ts-ignore //@ts-ignore if (rejectable[type]) { - info(`Document type ${type} was rejectable. Rejecting`); + info(`[job ${docJobId}] Document type ${type} was rejectable. Rejecting`); await rejectFlDocument(flId, docJobId, validationResult?.message); } } @@ -1119,7 +1131,7 @@ async function handleScrapedResult( if (assessmentId) { info( - `Assessment with id [${assessmentId}] already exists for document _id [${flId}].` + `[job ${docJobId}] Assessment with id [${assessmentId}] already exists for document _id [${flId}].` ); assessmentToFlId.set(assessmentId as string, { jobId: docJobId, @@ -1127,7 +1139,7 @@ async function handleScrapedResult( flId, }); } else { - info(`Assessment does not yet exist for document _id [${flId}]`); + info(`[job ${docJobId}] Assessment does not yet exist for document _id [${flId}]`); } let assess; @@ -1159,7 +1171,7 @@ async function handleScrapedResult( 'food-logiq-mirror' ] as unknown as FlObject; info( - `Assessment ${assessmentId} - bid: ${bid}; state: ${state}. Could not be modified.` + `[job ${docJobId}] Assessment ${assessmentId} - bid: ${bid}; state: ${state}. Could not be modified.` ); //TODO:This is maybe a problem causing cyclical re-runs of assessments? @@ -1180,7 +1192,7 @@ async function handleScrapedResult( } else throw cError as Error; } - info(`Spawned assessment [${assessmentId}] for business id [${bid}]`); + info(`[job ${docJobId}] Spawned assessment [${assessmentId}] for business id [${bid}]`); await postUpdate( CONNECTION, docJobId, @@ -1195,7 +1207,7 @@ async function handleScrapedResult( }); } } else { - info(`Skipping assessment for result of type [${flType}] [${type}].`); + info(`[job ${docJobId}] Skipping assessment for result of type [${flType}] [${type}].`); } } catch (cError: unknown) { error(cError); @@ -1211,7 +1223,7 @@ async function rejectFlDocument( jobId: string, message?: string ) { - info(`Rejecting FL document [${documentId}]. ${message}`); + info(`[job ${jobId}] Rejecting FL document [${documentId}]. ${message}`); // Reject to FL await axios({ @@ -1367,7 +1379,7 @@ async function queueAssessmentJob(change: JsonObject, path: string) { const { key: flDocumentId, type: flDocumentType, masterid } = indexConfig; if (!flDocumentType) { info( - `Assessment [${item._id}] could not find fl doc type prior to queueing. Ignoring.` + `[job ${docJobId}] Assessment [${item._id}] could not find fl doc type prior to queueing. Ignoring.` ); return; } @@ -1376,7 +1388,7 @@ async function queueAssessmentJob(change: JsonObject, path: string) { const docs = flTypes.get(flDocumentType)!; if (!assessmentType || !docs) { info( - `Assessment type of [${item._id}] was of type [${assessmentType}]. Ignoring.` + `[job ${docJobId}] Assessment type of [${item._id}] was of type [${assessmentType}]. Ignoring.` ); return; } @@ -1386,7 +1398,7 @@ async function queueAssessmentJob(change: JsonObject, path: string) { !Object.keys(docs.assessments).includes(assessmentType) ) { info( - `Assessment [${item._id}] was of type [${assessmentType}]. Ignoring.` + `[job ${docJobId}] Assessment [${item._id}] was of type [${assessmentType}]. Ignoring.` ); return; } @@ -1397,7 +1409,7 @@ async function queueAssessmentJob(change: JsonObject, path: string) { approvalUser === FL_TRELLIS_USER || approvalUser === APPROVAL_TRELLIS_USER; info( - `approvalInfo user ${ + `[job ${docJobId}] approvalInfo user ${ usersEqual ? 'matches our user' : `[${approvalUser}] does not match our users: [${FL_TRELLIS_USER} or ${APPROVAL_TRELLIS_USER}]` @@ -1437,7 +1449,7 @@ async function queueAssessmentJob(change: JsonObject, path: string) { [jobkey]: { _id: `resources/${jobkey}`, _rev: 0 }, }, }); - info('Posted job [assessment] at /resources/%s', jobkey); + info(`[job ${docJobId}] Posted job [assessment] at /resources/${jobkey}`); // Add it to the parent fl-sync job await CONNECTION.put({ @@ -1468,7 +1480,7 @@ async function queueAssessmentJob(change: JsonObject, path: string) { // Reject the assessment job; endJob(item._id, message); assessmentToFlId.delete(item._id); - info('REASONS', reasons); + info(`[job ${docJobId}] REASONS: ${reasons}`); // Reject the FL Document with a supplier message and reject the doc job if (flSyncJobs.get(docJobId)['allow-rejection'] !== false) { @@ -1484,7 +1496,7 @@ async function queueAssessmentJob(change: JsonObject, path: string) { await finishDocument(docJobId, flDocumentId, masterid, status); } else { info( - `Assessment ${item._id} failed logic, but cannot override approval. Calling finishDoc.` + `[job ${docJobId}] Assessment ${item._id} failed logic, but cannot override approval. Calling finishDoc.` ); await approveFlDocument(flDocumentId, docJobId); //TODO: remove this when/if FL is able to retrieve changes after approval updates @@ -1493,7 +1505,7 @@ async function queueAssessmentJob(change: JsonObject, path: string) { } else { // 2c. Job not handled by trellis system. const message = `Assessment not pending, approval status not set by Trellis. Skipping. Assessment: [${item._id}] User: [${approvalUser}] Status: [${status}]`; - info(message); + info(`[job ${docJobId}] ${message}`); return; } } catch (cError: unknown) { @@ -1523,12 +1535,12 @@ export async function postJob( const jobkey = headers['content-location']!.replace(/^\/resources\//, ''); await cancelDocJobs(indexConfig.key, jobkey) - + const _id = `resources/${jobkey}`; await oada.put({ path: pending, tree, data: { - [jobkey]: { _id: `resources/${jobkey}`, _rev: 0 }, + [jobkey]: { _id, _rev: 0 }, }, }); @@ -1539,15 +1551,15 @@ export async function postJob( services: { 'fl-sync': { jobs: { - [jobkey]: { _id: `resources/${jobkey}` }, + [jobkey]: { _id }, }, }, }, }, }); - info('Posted job [document] at /resources/%s', jobkey); - return `resources/${jobkey}`; + info(`[job ${_id}] Posted job [document] at /${_id}`); + return _id; } async function cancelDocJobs(itemId: string, newJobKey?: string) { diff --git a/src/vendorsReport.ts b/src/vendorsReport.ts index db52d50..6c0f191 100644 --- a/src/vendorsReport.ts +++ b/src/vendorsReport.ts @@ -42,6 +42,7 @@ const SERVICE_PATH = `/bookmarks/services/${SERVICE_NAME}`; const SUPPLIER = config.get('foodlogiq.testSupplier.id'); const FL_TRELLIS_USER = config.get('foodlogiq.trellisUser'); const CO_ID = config.get('foodlogiq.community.owner.id'); +const COMMUNITY_ID = config.get('foodlogiq.community.id'); const FL_DOMAIN = config.get('foodlogiq.domain'); const FL_TOKEN = config.get('foodlogiq.token'); const info = debug('fl-sync:vendorsReport:info'); @@ -672,7 +673,6 @@ async function tradingPartnerPrep06142023() { '5a68c37e6b390300014474dc', '5a786599a0ba6300010ce2c0', '5ac645a2fa59f800012dd928', - '5acbc2b1230156000101ab32', '5acbd1021772b90001b45a85', '5acbdaad230156000101ad90', '5acbdbf4627fec0001bb4283', @@ -749,7 +749,6 @@ async function tradingPartnerPrep06142023() { '60ba1e6e95c48d000e28bff9', '60ba1e50d2a730000eaca341', '60ba1ed1033190000e4b4a4b', - '60ba1f15f0ba6a000ec42129', '60ba1e8cd2a730000eaca35b', '60ba1f2ad2a730000eaca379', '60ba1f10b91309000ea97b97', @@ -1031,6 +1030,7 @@ type OldTradingPartner = { _id: string; }; +// Fix some old still-queued jobs that used the old masterid implementation async function fixJobs() { const oada = await connect({ domain, @@ -1092,6 +1092,44 @@ async function fixJobs() { process.exit(); } + +// Generate the Vendors list that still requires an SAP ID +// FYI this uses old non-v2 FL endpoints. v1 uses 'internalId' versus 'Internalid' +async function getFLVendorsList(date: string) { + const { data } = await axios({ + method: 'get', + url: `${FL_DOMAIN}/businesses/${CO_ID}/communities/${COMMUNITY_ID}/memberships`, + headers: { Authorization: FL_TOKEN }, + }) as unknown as any; + + let csvObj = data + .filter((s: any) => + s.locationGroup.name !== 'Internal' && + s.productGroup.name !== 'Internal' + ) + .filter((s: any) => s.internalId === '') + .map((s: any) => ({ + 'FL Name': s.business.name.replaceAll(',', ''), + 'FL Address': s.business.address.addressLineOne.replaceAll(',', ''), + 'FL City': s.business.address.city.replaceAll(',', ''), + 'FL State': s.business.address.region.replaceAll(',', ''), + 'FL Link': + `https://connect.foodlogiq.com/businesses/${CO_ID}/suppliers/detail/${s._id}/${COMMUNITY_ID}`, + })) + .sort((a:any, b:any) => { + if (a['FL Name'] > b['FL Name']) return 1; + if (a['FL Name'] < b['FL Name']) return -1; + return 0; + }) + const csvData = csvjson.toCSV(csvObj, { + delimiter: ',', + wrap: false, + headers: 'relative', + }); + + fs.writeFileSync(`Vendor Report${date}.csv`, csvData, { encoding: 'utf8' }); +} + setInterval(() => { console.log('stay alive'); }, 3000); @@ -1106,4 +1144,6 @@ setInterval(() => { //await tradingPartnerPrep06142023(); //await tradingPartnerFix07102023(); //await updateFlInternalIds(); -await fixJobs(); +//await fixJobs(); +await getFLVendorsList('2023-08-09'); +process.exit(); \ No newline at end of file