From 507559566963fc89af15496dbb3994031a2efa5c Mon Sep 17 00:00:00 2001 From: Leonardo Viva Date: Mon, 27 May 2024 09:22:52 -0300 Subject: [PATCH 01/20] feat: fetch access --- src/imports/access/getAccessForDocument.ts | 44 ++++++++++++++++++++++ src/imports/model/MetaAccess.ts | 2 + src/imports/model/User.ts | 8 ++++ src/imports/utils/accessUtils.ts | 24 ++++++++++++ src/server/routes/api/access/index.ts | 16 ++++++++ src/server/routes/index.ts | 2 + 6 files changed, 96 insertions(+) create mode 100644 src/imports/access/getAccessForDocument.ts create mode 100644 src/server/routes/api/access/index.ts diff --git a/src/imports/access/getAccessForDocument.ts b/src/imports/access/getAccessForDocument.ts new file mode 100644 index 00000000..2e095111 --- /dev/null +++ b/src/imports/access/getAccessForDocument.ts @@ -0,0 +1,44 @@ +import { getUserSafe } from '@imports/auth/getUser'; +import { MetaAccess } from '@imports/model/MetaAccess'; +import { MetaObject } from '@imports/model/MetaObject'; +import { KonectyResult } from '@imports/types/result'; +import { checkMetaOperation, getAccessFor } from '@imports/utils/accessUtils'; +import { errorReturn, successReturn } from '@imports/utils/return'; +import { Span } from '@opentelemetry/api'; +import filter from 'lodash/filter'; + +type GetAccessForParams = { + document: string; + authTokenId: string; + + tracingSpan?: Span; +}; + +export default async function getAccessForDocument({ document, authTokenId, tracingSpan }: GetAccessForParams): Promise> { + tracingSpan?.setAttribute('document', document); + tracingSpan?.addEvent('Get User', { authTokenId }); + + const userResponse = await getUserSafe(authTokenId); + if (userResponse.success === false) { + return errorReturn(userResponse.errors); + } + + const user = userResponse.data; + const access = getAccessFor(document, user); + + if (access === false || access.isReadable !== true) { + return errorReturn(`[${document}] You don't have permission for this document`); + } + + tracingSpan?.addEvent('Check Meta Operation'); + + const metaOperationAccess = checkMetaOperation({ user, operation: 'readAccess', document }); + if (metaOperationAccess === false) { + return errorReturn(`[${document}] You don't have permission to read access`); + } + + tracingSpan?.addEvent('Filter Accesses'); + const documentAccesses = filter(MetaObject.Access, { document }); + + return successReturn(documentAccesses); +} diff --git a/src/imports/model/MetaAccess.ts b/src/imports/model/MetaAccess.ts index ad9334e2..5c401970 100644 --- a/src/imports/model/MetaAccess.ts +++ b/src/imports/model/MetaAccess.ts @@ -12,6 +12,8 @@ export const FieldAccessSchema = z.record( export type FieldAccess = z.infer; export const MetaAccessSchema = z.object({ + document: z.string(), + fields: z.record(FieldAccessSchema), fieldDefaults: z.object({ isUpdatable: z.boolean().optional(), diff --git a/src/imports/model/User.ts b/src/imports/model/User.ts index ba703081..64b5efdb 100644 --- a/src/imports/model/User.ts +++ b/src/imports/model/User.ts @@ -17,6 +17,14 @@ export const UserModel = z.object({ access: z .object({ defaults: z.string().optional(), + + meta: z + .object({ + readAccess: z.boolean().or(z.string()).optional(), + updateAccess: z.boolean().or(z.string()).optional(), + createAccess: z.boolean().optional(), + }) + .optional(), }) .catchall(z.any()) .optional(), diff --git a/src/imports/utils/accessUtils.ts b/src/imports/utils/accessUtils.ts index 704d2a13..61648680 100644 --- a/src/imports/utils/accessUtils.ts +++ b/src/imports/utils/accessUtils.ts @@ -152,3 +152,27 @@ export function removeUnauthorizedDataForRead(metaAccess: MetaAccess, data: Reco return newData; } + +type CheckMetaOpParams = { + user: User; + operation: keyof Required['access']>['meta']; + document: string; +}; + +export function checkMetaOperation({ user, operation, document }: CheckMetaOpParams) { + const meta = user.access?.meta; + if (!meta) { + return false; + } + + const operationValue = meta[operation]; + if (operationValue == null) { + return false; + } + + if (typeof operationValue === 'string') { + return operationValue.split(',').find(doc => doc.trim() === document) != null; + } + + return operationValue === true; +} diff --git a/src/server/routes/api/access/index.ts b/src/server/routes/api/access/index.ts new file mode 100644 index 00000000..4967b6fa --- /dev/null +++ b/src/server/routes/api/access/index.ts @@ -0,0 +1,16 @@ +import { FastifyPluginCallback } from 'fastify'; +import fp from 'fastify-plugin'; + +import getAccessForDocument from '@imports/access/getAccessForDocument'; +import { getAuthTokenIdFromReq } from '@imports/utils/sessionUtils'; + +const accessApi: FastifyPluginCallback = (fastify, _, done) => { + fastify.get<{ Params: { document: string } }>('/rest/access/:document', async function (req, reply) { + const result = await getAccessForDocument({ document: req.params.document, authTokenId: getAuthTokenIdFromReq(req) ?? 'no-token' }); + reply.send(result); + }); + + done(); +}; + +export default fp(accessApi); diff --git a/src/server/routes/index.ts b/src/server/routes/index.ts index 01d81bcf..c68863e9 100644 --- a/src/server/routes/index.ts +++ b/src/server/routes/index.ts @@ -8,6 +8,7 @@ import proxy from '@fastify/http-proxy'; import initializeInstrumentation from '@imports/telemetry'; import { logger } from '@imports/utils/logger'; +import accessApi from '@server/routes/api/access'; import documentApi from './api/document'; import formApi from './api/form'; import listViewApi from './api/list-view'; @@ -46,6 +47,7 @@ fastify.register(cookie, { fastify.register(cors, getCorsConfig()); +fastify.register(accessApi); fastify.register(metasByDocumentApi); fastify.register(documentApi); fastify.register(formApi); From dce7724960708add469f3577e1608b34a8b6cd7b Mon Sep 17 00:00:00 2001 From: Leonardo Viva Date: Mon, 27 May 2024 10:36:34 -0300 Subject: [PATCH 02/20] feat: update access --- src/imports/access/updateAccess.ts | 100 ++++++++++++++++++++++++++ src/imports/model/MetaAccess.ts | 3 + src/server/routes/api/access/index.ts | 25 ++++++- 3 files changed, 127 insertions(+), 1 deletion(-) create mode 100644 src/imports/access/updateAccess.ts diff --git a/src/imports/access/updateAccess.ts b/src/imports/access/updateAccess.ts new file mode 100644 index 00000000..6dcb69b8 --- /dev/null +++ b/src/imports/access/updateAccess.ts @@ -0,0 +1,100 @@ +import { getUserSafe } from '@imports/auth/getUser'; +import { Condition } from '@imports/model/Filter'; +import { MetaAccess, MetaAccessSchema } from '@imports/model/MetaAccess'; +import { MetaObject } from '@imports/model/MetaObject'; +import { KonectyResult } from '@imports/types/result'; +import { checkMetaOperation } from '@imports/utils/accessUtils'; +import { errorReturn, successReturn } from '@imports/utils/return'; +import { Span } from '@opentelemetry/api'; +import find from 'lodash/find'; +import { UpdateFilter } from 'mongodb'; +import { z } from 'zod'; + +const AccessUpdateSchema = z.union([ + z.object({ + fields: z + .object({ + fieldNames: z.array(z.string()), + allow: z.boolean(), + condition: Condition.optional(), + }) + .array(), + }), + MetaAccessSchema.pick({ readFilter: true }).required(), + MetaAccessSchema.pick({ updateFilter: true }).required(), +]); +export type AccessUpdate = z.infer; + +type UpdateAccessParams = { + document: string; + accessName: string; + + data: AccessUpdate; + authTokenId: string; + + tracingSpan?: Span; +}; + +export default async function updateAccess({ document, accessName, data, authTokenId, tracingSpan }: UpdateAccessParams): Promise> { + tracingSpan?.setAttribute('document', document); + tracingSpan?.addEvent('Get User', { authTokenId }); + + const userResponse = await getUserSafe(authTokenId); + if (userResponse.success === false) { + return errorReturn(userResponse.errors); + } + + const user = userResponse.data; + + tracingSpan?.addEvent('Check Meta Operation'); + + const metaOperationAccess = checkMetaOperation({ user, operation: 'updateAccess', document }); + if (metaOperationAccess === false) { + tracingSpan?.setAttribute('error', "You don't have permission to update access"); + return errorReturn(`[${document}] You don't have permission to update access`); + } + + tracingSpan?.addEvent('Find Access', { accessName }); + const access = find(MetaObject.Access, { document, name: accessName }); + if (!access) { + tracingSpan?.setAttribute('error', 'Access not found'); + return errorReturn(`[${document}] Access not found`); + } + + tracingSpan?.addEvent('Parse update schema'); + const parseResponse = AccessUpdateSchema.safeParse(data); + if (parseResponse.success === false) { + const errors = parseResponse.error.flatten(); + const errorMessages = Object.values(errors.fieldErrors).concat(errors.formErrors).flat(); + tracingSpan?.setAttribute('error', errorMessages.join(', ')); + + return errorReturn(errorMessages); + } + + const updateObj: Required, '$set'>> = { $set: {} }; + + if ('fields' in data) { + for (const { fieldNames, allow, condition } of data.fields) { + for (const fieldName of fieldNames) { + updateObj.$set[`fields.${fieldName}`] = { allow, condition }; + } + } + } + + if ('readFilter' in data) { + updateObj.$set = { ...updateObj.$set, readFilter: data.readFilter }; + } + + if ('updateFilter' in data) { + updateObj.$set = { ...updateObj.$set, updateFilter: data.updateFilter }; + } + + if (Object.keys(updateObj.$set).length === 0) { + tracingSpan?.setAttribute('error', 'Nothing changed'); + return errorReturn('Nothing changed'); + } + + tracingSpan?.addEvent('Update Access'); + const result = await MetaObject.MetaObject.findOneAndUpdate({ _id: access._id }, updateObj, { returnDocument: 'after', ignoreUndefined: true }); + return successReturn(result); +} diff --git a/src/imports/model/MetaAccess.ts b/src/imports/model/MetaAccess.ts index 5c401970..e092613d 100644 --- a/src/imports/model/MetaAccess.ts +++ b/src/imports/model/MetaAccess.ts @@ -12,7 +12,10 @@ export const FieldAccessSchema = z.record( export type FieldAccess = z.infer; export const MetaAccessSchema = z.object({ + _id: z.string(), document: z.string(), + name: z.string(), + type: z.literal('access'), fields: z.record(FieldAccessSchema), fieldDefaults: z.object({ diff --git a/src/server/routes/api/access/index.ts b/src/server/routes/api/access/index.ts index 4967b6fa..ffba9303 100644 --- a/src/server/routes/api/access/index.ts +++ b/src/server/routes/api/access/index.ts @@ -2,11 +2,34 @@ import { FastifyPluginCallback } from 'fastify'; import fp from 'fastify-plugin'; import getAccessForDocument from '@imports/access/getAccessForDocument'; +import updateAccess, { AccessUpdate } from '@imports/access/updateAccess'; import { getAuthTokenIdFromReq } from '@imports/utils/sessionUtils'; const accessApi: FastifyPluginCallback = (fastify, _, done) => { fastify.get<{ Params: { document: string } }>('/rest/access/:document', async function (req, reply) { - const result = await getAccessForDocument({ document: req.params.document, authTokenId: getAuthTokenIdFromReq(req) ?? 'no-token' }); + const { tracer } = req.openTelemetry(); + const tracingSpan = tracer.startSpan('GET getAccess'); + + const result = await getAccessForDocument({ document: req.params.document, authTokenId: getAuthTokenIdFromReq(req) ?? 'no-token', tracingSpan }); + + tracingSpan.end(); + reply.send(result); + }); + + fastify.post<{ Params: { document: string; accessName: string } }>('/rest/access/:document/:accessName', async function (req, reply) { + const { tracer } = req.openTelemetry(); + const tracingSpan = tracer.startSpan('POST updateAccess'); + + const result = await updateAccess({ + document: req.params.document, + authTokenId: getAuthTokenIdFromReq(req) ?? 'no-token', + accessName: req.params.accessName, + data: req.body as AccessUpdate, + + tracingSpan, + }); + + tracingSpan.end(); reply.send(result); }); From d4c26771b0199ec7e82c1e7363e82a2d77b7892c Mon Sep 17 00:00:00 2001 From: Leonardo Viva Date: Fri, 14 Jun 2024 09:21:44 -0300 Subject: [PATCH 03/20] fix: update access operation --- src/imports/access/updateAccess.ts | 5 +++-- src/imports/model/MetaAccess.ts | 4 ++-- src/server/routes/api/access/index.ts | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/imports/access/updateAccess.ts b/src/imports/access/updateAccess.ts index 6dcb69b8..c637bc32 100644 --- a/src/imports/access/updateAccess.ts +++ b/src/imports/access/updateAccess.ts @@ -16,6 +16,7 @@ const AccessUpdateSchema = z.union([ .object({ fieldNames: z.array(z.string()), allow: z.boolean(), + operation: z.literal('READ').or(z.literal('UPDATE')).or(z.literal('DELETE')).or(z.literal('CREATE')), condition: Condition.optional(), }) .array(), @@ -74,9 +75,9 @@ export default async function updateAccess({ document, accessName, data, authTok const updateObj: Required, '$set'>> = { $set: {} }; if ('fields' in data) { - for (const { fieldNames, allow, condition } of data.fields) { + for (const { fieldNames, allow, condition, operation } of data.fields) { for (const fieldName of fieldNames) { - updateObj.$set[`fields.${fieldName}`] = { allow, condition }; + updateObj.$set[`fields.${fieldName}.${operation}`] = { allow, condition }; } } } diff --git a/src/imports/model/MetaAccess.ts b/src/imports/model/MetaAccess.ts index e092613d..e872ff49 100644 --- a/src/imports/model/MetaAccess.ts +++ b/src/imports/model/MetaAccess.ts @@ -29,8 +29,8 @@ export const MetaAccessSchema = z.object({ isReadable: z.boolean().optional(), isDeletable: z.boolean().optional(), - readFilter: KonFilter.optional(), - updateFilter: KonFilter.optional(), + readFilter: KonFilter.extend({ allow: z.boolean().optional() }).optional(), + updateFilter: KonFilter.extend({ allow: z.boolean().optional() }).optional(), }); export type MetaAccess = z.infer; diff --git a/src/server/routes/api/access/index.ts b/src/server/routes/api/access/index.ts index ffba9303..ef74a5f8 100644 --- a/src/server/routes/api/access/index.ts +++ b/src/server/routes/api/access/index.ts @@ -16,9 +16,9 @@ const accessApi: FastifyPluginCallback = (fastify, _, done) => { reply.send(result); }); - fastify.post<{ Params: { document: string; accessName: string } }>('/rest/access/:document/:accessName', async function (req, reply) { + fastify.put<{ Params: { document: string; accessName: string } }>('/rest/access/:document/:accessName', async function (req, reply) { const { tracer } = req.openTelemetry(); - const tracingSpan = tracer.startSpan('POST updateAccess'); + const tracingSpan = tracer.startSpan('PUT updateAccess'); const result = await updateAccess({ document: req.params.document, From 8ca4f84fa94f055bcca80f930889711000307c8d Mon Sep 17 00:00:00 2001 From: Leonardo Viva Date: Mon, 8 Jul 2024 09:04:33 -0300 Subject: [PATCH 04/20] refactor: update lookup references --- src/imports/data/data.js | 4 +- .../updateReferences/lookupReference.js | 195 ++++-------------- .../updateReferences/lookupReferences.js | 51 ++--- src/imports/konsistent/utils.js | 6 +- src/imports/meta/buildReferences.ts | 3 +- 5 files changed, 65 insertions(+), 194 deletions(-) diff --git a/src/imports/data/data.js b/src/imports/data/data.js index 3db9d086..c92d78c8 100644 --- a/src/imports/data/data.js +++ b/src/imports/data/data.js @@ -1116,7 +1116,7 @@ export async function update({ authTokenId, document, data, contextUser, tracing value: data.data[key], actionType: 'update', objectOriginalValues: record, - objectNewValues: data.data, + objectNewValues: bodyData, idsToUpdate: query._id.$in, }); if (lookupValidateResult.success === false) { @@ -1173,7 +1173,7 @@ export async function update({ authTokenId, document, data, contextUser, tracing value: data.data[fieldName], actionType: 'update', objectOriginalValues: record, - objectNewValues: data.data, + objectNewValues: bodyData, idsToUpdate: query._id.$in, }); if (result.success === false) { diff --git a/src/imports/konsistent/updateReferences/lookupReference.js b/src/imports/konsistent/updateReferences/lookupReference.js index 7a5694f0..44b679ba 100644 --- a/src/imports/konsistent/updateReferences/lookupReference.js +++ b/src/imports/konsistent/updateReferences/lookupReference.js @@ -1,183 +1,66 @@ -import compact from 'lodash/compact'; -import get from 'lodash/get'; -import has from 'lodash/has'; +import groupBy from 'lodash/groupBy'; import isArray from 'lodash/isArray'; import merge from 'lodash/merge'; import pick from 'lodash/pick'; -import uniq from 'lodash/uniq'; import { MetaObject } from '@imports/model/MetaObject'; +import { convertStringOfFieldsSeparatedByCommaIntoObjectToFind } from '@imports/utils/convertStringOfFieldsSeparatedByCommaIntoObjectToFind'; import { logger } from '@imports/utils/logger'; +import { getFieldNamesOfPaths } from '../utils'; -export default async function updateLookupReference(metaName, fieldName, field, record, relatedMetaName) { - // Try to get related meta - const meta = MetaObject.Meta[metaName]; - if (!meta) { - return logger.error(`MetaObject.Meta ${metaName} does not exists`); - } +async function getDescriptionAndInheritedFieldsToUpdate({ record, metaField, meta }) { + const fieldsToUpdate = {} - // Try to get related model - const collection = MetaObject.Collections[metaName]; - if (collection == null) { - return logger.error(`Model ${metaName} does not exists`); + if (isArray(metaField.descriptionFields) && metaField.descriptionFields.length > 0) { + const updateKey = metaField.isList ? `${metaField.name}.$` : `${metaField.name}`; + const descriptionFieldsValue = pick(record, Array.from(new Set(['_id'].concat(metaField.descriptionFields)))); + + fieldsToUpdate[updateKey] = descriptionFieldsValue; } - // Define field to query and field to update - const fieldToQuery = `${fieldName}._id`; - let fieldToUpdate = fieldName; + if (isArray(metaField.inheritedFields) && metaField.inheritedFields.length > 0) { + const inheritedFields = metaField.inheritedFields.filter(inheritedField => ['always', 'hierarchy_always'].includes(inheritedField.inherit)); + const fieldsToInherit = inheritedFields.map(inheritedField => meta.fields[inheritedField.fieldName]).filter(Boolean); - // If field is isList then use .$ into field to update - // to find in arrays and update only one item from array - if (field.isList === true) { - fieldToUpdate = `${fieldName}.$`; - } + const { true: lookupFields = [], false: nonLookupFields = [] } = groupBy(fieldsToInherit, field => field.type === 'lookup'); - // Define query with record id - const query = {}; - query[fieldToQuery] = record._id; + for (const field of nonLookupFields) { + fieldsToUpdate[field.name] = record[field.name]; + } - // Init object of data to set - const updateData = { $set: {} }; + for await (const lookupField of lookupFields) { + const keysToFind = [].concat(lookupField.descriptionFields || [], lookupField.inheritedFields || []).map(getFieldNamesOfPaths).join(); + const projection = convertStringOfFieldsSeparatedByCommaIntoObjectToFind(keysToFind); - // Add dynamic field name to update into object to update - updateData.$set[fieldToUpdate] = {}; + const Collection = MetaObject.Collections[lookupField.document]; + const lookupRecord = await Collection.findOne({ _id: record[lookupField.name]._id }, { projection }); - // If there are description fields - if (isArray(field.descriptionFields) && field.descriptionFields.length > 0) { - // Execute method to copy fields and values using an array of paths + const result = await getDescriptionAndInheritedFieldsToUpdate({ record: lookupRecord, metaField: lookupField, meta }); + merge(fieldsToUpdate, result); + } + } + + return fieldsToUpdate; +} - const descriptionFieldsValue = pick(record, Array.from(new Set(['_id'].concat(field.descriptionFields)))); - merge(updateData.$set[fieldToUpdate], descriptionFieldsValue); +export default async function updateLookupReference(metaName, fieldName, field, record, relatedMetaName) { + const meta = MetaObject.Meta[metaName]; + if (!meta) { + return logger.error(`MetaObject.Meta ${metaName} does not exists`); } - // If there are inherit fields - if (isArray(field.inheritedFields) && field.inheritedFields.length > 0) { - // For each inherited field - for (var inheritedField of field.inheritedFields) { - if (['always', 'hierarchy_always'].includes(inheritedField.inherit)) { - // Get field meta - var inheritedMetaField = meta.fields[inheritedField.fieldName]; - - if (inheritedField.inherit === 'hierarchy_always') { - if (get(inheritedMetaField, 'type') !== 'lookup' || inheritedMetaField.isList !== true) { - logger.error(`Not lookup or not isList field ${inheritedField.fieldName} in ${metaName}`); - continue; - } - if (!record[inheritedField.fieldName]) { - record[inheritedField.fieldName] = []; - } - record[inheritedField.fieldName].push({ - _id: record._id, - }); - } - - // If field is lookup - if (get(inheritedMetaField, 'type') === 'lookup') { - // Get model to find record - const lookupCollection = MetaObject.Collections[inheritedMetaField.document]; - - if (!lookupCollection) { - logger.error(`Document ${inheritedMetaField.document} not found`); - continue; - } - - if (has(record, `${inheritedField.fieldName}._id`) || (inheritedMetaField.isList === true && get(record, `${inheritedField.fieldName}.length`) > 0)) { - var lookupRecord, subQuery; - if (inheritedMetaField.isList !== true) { - subQuery = { _id: record[inheritedField.fieldName]._id.valueOf() }; - - // Find records - lookupRecord = await lookupCollection.findOne(subQuery); - - // If no record found log error - if (!lookupRecord) { - logger.error( - `Record not found for field ${inheritedField.fieldName} with _id [${subQuery._id}] on document [${inheritedMetaField.document}] not found`, - ); - continue; - } - - // Else copy description fields - if (isArray(inheritedMetaField.descriptionFields)) { - if (!updateData.$set[inheritedField.fieldName]) { - updateData.$set[inheritedField.fieldName] = {}; - } - - const descriptionFieldsValue = pick(lookupRecord, Array.from(new Set(['_id'].concat(inheritedMetaField.descriptionFields)))); - merge(updateData.$set[inheritedField.fieldName], descriptionFieldsValue); - } - - // End copy inherited values - if (isArray(inheritedMetaField.inheritedFields)) { - for (let inheritedMetaFieldItem of inheritedMetaField.inheritedFields) { - if (inheritedMetaFieldItem.inherit === 'always') { - updateData.$set[inheritedMetaFieldItem.fieldName] = lookupRecord[inheritedMetaFieldItem.fieldName]; - } - } - } - } else if (get(record, `${inheritedField.fieldName}.length`, 0) > 0) { - let ids = record[inheritedField.fieldName].map(item => item._id); - ids = compact(uniq(ids)); - subQuery = { - _id: { - $in: ids, - }, - }; - - const subOptions = {}; - if (isArray(inheritedMetaField.descriptionFields)) { - subOptions.projection = inheritedMetaField.descriptionFields.reduce((obj, item) => { - const key = item.split('.')[0]; - if (obj[key] == null) { - obj[key] = 1; - } - return obj; - }, {}); - } - - // Find records - const lookupRecords = await lookupCollection.find(subQuery, subOptions).toArray(); - const lookupRecordsById = lookupRecords.reduce((obj, item) => { - obj[item._id] = item; - return obj; - }, {}); - - record[inheritedField.fieldName].forEach(function (item) { - lookupRecord = lookupRecordsById[item._id]; - - // If no record found log error - if (!lookupRecord) { - logger.error( - `Record not found for field ${inheritedField.fieldName} with _id [${item._id}] on document [${inheritedMetaField.document}] not found`, - ); - return; - } - - // Else copy description fields - if (isArray(inheritedMetaField.descriptionFields)) { - const tempValue = pick(lookupRecord, Array.from(new Set(['_id'].concat(inheritedMetaField.descriptionFields)))); - if (updateData.$set[inheritedField.fieldName] == null) { - updateData.$set[inheritedField.fieldName] = []; - } - return updateData.$set[inheritedField.fieldName].push(tempValue); - } - }); - } - } - } else { - // Copy data into object to update if inherit method is 'always' - updateData.$set[inheritedField.fieldName] = record[inheritedField.fieldName]; - } - } - } + const collection = MetaObject.Collections[metaName]; + if (collection == null) { + return logger.error(`Model ${metaName} does not exists`); } try { - // Execute update and get affected records + const updateData = await getDescriptionAndInheritedFieldsToUpdate({ record, metaField: field, meta }); + + const query = { [`${fieldName}._id`]: record._id }; const updateResult = await collection.updateMany(query, updateData); - // If there are affected records then log if (updateResult.modifiedCount > 0) { logger.debug(`🔗 ${relatedMetaName} > ${metaName}.${fieldName} (${updateResult.modifiedCount})`); } diff --git a/src/imports/konsistent/updateReferences/lookupReferences.js b/src/imports/konsistent/updateReferences/lookupReferences.js index a35d5d57..7595454f 100644 --- a/src/imports/konsistent/updateReferences/lookupReferences.js +++ b/src/imports/konsistent/updateReferences/lookupReferences.js @@ -9,50 +9,39 @@ import uniq from 'lodash/uniq'; import updateLookupReference from '@imports/konsistent/updateReferences/lookupReference'; import { MetaObject } from '@imports/model/MetaObject'; import { logger } from '@imports/utils/logger'; - +import { getFieldNamesOfPaths } from '../utils'; + +/** + * When some document changes, verify if it's a lookup in some other document. + * If it is, update description & inherited fields in all related documents. + * @param {string} metaName + * @param {string} id + * @param {object} data + * @returns {Promise} + */ export default async function updateLookupReferences(metaName, id, data) { - // Get references from meta - let field, fieldName, fields; const references = MetaObject.References[metaName]; - // Verify if exists reverse relations if (!isObject(references) || size(keys(references.from)) === 0) { return; } - // Get model const collection = MetaObject.Collections[metaName]; if (collection == null) { throw new Error(`Collection ${metaName} not found`); } - // Define object to receive only references that have reference fields in changed data const referencesToUpdate = {}; - - // Get all keys that was updated const updatedKeys = Object.keys(data); // Iterate over all relations to verify if each relation have fields in changed keys for (var referenceDocumentName in references.from) { - fields = references.from[referenceDocumentName]; - for (fieldName in fields) { - var key; - field = fields[fieldName]; - let keysToUpdate = []; - // Split each key to get only first key of array of paths - if (size(field.descriptionFields) > 0) { - for (key of field.descriptionFields) { - keysToUpdate.push(key.split('.')[0]); - } - } - - if (size(field.inheritedFields) > 0) { - for (key of field.inheritedFields) { - keysToUpdate.push(key.fieldName.split('.')[0]); - } - } + const fields = references.from[referenceDocumentName]; + for (const fieldName in fields) { + const field = fields[fieldName]; + let keysToUpdate = [].concat(field.descriptionFields || [], field.inheritedFields || []).map(getFieldNamesOfPaths); - // Remove duplicated fields, can exists because we splited paths to get only first part + // Remove duplicated fields keysToUpdate = uniq(keysToUpdate); // Get only keys that exists in references and list of updated keys keysToUpdate = intersection(keysToUpdate, updatedKeys); @@ -67,27 +56,23 @@ export default async function updateLookupReferences(metaName, id, data) { } } - // If there are 0 relations to process then abort if (Object.keys(referencesToUpdate).length === 0) { return; } - // Find record with all information, not only udpated data, to can copy all related fields const record = await collection.findOne({ _id: id }); - // If no record was found log error and abort if (!record) { return logger.error(`Can't find record ${id} from ${metaName}`); } logger.debug(`Updating references for ${metaName} - ${Object.keys(referencesToUpdate).join(", ")}`); - // Iterate over relations to process and iterate over each related field to execute a method to update relations await BluebirdPromise.mapSeries(Object.keys(referencesToUpdate), async referenceDocumentName => { - fields = referencesToUpdate[referenceDocumentName]; + const fields = referencesToUpdate[referenceDocumentName]; await BluebirdPromise.mapSeries(Object.keys(fields), async fieldName => { - field = fields[fieldName]; + const field = fields[fieldName]; return updateLookupReference(referenceDocumentName, fieldName, field, record, metaName); }); }); -} \ No newline at end of file +} diff --git a/src/imports/konsistent/utils.js b/src/imports/konsistent/utils.js index 37b5120a..7a626ccf 100644 --- a/src/imports/konsistent/utils.js +++ b/src/imports/konsistent/utils.js @@ -1,8 +1,8 @@ -import isArray from 'lodash/isArray'; -import isObject from 'lodash/isObject'; import each from 'lodash/each'; import get from 'lodash/get'; import has from 'lodash/has'; +import isArray from 'lodash/isArray'; +import isObject from 'lodash/isObject'; import { MetaObject } from '@imports/model/MetaObject'; @@ -41,6 +41,8 @@ export function getFirstPartOfArrayOfPaths(paths) { return paths.map(i => i.split('.')[0]); } +export const getFieldNamesOfPaths = (fieldConf) => (fieldConf.fieldName ?? fieldConf).split('.')[0]; + export function formatValue(value, field, ignoreIsList) { if (!value) { return ''; diff --git a/src/imports/meta/buildReferences.ts b/src/imports/meta/buildReferences.ts index bf0aae1a..f4066b05 100644 --- a/src/imports/meta/buildReferences.ts +++ b/src/imports/meta/buildReferences.ts @@ -2,7 +2,7 @@ import { Field } from '@imports/model/Field'; import { Relation } from '@imports/model/Relation'; import { MetaObjectType } from '@imports/types/metadata'; -type Reference = Pick & { field: string }; +type Reference = Pick & { field: string }; type References = { [document: string]: { @@ -40,6 +40,7 @@ export default function buildReferences(Meta: Record) { [fieldName]: { type: field.type, field: fieldName, + name: fieldName, isList: field.isList, descriptionFields: field.descriptionFields, detailFields: field.detailFields, From ba5c33e212609511cf799808cc2eee0cd29be33e Mon Sep 17 00:00:00 2001 From: Leonardo Viva Date: Mon, 8 Jul 2024 09:22:04 -0300 Subject: [PATCH 05/20] fix: update atomic operator --- src/imports/konsistent/updateReferences/lookupReference.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/imports/konsistent/updateReferences/lookupReference.js b/src/imports/konsistent/updateReferences/lookupReference.js index 44b679ba..9bea3fc4 100644 --- a/src/imports/konsistent/updateReferences/lookupReference.js +++ b/src/imports/konsistent/updateReferences/lookupReference.js @@ -57,9 +57,12 @@ export default async function updateLookupReference(metaName, fieldName, field, try { const updateData = await getDescriptionAndInheritedFieldsToUpdate({ record, metaField: field, meta }); + if (Object.keys(updateData).length === 0) { + return; + } const query = { [`${fieldName}._id`]: record._id }; - const updateResult = await collection.updateMany(query, updateData); + const updateResult = await collection.updateMany(query, { $set: updateData }); if (updateResult.modifiedCount > 0) { logger.debug(`🔗 ${relatedMetaName} > ${metaName}.${fieldName} (${updateResult.modifiedCount})`); From 46d196c6798786d4938a388ea4d98be538929279 Mon Sep 17 00:00:00 2001 From: Leonardo Viva Date: Mon, 8 Jul 2024 14:19:07 -0300 Subject: [PATCH 06/20] fix: multi leven inheritedFields --- src/imports/konsistent/createHistory.js | 29 ++----------------- .../updateReferences/lookupReference.js | 29 ++++++++++++++++--- .../updateReferences/lookupReferences.js | 1 - 3 files changed, 28 insertions(+), 31 deletions(-) diff --git a/src/imports/konsistent/createHistory.js b/src/imports/konsistent/createHistory.js index bf404c13..5412258a 100644 --- a/src/imports/konsistent/createHistory.js +++ b/src/imports/konsistent/createHistory.js @@ -1,6 +1,7 @@ import { MetaObject } from "@imports/model/MetaObject"; import { logger } from "@imports/utils/logger"; import get from "lodash/get"; +import pick from "lodash/pick"; import { v4 as uuidV4 } from 'uuid'; export default async function createHistory(metaName, action, id, data, updatedBy, updatedAt, changed) { @@ -9,14 +10,12 @@ export default async function createHistory(metaName, action, id, data, updatedB return; } - const startTime = process.hrtime(); const changeId = uuidV4(); - const historyData = {}; const meta = MetaObject.Meta[metaName]; - // Remove fields that is marked to ignore history + // Remove fields marked to ignore history for (let key in changed) { const value = data[key]; const field = meta.fields[key]; @@ -27,8 +26,6 @@ export default async function createHistory(metaName, action, id, data, updatedB // Get history collection const history = MetaObject.Collections[`${metaName}.History`]; - - // If can't get history collection terminate this method if (!history) { return logger.error(`Can't get History collection from ${metaName}`); } @@ -37,36 +34,16 @@ export default async function createHistory(metaName, action, id, data, updatedB const userDetailFields = ["_id"].concat(get(meta, "fields._user.detailFields", ["name", "active"])); - // Define base data to history const historyItem = { dataId: id, createdAt: updatedAt, - createdBy: get(updatedBy, userDetailFields), + createdBy: pick(updatedBy, userDetailFields), data: historyData, type: action, }; - // Create history! try { await history.updateOne(historyQuery, { $set: historyItem, $setOnInsert: historyQuery }, { upsert: true }); - - const updateTime = process.hrtime(startTime); - // Log operation to shell - let log = metaName; - - switch (action) { - case 'create': - log = `${updateTime[0]}s ${updateTime[1] / 1000000}ms => Create history to create action over ${log}`; - break; - case 'update': - log = `${updateTime[0]}s ${updateTime[1] / 1000000}ms => Create history to update action over ${log}`; - break; - case 'delete': - log = `${updateTime[0]}s ${updateTime[1] / 1000000}ms => Create history to delete action over ${log}`; - break; - } - - logger.debug(log); } catch (e) { logger.error(e, 'Error on create history'); } diff --git a/src/imports/konsistent/updateReferences/lookupReference.js b/src/imports/konsistent/updateReferences/lookupReference.js index 9bea3fc4..f60a82f7 100644 --- a/src/imports/konsistent/updateReferences/lookupReference.js +++ b/src/imports/konsistent/updateReferences/lookupReference.js @@ -2,12 +2,14 @@ import groupBy from 'lodash/groupBy'; import isArray from 'lodash/isArray'; import merge from 'lodash/merge'; +import mergeWith from 'lodash/mergeWith'; import pick from 'lodash/pick'; import { MetaObject } from '@imports/model/MetaObject'; import { convertStringOfFieldsSeparatedByCommaIntoObjectToFind } from '@imports/utils/convertStringOfFieldsSeparatedByCommaIntoObjectToFind'; import { logger } from '@imports/utils/logger'; import { getFieldNamesOfPaths } from '../utils'; +import updateLookupReferences from './lookupReferences'; async function getDescriptionAndInheritedFieldsToUpdate({ record, metaField, meta }) { const fieldsToUpdate = {} @@ -29,15 +31,27 @@ async function getDescriptionAndInheritedFieldsToUpdate({ record, metaField, met fieldsToUpdate[field.name] = record[field.name]; } + // For inherited lookup fields we need to inherit recursively and merge all results for await (const lookupField of lookupFields) { const keysToFind = [].concat(lookupField.descriptionFields || [], lookupField.inheritedFields || []).map(getFieldNamesOfPaths).join(); const projection = convertStringOfFieldsSeparatedByCommaIntoObjectToFind(keysToFind); const Collection = MetaObject.Collections[lookupField.document]; - const lookupRecord = await Collection.findOne({ _id: record[lookupField.name]._id }, { projection }); - - const result = await getDescriptionAndInheritedFieldsToUpdate({ record: lookupRecord, metaField: lookupField, meta }); - merge(fieldsToUpdate, result); + const lookupRecord = await Collection.find({ _id: { $in: [].concat(record[lookupField.name]).map(v => v._id) } }, { projection }).toArray(); + + for await (const lookupRec of lookupRecord) { + const result = await getDescriptionAndInheritedFieldsToUpdate({ record: lookupRec, metaField: lookupField, meta }); + if (lookupField.isList) { + mergeWith(fieldsToUpdate, result, (objValue = [], srcValue = [], key) => /\$$/.test(key) ? [].concat(objValue, srcValue) : undefined); + } else { + merge(fieldsToUpdate, result); + } + } + + if (fieldsToUpdate[`${lookupField.name}.$`]) { + fieldsToUpdate[lookupField.name] = fieldsToUpdate[`${lookupField.name}.$`]; + delete fieldsToUpdate[`${lookupField.name}.$`]; + } } } @@ -66,10 +80,17 @@ export default async function updateLookupReference(metaName, fieldName, field, if (updateResult.modifiedCount > 0) { logger.debug(`🔗 ${relatedMetaName} > ${metaName}.${fieldName} (${updateResult.modifiedCount})`); + const projection = convertStringOfFieldsSeparatedByCommaIntoObjectToFind(Object.keys(updateData).join()); + + const modified = await collection.find(query, { projection }).toArray(); + await Promise.all(modified.map(async (modifiedRecord) => + updateLookupReferences(metaName, modifiedRecord._id, modifiedRecord) + )); } return updateResult.modifiedCount; } catch (e) { logger.error(e, 'Error updating lookup reference'); + logger.error({ metaName, fieldName, field, relatedMetaName }) } } \ No newline at end of file diff --git a/src/imports/konsistent/updateReferences/lookupReferences.js b/src/imports/konsistent/updateReferences/lookupReferences.js index 7595454f..3c00ef58 100644 --- a/src/imports/konsistent/updateReferences/lookupReferences.js +++ b/src/imports/konsistent/updateReferences/lookupReferences.js @@ -61,7 +61,6 @@ export default async function updateLookupReferences(metaName, id, data) { } const record = await collection.findOne({ _id: id }); - if (!record) { return logger.error(`Can't find record ${id} from ${metaName}`); } From 9fa4a8d0702c4383fcb46310d74065b18725cdfd Mon Sep 17 00:00:00 2001 From: Leonardo Viva Date: Mon, 8 Jul 2024 15:28:43 -0300 Subject: [PATCH 07/20] Merge branch 'feat/db-transactions' into refactor/lookup-reference --- src/imports/consts.js | 3 + src/imports/data/data.js | 1337 +++++++++-------- src/imports/konsistent/createHistory.js | 27 +- .../konsistent/processIncomingChange.ts | 32 +- .../konsistent/processReverseLookups.js | 28 +- .../updateReferences/lookupReference.js | 16 +- .../updateReferences/lookupReferences.js | 18 +- .../updateReferences/relationReference.ts | 56 +- .../updateReferences/relationReferences.ts | 122 +- .../meta/copyDescriptionAndInheritedFields.js | 6 +- src/imports/meta/getNextCode/index.ts | 13 +- src/imports/meta/loadMetaObjects.ts | 12 +- .../meta/validateAndProcessValueFor.js | 155 +- 13 files changed, 900 insertions(+), 925 deletions(-) diff --git a/src/imports/consts.js b/src/imports/consts.js index e222bbfb..e66075bb 100644 --- a/src/imports/consts.js +++ b/src/imports/consts.js @@ -11,3 +11,6 @@ export const DEFAULT_JPEG_MAX_SIZE = 3840; export const DEFAULT_THUMBNAIL_SIZE = 200; export const DEFAULT_EXPIRATION = 31536000; export const ALLOWED_CORS_FILE_TYPES = ['png', 'jpg', 'gif', 'jpeg', 'webp']; + +export const WRITE_TIMEOUT = 3e4; // 30 seconds +export const TRANSACTION_OPTIONS = { readConcern: { level: 'majority' }, writeConcern: { w: 'majority', wtimeoutMS: WRITE_TIMEOUT } }; diff --git a/src/imports/data/data.js b/src/imports/data/data.js index c92d78c8..e78dc278 100644 --- a/src/imports/data/data.js +++ b/src/imports/data/data.js @@ -32,7 +32,9 @@ import { logger } from '../utils/logger'; import { clearProjectionPathCollision, filterConditionToFn, parseFilterObject } from './filterUtils'; import { getUserSafe } from '@imports/auth/getUser'; +import { TRANSACTION_OPTIONS } from '@imports/consts'; import { find } from "@imports/data/api"; +import { client } from '@imports/database'; import processIncomingChange from '@imports/konsistent/processIncomingChange'; import objectsDiff from '@imports/utils/objectsDiff'; import { dateToString, stringToDate } from '../data/dateParser'; @@ -47,7 +49,7 @@ import { randomId } from '../utils/random'; import { errorReturn, successReturn } from '../utils/return'; import populateDetailFields from './populateDetailFields/fromArray'; -const WRITE_TIMEOUT = 3e4; // 30 seconds + export async function getNextUserFromQueue({ authTokenId, document, queueId, contextUser }) { const { success, data: user, errors } = await getUserSafe(authTokenId, contextUser); @@ -492,394 +494,421 @@ export async function create({ authTokenId, document, data, contextUser, upsert, } } - tracingSpan?.addEvent('Processing login'); - const processLoginResult = await processCollectionLogin({ meta: metaObject, data }); - if (processLoginResult.success === false) { - return processLoginResult; - } - - const cleanedData = Object.keys(data).reduce((acc, key) => { - if (data[key] == null || data[key] === '') { - return acc; - } - acc[key] = data[key]; - return acc; - }, {}); - - if (cleanedData._user == null) { - cleanedData._user = { _id: user._id }; - - if (metaObject.name !== 'QueueUser' && isString(data?.queue?._id)) { - tracingSpan?.addEvent('Deriving _user from passed queue', { queueId: data.queue._id }); - - const userQueueResult = await getNextUserFromQueue({ document, queueId: data.queue._id, contextUser: user }); - if (userQueueResult.success == false) { - return userQueueResult; + const dbSession = client.startSession({ defaultTransactionOptions: TRANSACTION_OPTIONS }); + try { + const transactionResult = await dbSession.withTransaction(async function createTransaction() { + tracingSpan?.addEvent('Processing login'); + const processLoginResult = await processCollectionLogin({ meta: metaObject, data }); + if (processLoginResult.success === false) { + return processLoginResult; } - cleanedData._user = { _id: userQueueResult.data.user }; - } - - if (metaObject.fields._user?.isList === true) { - cleanedData._user = [cleanedData._user]; - } - } - - tracingSpan?.addEvent('Validating _user'); - const validateUserResult = await validateAndProcessValueFor({ - meta: metaObject, - fieldName: '_user', - value: cleanedData._user, - actionType: 'insert', - objectOriginalValues: data, - objectNewValues: cleanedData, - }); - if (validateUserResult.success === false) { - return validateUserResult; - } - - if (validateUserResult.data != null) { - cleanedData._user = validateUserResult.data; - } + const cleanedData = Object.keys(data).reduce((acc, key) => { + if (data[key] == null || data[key] === '') { + return acc; + } + acc[key] = data[key]; + return acc; + }, {}); - tracingSpan?.addEvent('Calculating create permissions'); - const fieldPermissionResult = Object.keys(cleanedData).map(fieldName => { - const accessField = getFieldPermissions(access, fieldName); - if (accessField.isCreatable !== true) { - return errorReturn(`[${document}] You don't have permission to create field ${fieldName}`); - } + if (cleanedData._user == null) { + cleanedData._user = { _id: user._id }; - const accessFieldConditions = getFieldConditions(access, fieldName); - if (accessFieldConditions.CREATE != null) { - const getConditionFilterResult = filterConditionToFn(accessFieldConditions.CREATE, metaObject, { user }); + if (metaObject.name !== 'QueueUser' && isString(data?.queue?._id)) { + tracingSpan?.addEvent('Deriving _user from passed queue', { queueId: data.queue._id }); - if (getConditionFilterResult.success === false) { - return getConditionFilterResult; - } - - const isAllowToCreateField = getConditionFilterResult.data(cleanedData); + const userQueueResult = await getNextUserFromQueue({ document, queueId: data.queue._id, contextUser: user }); + if (userQueueResult.success == false) { + return userQueueResult; + } + cleanedData._user = { _id: userQueueResult.data.user }; + } - if (isAllowToCreateField === false) { - return errorReturn(`[${document}] You don't have permission to create field ${fieldName}`); + if (metaObject.fields._user?.isList === true) { + cleanedData._user = [cleanedData._user]; + } } - } - return successReturn(); - }); - - if (fieldPermissionResult.some(result => result.success === false)) { - return errorReturn( - fieldPermissionResult - .filter(result => result.success === false) - .map(result => result.errors) - .flat(), - ); - } - - const emailsToSend = []; - - tracingSpan?.addEvent('Validate&ProcessValueFor lookups'); - const validationResults = await BluebirdPromise.mapSeries( - Object.keys(metaObject.fields).filter(k => metaObject.fields[k]?.type === 'lookup'), - async key => { - const fieldToValidate = metaObject.fields[key]; - - const value = data[fieldToValidate.name]; - const result = await validateAndProcessValueFor({ + tracingSpan?.addEvent('Validating _user'); + const validateUserResult = await validateAndProcessValueFor({ meta: metaObject, - fieldName: key, - value, + fieldName: '_user', + value: cleanedData._user, actionType: 'insert', objectOriginalValues: data, objectNewValues: cleanedData, - }); - if (result.success === false) { - return result; + }, dbSession); + + if (validateUserResult.success === false) { + return validateUserResult; } - if (result.data != null) { - cleanedData[key] = result.data; + + if (validateUserResult.data != null) { + cleanedData._user = validateUserResult.data; } - return successReturn(); - }, - ); + tracingSpan?.addEvent('Calculating create permissions'); + const fieldPermissionResult = Object.keys(cleanedData).map(fieldName => { + const accessField = getFieldPermissions(access, fieldName); + if (accessField.isCreatable !== true) { + return errorReturn(`[${document}] You don't have permission to create field ${fieldName}`); + } - if (validationResults.some(result => result.success === false)) { - return errorReturn( - validationResults - .filter(result => result.success === false) - .map(result => result.errors) - .flat(), - ); - } + const accessFieldConditions = getFieldConditions(access, fieldName); + if (accessFieldConditions.CREATE != null) { + const getConditionFilterResult = filterConditionToFn(accessFieldConditions.CREATE, metaObject, { user }); - if (metaObject.scriptBeforeValidation != null) { - tracingSpan?.addEvent('Running scriptBeforeValidation'); - const scriptResult = await runScriptBeforeValidation({ - script: metaObject.scriptBeforeValidation, - data: cleanedData, - user, - meta: metaObject, - extraData: { original: {}, request: data, validated: cleanedData }, - }); + if (getConditionFilterResult.success === false) { + return getConditionFilterResult; + } - if (scriptResult.success === false) { - return scriptResult; - } + const isAllowToCreateField = getConditionFilterResult.data(cleanedData); - if (scriptResult.data?.result != null && isObject(scriptResult.data.result)) { - Object.assign(cleanedData, scriptResult.data.result); - } + if (isAllowToCreateField === false) { + return errorReturn(`[${document}] You don't have permission to create field ${fieldName}`); + } + } - if (scriptResult.data?.emailsToSend != null && isArray(scriptResult.data.emailsToSend)) { - emailsToSend.push(...scriptResult.data.emailsToSend); - } - } + return successReturn(); + }); - // Pega os valores padrão dos campos que não foram informados - Object.entries(metaObject.fields).forEach(([key, field]) => { - if (field.type !== 'autoNumber' && cleanedData[key] == null) { - if (field.defaultValue != null || (field.defaultValues != null && field.defaultValues.length > 0)) { - const getDefaultValue = () => { - if (field.defaultValue != null) { - return field.defaultValue; - } + if (fieldPermissionResult.some(result => result.success === false)) { + await dbSession.abortTransaction(); + return errorReturn( + fieldPermissionResult + .filter(result => result.success === false) + .map(result => result.errors) + .flat(), + ); + } - if (field.defaultValues != null && field.defaultValues.length > 0) { - // Work around to fix picklist behavior - if (field.type === 'picklist') { - const value = get(field, 'defaultValues.0.pt_BR'); - if (value == null) { - const lang = first(Object.keys(first(field.defaultValues))); - return get(first(field.defaultValues), lang); - } - return value; - } else { - return field.defaultValues; - } + const emailsToSend = []; + + tracingSpan?.addEvent('Validate&ProcessValueFor lookups'); + const validationResults = await BluebirdPromise.mapSeries( + Object.keys(metaObject.fields).filter(k => metaObject.fields[k]?.type === 'lookup'), + async key => { + const fieldToValidate = metaObject.fields[key]; + + const value = data[fieldToValidate.name]; + const result = await validateAndProcessValueFor({ + meta: metaObject, + fieldName: key, + value, + actionType: 'insert', + objectOriginalValues: data, + objectNewValues: cleanedData, + }, dbSession); + if (result.success === false) { + return result; + } + if (result.data != null) { + cleanedData[key] = result.data; } - }; - cleanedData[key] = getDefaultValue(); - } - } - }); + return successReturn(); + }, + ); - tracingSpan?.addEvent('Validate&processValueFor all fields'); - const validateAllFieldsResult = await BluebirdPromise.mapSeries(Object.keys(metaObject.fields), async key => { - const value = cleanedData[key]; - const field = metaObject.fields[key]; + if (validationResults.some(result => result.success === false)) { + await dbSession.abortTransaction(); + return errorReturn( + validationResults + .filter(result => result.success === false) + .map(result => result.errors) + .flat(), + ); + } - if (field.type === 'autoNumber' && (ignoreAutoNumber || value != null)) { - return successReturn(); - } + if (metaObject.scriptBeforeValidation != null) { + tracingSpan?.addEvent('Running scriptBeforeValidation'); + const scriptResult = await runScriptBeforeValidation({ + script: metaObject.scriptBeforeValidation, + data: cleanedData, + user, + meta: metaObject, + extraData: { original: {}, request: data, validated: cleanedData }, + }); - const result = await validateAndProcessValueFor({ - meta: metaObject, - fieldName: key, - value, - actionType: 'insert', - objectOriginalValues: data, - objectNewValues: cleanedData, - }); - if (result.success === false) { - return result; - } - if (result.data != null) { - cleanedData[key] = result.data; - } - return successReturn(); - }); + if (scriptResult.success === false) { + return scriptResult; + } - if (validateAllFieldsResult.some(result => result.success === false)) { - return errorReturn( - validateAllFieldsResult - .filter(result => result.success === false) - .map(result => result.errors) - .flat(), - ); - } + if (scriptResult.data?.result != null && isObject(scriptResult.data.result)) { + Object.assign(cleanedData, scriptResult.data.result); + } - if (metaObject.validationScript != null) { - tracingSpan?.addEvent('Running validation script'); + if (scriptResult.data?.emailsToSend != null && isArray(scriptResult.data.emailsToSend)) { + emailsToSend.push(...scriptResult.data.emailsToSend); + } + } - const validation = await processValidationScript({ script: metaObject.validationScript, validationData: metaObject.validationData, fullData: extend({}, data, cleanedData), user }); - if (validation.success === false) { - logger.error(validation, `Create - Script Validation Error - ${validation.reason}`); - return errorReturn(`[${document}] ${validation.reason}`); - } - } + // Pega os valores padrão dos campos que não foram informados + Object.entries(metaObject.fields).forEach(([key, field]) => { + if (field.type !== 'autoNumber' && cleanedData[key] == null) { + if (field.defaultValue != null || (field.defaultValues != null && field.defaultValues.length > 0)) { + const getDefaultValue = () => { + if (field.defaultValue != null) { + return field.defaultValue; + } - if (Object.keys(cleanedData).length > 0) { - const insertedQuery = {}; + if (field.defaultValues != null && field.defaultValues.length > 0) { + // Work around to fix picklist behavior + if (field.type === 'picklist') { + const value = get(field, 'defaultValues.0.pt_BR'); + if (value == null) { + const lang = first(Object.keys(first(field.defaultValues))); + return get(first(field.defaultValues), lang); + } + return value; + } else { + return field.defaultValues; + } + } + }; - const now = DateTime.local().toJSDate(); + cleanedData[key] = getDefaultValue(); + } + } + }); - const newRecord = Object.assign({}, cleanedData, { - _id: get(cleanedData, '_id', randomId()), - _createdAt: get(cleanedData, '_createdAt', now), - _createdBy: get(cleanedData, '_createdBy', pick(user, ['_id', 'name', 'group'])), - _updatedAt: get(cleanedData, '_updatedAt', now), - _updatedBy: get(cleanedData, '_updatedBy', pick(user, ['_id', 'name', 'group'])), - }); + tracingSpan?.addEvent('Validate&processValueFor all fields'); + const validateAllFieldsResult = await BluebirdPromise.mapSeries(Object.keys(metaObject.fields), async key => { + const value = cleanedData[key]; + const field = metaObject.fields[key]; - try { - if (processLoginResult.data != null) { - await MetaObject.Collections['User'].insertOne({ - _id: newRecord._id, - ...processLoginResult.data, - }); + if (field.type === 'autoNumber' && (ignoreAutoNumber || value != null)) { + return successReturn(); + } - const loginFieldResult = await validateAndProcessValueFor({ + const result = await validateAndProcessValueFor({ meta: metaObject, - fieldName: get(metaObject, 'login.field', 'login'), - value: processLoginResult.data, + fieldName: key, + value, actionType: 'insert', objectOriginalValues: data, objectNewValues: cleanedData, - }); + }, dbSession); - if (loginFieldResult.success === false) { - return loginFieldResult; + if (result.success === false) { + return result; } - - set(newRecord, get(metaObject, 'login.field', 'login'), loginFieldResult.data); - } - if (upsert != null && isObject(upsert)) { - const updateOperation = { - $setOnInsert: {}, - $set: {}, - }; - if (updateOnUpsert != null && isObject(updateOnUpsert)) { - Object.keys(newRecord).forEach(key => { - if (updateOnUpsert[key] != null) { - set(updateOperation, `$set.${key}`, newRecord[key]); - } else { - set(updateOperation, `$setOnInsert.${key}`, newRecord[key]); - } - }); - } else { - set(updateOperation, '$setOnInsert', newRecord); + if (result.data != null) { + cleanedData[key] = result.data; } + return successReturn(); + }); - if (isEmpty(updateOperation.$set)) { - unset(updateOperation, '$set'); - } + if (validateAllFieldsResult.some(result => result.success === false)) { + await dbSession.abortTransaction(); + return errorReturn( + validateAllFieldsResult + .filter(result => result.success === false) + .map(result => result.errors) + .flat(), + ); + } - if (isEmpty(updateOperation.$setOnInsert)) { - unset(updateOperation, '$setOnInsert'); - } + if (metaObject.validationScript != null) { + tracingSpan?.addEvent('Running validation script'); - tracingSpan?.addEvent('Upserting record'); - const upsertResult = await collection.updateOne(stringToDate(upsert), stringToDate(updateOperation), { - upsert: true, - writeConcern: { w: 'majority', wtimeoutMS: WRITE_TIMEOUT }, - }); - if (upsertResult.upsertedId != null) { - set(insertedQuery, '_id', upsertResult.upsertedId); - tracingSpan?.addEvent('Record upserted', { upsertedId: upsertResult.upsertedId }); - } else if (upsertResult.modifiedCount > 0) { - const upsertedRecord = await collection.findOne(stringToDate(upsert)); - if (upsertedRecord != null) { - set(insertedQuery, '_id', upsertedRecord._id); - tracingSpan?.addEvent('Record updated', { upsertedId: upsertedRecord._id }); - } + const validation = await processValidationScript({ script: metaObject.validationScript, validationData: metaObject.validationData, fullData: extend({}, data, cleanedData), user }); + if (validation.success === false) { + await dbSession.abortTransaction(); + logger.error(validation, `Create - Script Validation Error - ${validation.reason}`); + return errorReturn(`[${document}] ${validation.reason}`); } - } else { - const insertResult = await collection.insertOne(stringToDate(newRecord), { writeConcern: { w: 'majority', wtimeoutMS: WRITE_TIMEOUT } }); - set(insertedQuery, '_id', insertResult.insertedId); - tracingSpan?.addEvent('Record inserted', { insertedId: insertResult.insertedId }); } - } catch (e) { - logger.error(e, `Error on insert ${MetaObject.Namespace.ns}.${document}: ${e.message}`); - tracingSpan?.addEvent('Error on insert', { error: e.message }); - tracingSpan?.setAttribute({ error: e.message }); - if (e.code === 11000) { - return errorReturn(`[${document}] Duplicate key error`); - } - return errorReturn(`[${document}] ${e.message}`); - } + if (Object.keys(cleanedData).length > 0) { + const insertedQuery = {}; - if (insertedQuery._id == null) { - tracingSpan?.setAttribute({ error: 'InsertedQuery id is null' }); - return errorReturn(`[${document}] Error on insert, there is no affected record`); - } + const now = DateTime.local().toJSDate(); - const affectedRecord = await collection.findOne(insertedQuery, { readConcern: { level: 'majority' } }); - const resultRecord = removeUnauthorizedDataForRead(access, affectedRecord, user, metaObject); + const newRecord = Object.assign({}, cleanedData, { + _id: get(cleanedData, '_id', randomId()), + _createdAt: get(cleanedData, '_createdAt', now), + _createdBy: get(cleanedData, '_createdBy', pick(user, ['_id', 'name', 'group'])), + _updatedAt: get(cleanedData, '_updatedAt', now), + _updatedBy: get(cleanedData, '_updatedBy', pick(user, ['_id', 'name', 'group'])), + }); - if (isEmpty(MetaObject.Namespace.onCreate) === false) { - const hookData = { - action: 'create', - ns: MetaObject.Namespace.ns, - documentName: document, - user: pick(user, ['_id', 'code', 'name', 'active', 'username', 'nickname', 'group', 'emails', 'locale']), - data: [resultRecord], // Find records before apply access filter to query - }; + try { + if (processLoginResult.data != null) { + await MetaObject.Collections['User'].insertOne({ + _id: newRecord._id, + ...processLoginResult.data, + }, { session: dbSession }); + + const loginFieldResult = await validateAndProcessValueFor({ + meta: metaObject, + fieldName: get(metaObject, 'login.field', 'login'), + value: processLoginResult.data, + actionType: 'insert', + objectOriginalValues: data, + objectNewValues: cleanedData, + }, dbSession); + + if (loginFieldResult.success === false) { + return loginFieldResult; + } - const urls = [].concat(MetaObject.Namespace.onCreate); - tracingSpan?.addEvent('Running onCreate hooks', { urls }); + set(newRecord, get(metaObject, 'login.field', 'login'), loginFieldResult.data); + } + if (upsert != null && isObject(upsert)) { + const updateOperation = { + $setOnInsert: {}, + $set: {}, + }; + if (updateOnUpsert != null && isObject(updateOnUpsert)) { + Object.keys(newRecord).forEach(key => { + if (updateOnUpsert[key] != null) { + set(updateOperation, `$set.${key}`, newRecord[key]); + } else { + set(updateOperation, `$setOnInsert.${key}`, newRecord[key]); + } + }); + } else { + set(updateOperation, '$setOnInsert', newRecord); + } - await BluebirdPromise.mapSeries(urls, async url => { - try { - const hookUrl = url.replace('${dataId}', insertedQuery._id).replace('${documentId}', `${MetaObject.Namespace.ns}:${document}`); - const hookResponse = await fetch(hookUrl, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify(hookData), - }); - if (hookResponse.status === 200) { - logger.info(`Hook ${hookUrl} executed successfully`); + if (isEmpty(updateOperation.$set)) { + unset(updateOperation, '$set'); + } + + if (isEmpty(updateOperation.$setOnInsert)) { + unset(updateOperation, '$setOnInsert'); + } + + tracingSpan?.addEvent('Upserting record'); + const upsertResult = await collection.updateOne(stringToDate(upsert), stringToDate(updateOperation), { + upsert: true, + session: dbSession, + }); + if (upsertResult.upsertedId != null) { + set(insertedQuery, '_id', upsertResult.upsertedId); + tracingSpan?.addEvent('Record upserted', { upsertedId: upsertResult.upsertedId }); + } else if (upsertResult.modifiedCount > 0) { + const upsertedRecord = await collection.findOne(stringToDate(upsert), { session: dbSession }); + if (upsertedRecord != null) { + set(insertedQuery, '_id', upsertedRecord._id); + tracingSpan?.addEvent('Record updated', { upsertedId: upsertedRecord._id }); + } + } } else { - logger.error(`Error on hook ${url}: ${hookResponse.statusText}`); + const insertResult = await collection.insertOne(stringToDate(newRecord), { session: dbSession }); + set(insertedQuery, '_id', insertResult.insertedId); + tracingSpan?.addEvent('Record inserted', { insertedId: insertResult.insertedId }); } } catch (e) { - logger.error(e, `Error on hook ${url}: ${e.message}`); + logger.error(e, `Error on insert ${MetaObject.Namespace.ns}.${document}: ${e.message}`); + tracingSpan?.addEvent('Error on insert', { error: e.message }); + tracingSpan?.setAttribute({ error: e.message }); + await dbSession.abortTransaction(); + + if (e.code === 11000) { + return errorReturn(`[${document}] Duplicate key error`); + } + return errorReturn(`[${document}] ${e.message}`); } - }); - } - if (metaObject.scriptAfterSave != null) { - tracingSpan?.addEvent('Running scriptAfterSave'); - await runScriptAfterSave({ script: metaObject.scriptAfterSave, data: [resultRecord], user }); - } + if (insertedQuery._id == null) { + tracingSpan?.setAttribute({ error: 'InsertedQuery id is null' }); + return errorReturn(`[${document}] Error on insert, there is no affected record`); + } - if (emailsToSend.length > 0) { - tracingSpan?.addEvent('Sending emails'); - const messagesCollection = MetaObject.Collections['Message']; - const now = DateTime.local().toJSDate(); - await messagesCollection.insertMany( - emailsToSend.map(email => - Object.assign( - {}, - { - _id: randomId(), - _createdAt: now, - _createdBy: pick(user, ['_id', 'name', 'group']), - _updatedAt: now, - _updatedBy: { ...pick(user, ['_id', 'name', 'group']), ts: now }, - }, - email, - ), - ), - ); - } + const affectedRecord = await collection.findOne(insertedQuery, { session: dbSession }); + const resultRecord = removeUnauthorizedDataForRead(access, affectedRecord, user, metaObject); + + if (isEmpty(MetaObject.Namespace.onCreate) === false) { + const hookData = { + action: 'create', + ns: MetaObject.Namespace.ns, + documentName: document, + user: pick(user, ['_id', 'code', 'name', 'active', 'username', 'nickname', 'group', 'emails', 'locale']), + data: [resultRecord], // Find records before apply access filter to query + }; + + const urls = [].concat(MetaObject.Namespace.onCreate); + tracingSpan?.addEvent('Running onCreate hooks', { urls }); + + await BluebirdPromise.mapSeries(urls, async url => { + try { + const hookUrl = url.replace('${dataId}', insertedQuery._id).replace('${documentId}', `${MetaObject.Namespace.ns}:${document}`); + const hookResponse = await fetch(hookUrl, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(hookData), + }); + if (hookResponse.status === 200) { + logger.info(`Hook ${hookUrl} executed successfully`); + } else { + logger.error(`Error on hook ${url}: ${hookResponse.statusText}`); + } + } catch (e) { + logger.error(e, `Error on hook ${url}: ${e.message}`); + } + }); + } - if (resultRecord != null) { - if (MetaObject.Namespace.plan?.useExternalKonsistent !== true) { - try { - tracingSpan?.addEvent('Processing sync Konsistent'); - await processIncomingChange(document, resultRecord, 'create', user, resultRecord); - } catch (e) { - tracingSpan?.addEvent('Error on Konsistent', { error: e.message }); - logger.error(e, `Error on processIncomingChange ${document}: ${e.message}`); + if (metaObject.scriptAfterSave != null) { + tracingSpan?.addEvent('Running scriptAfterSave'); + await runScriptAfterSave({ script: metaObject.scriptAfterSave, data: [resultRecord], user }); + } + + if (emailsToSend.length > 0) { + tracingSpan?.addEvent('Sending emails'); + const messagesCollection = MetaObject.Collections['Message']; + const now = DateTime.local().toJSDate(); + await messagesCollection.insertMany( + emailsToSend.map(email => + Object.assign( + {}, + { + _id: randomId(), + _createdAt: now, + _createdBy: pick(user, ['_id', 'name', 'group']), + _updatedAt: now, + _updatedBy: { ...pick(user, ['_id', 'name', 'group']), ts: now }, + }, + email, + ), + ), + { session: dbSession } + ); + } + + if (resultRecord != null) { + if (MetaObject.Namespace.plan?.useExternalKonsistent !== true) { + try { + tracingSpan?.addEvent('Processing sync Konsistent'); + await processIncomingChange(document, resultRecord, 'create', user, resultRecord, dbSession); + } catch (e) { + tracingSpan?.addEvent('Error on Konsistent', { error: e.message }); + logger.error(e, `Error on processIncomingChange ${document}: ${e.message}`); + await dbSession.abortTransaction(); + return errorReturn(`[${document}] Error on Konsistent: ${e.message}`); + } + } + return successReturn([dateToString(resultRecord)]); } } - return successReturn([dateToString(resultRecord)]); + }); + + if (transactionResult != null && transactionResult.success != null) { + tracingSpan?.addEvent('Operation result', omit(transactionResult, ['data'])); + return transactionResult; } + } catch (e) { + tracingSpan?.addEvent('Error on transaction', { error: e.message }); + tracingSpan?.setAttribute({ error: e.message }); + logger.error(e, "outer logger") + } + finally { + tracingSpan?.addEvent('Ending session'); + dbSession.endSession(); } return errorReturn(`[${document}] Error on insert, there is no affected record`); @@ -983,382 +1012,404 @@ export async function update({ authTokenId, document, data, contextUser, tracing } } - tracingSpan?.addEvent('Processing login'); - const processLoginResult = await processCollectionLogin({ meta: metaObject, data }); - if (processLoginResult.success === false) { - return processLoginResult; - } - - const fieldFilterConditions = Object.keys(data.data).reduce((acc, fieldName) => { - const accessFieldConditions = getFieldConditions(access, fieldName); - if (accessFieldConditions.UPDATE) { - acc.push(accessFieldConditions.UPDATE); - } - return acc; - }, []); + const dbSession = client.startSession({ defaultTransactionOptions: TRANSACTION_OPTIONS }); + try { + const transactionResult = await dbSession.withTransaction(async function updateTransaction() { + tracingSpan?.addEvent('Processing login'); + const processLoginResult = await processCollectionLogin({ meta: metaObject, data }); + if (processLoginResult.success === false) { + return processLoginResult; + } - const filter = { - match: 'and', - filters: [], - }; + const fieldFilterConditions = Object.keys(data.data).reduce((acc, fieldName) => { + const accessFieldConditions = getFieldConditions(access, fieldName); + if (accessFieldConditions.UPDATE) { + acc.push(accessFieldConditions.UPDATE); + } + return acc; + }, []); - if (isObject(access.updateFilter)) { - filter.filters.push(access.updateFilter); - } + const filter = { + match: 'and', + filters: [], + }; - if (fieldFilterConditions.length > 0) { - set(filter, 'conditions', fieldFilterConditions); - } + if (isObject(access.updateFilter)) { + filter.filters.push(access.updateFilter); + } - tracingSpan?.addEvent('Parsing filter'); - const updateFilterResult = parseFilterObject(filter, metaObject, { user }); + if (fieldFilterConditions.length > 0) { + set(filter, 'conditions', fieldFilterConditions); + } - const query = Object.assign({ _id: { $in: [] } }, updateFilterResult); + tracingSpan?.addEvent('Parsing filter'); + const updateFilterResult = parseFilterObject(filter, metaObject, { user }); - if (isArray(query._id.$in)) { - data.ids.forEach(id => { - query._id.$in.push(id._id); - }); - } + const query = Object.assign({ _id: { $in: [] } }, updateFilterResult); - const options = {}; + if (isArray(query._id.$in)) { + data.ids.forEach(id => { + query._id.$in.push(id._id); + }); + } - if (metaObject.scriptBeforeValidation == null && metaObject.validationScript == null && metaObject.scriptAfterSave == null) { - set(options, 'fields', { - _updatedAt: 1, - }); - } + const options = { session: dbSession }; - tracingSpan?.addEvent('Finding records to update', { query, options }); - const existsRecords = await collection.find(query, options).toArray(); + if (metaObject.scriptBeforeValidation == null && metaObject.validationScript == null && metaObject.scriptAfterSave == null) { + set(options, 'fields', { + _updatedAt: 1, + }); + } - // Validate if user have permission to update each record that he are trying - const forbiddenRecords = data.ids.filter(id => { - const record = existsRecords.find(record => record._id === id._id); - if (record == null) { - return true; - } - return false; - }); + tracingSpan?.addEvent('Finding records to update', { query, options }); + const existsRecords = await collection.find(query, options).toArray(); - if (forbiddenRecords.length > 0) { - return errorReturn(`[${document}] You don't have permission to update records ${forbiddenRecords.map(record => record._id).join(', ')} or they don't exists`); - } + // Validate if user have permission to update each record that he are trying + const forbiddenRecords = data.ids.filter(id => { + const record = existsRecords.find(record => record._id === id._id); + if (record == null) { + return true; + } + return false; + }); - // outdateRecords are records that user are trying to update but they are out of date - if (metaObject.ignoreUpdatedAt !== true) { - const outdateRecords = data.ids.filter(id => { - const record = existsRecords.find(record => record._id === id._id); - if (record == null) { - return true; - } - if (DateTime.fromJSDate(record._updatedAt).diff(DateTime.fromISO(id._updatedAt.$date)).milliseconds !== 0) { - return true; + if (forbiddenRecords.length > 0) { + return errorReturn(`[${document}] You don't have permission to update records ${forbiddenRecords.map(record => record._id).join(', ')} or they don't exists`); } - return false; - }); - if (outdateRecords.length > 0) { - const mapOfFieldsToUpdateForHistoryQuery = Object.keys(data.data).reduce((acc, fieldName) => { - acc.push({ [`diffs.${fieldName}`]: { $exists: 1 } }); - return acc; - }, []); - const outOfDateQuery = { - $or: outdateRecords.map(record => ({ - dataId: record._id, - createdAt: { - $gt: DateTime.fromISO(record._updatedAt.$date).toJSDate(), - }, - $or: mapOfFieldsToUpdateForHistoryQuery, - })), - }; - - const historyCollection = MetaObject.Collections[`${document}.History`]; + // outdateRecords are records that user are trying to update but they are out of date + if (metaObject.ignoreUpdatedAt !== true) { + const outdateRecords = data.ids.filter(id => { + const record = existsRecords.find(record => record._id === id._id); + if (record == null) { + return true; + } + if (DateTime.fromJSDate(record._updatedAt).diff(DateTime.fromISO(id._updatedAt.$date)).milliseconds !== 0) { + return true; + } + return false; + }); - tracingSpan?.addEvent('Finding out of date records', { outOfDateQuery }); - const outOfDateRecords = await historyCollection.find(outOfDateQuery).toArray(); + if (outdateRecords.length > 0) { + const mapOfFieldsToUpdateForHistoryQuery = Object.keys(data.data).reduce((acc, fieldName) => { + acc.push({ [`data.${fieldName}`]: { $exists: 1 } }); + return acc; + }, []); + const outOfDateQuery = { + $or: outdateRecords.map(record => ({ + dataId: record._id, + createdAt: { + $gt: DateTime.fromISO(record._updatedAt.$date).toJSDate(), + }, + $or: mapOfFieldsToUpdateForHistoryQuery, + })), + }; + + const historyCollection = MetaObject.Collections[`${document}.History`]; + + tracingSpan?.addEvent('Finding out of date records', { outOfDateQuery }); + const outOfDateRecords = await historyCollection.find(outOfDateQuery, { session: dbSession }).toArray(); + + if (outOfDateRecords.length > 0) { + const errorMessage = outOfDateRecords.reduce((acc, history) => { + Object.keys(data.data).forEach(fieldName => { + if (history.data[fieldName] != null) { + acc.push( + `[${document}] Record ${history.dataId} is out of date, field ${fieldName} was updated at ${DateTime.fromJSDate(history.createdAt).toISO()} by ${get(history, "updatedBy.name", "Unknown")}`, + ); + } + }); + return acc; + }, []); - if (outOfDateRecords.length > 0) { - const errorMessage = outOfDateRecords.reduce((acc, record) => { - Object.keys(data.data).forEach(fieldName => { - if (record.diffs[fieldName] != null) { - acc.push( - `[${document}] Record ${record.dataId} is out of date, field ${fieldName} was updated at ${DateTime.fromJSDate(record.createdAt).toISO()} by ${record.createdBy.name - }`, - ); + if (errorMessage.length > 0) { + return errorReturn(errorMessage.join('\n')); } - }); - return acc; - }, []); - - if (errorMessage.length > 0) { - return errorReturn(errorMessage.join('\n')); + } } } - } - } - const emailsToSend = []; - - const updateResults = await BluebirdPromise.mapSeries(existsRecords, async record => { - const bodyData = {}; + const emailsToSend = []; + + const updateResults = await BluebirdPromise.mapSeries(existsRecords, async record => { + const bodyData = {}; + + if (metaObject.scriptBeforeValidation != null) { + tracingSpan?.addEvent('Validate&ProcessValueFor lookups'); + + const lookupValues = {}; + const validateLookupsResults = await BluebirdPromise.mapSeries( + Object.keys(data.data).filter(key => metaObject.fields[key]?.type === 'lookup'), + async key => { + const lookupValidateResult = await validateAndProcessValueFor({ + meta: metaObject, + fieldName: key, + value: data.data[key], + actionType: 'update', + objectOriginalValues: record, + objectNewValues: bodyData, + idsToUpdate: query._id.$in, + }, dbSession); + if (lookupValidateResult.success === false) { + return lookupValidateResult; + } + if (lookupValidateResult.data != null) { + set(lookupValues, key, lookupValidateResult.data); + } + return successReturn(); + }, + ); - if (metaObject.scriptBeforeValidation != null) { - tracingSpan?.addEvent('Validate&ProcessValueFor lookups'); + if (validateLookupsResults.some(result => result.success === false)) { + return errorReturn( + validateLookupsResults + .filter(result => result.success === false) + .map(result => result.errors) + .flat(), + ); + } - const lookupValues = {}; - const validateLookupsResults = await BluebirdPromise.mapSeries( - Object.keys(data.data).filter(key => metaObject.fields[key]?.type === 'lookup'), - async key => { - const lookupValidateResult = await validateAndProcessValueFor({ + tracingSpan?.addEvent('Running scriptBeforeValidation'); + const extraData = { + original: existsRecords.find(r => r._id === record._id), + request: data.data, + validated: lookupValues, + }; + const scriptResult = await runScriptBeforeValidation({ + script: metaObject.scriptBeforeValidation, + data: extend({}, record, data.data, lookupValues), + user, meta: metaObject, - fieldName: key, - value: data.data[key], - actionType: 'update', - objectOriginalValues: record, - objectNewValues: bodyData, - idsToUpdate: query._id.$in, + extraData, }); - if (lookupValidateResult.success === false) { - return lookupValidateResult; + + if (scriptResult.success === false) { + return scriptResult; + } + + if (scriptResult.data?.result != null && isObject(scriptResult.data.result)) { + Object.assign(bodyData, scriptResult.data.result); } - if (lookupValidateResult.data != null) { - set(lookupValues, key, lookupValidateResult.data); + if (scriptResult.data?.emailsToSend != null && isArray(scriptResult.data.emailsToSend)) { + emailsToSend.push(...scriptResult.data.emailsToSend); + } + } + + tracingSpan?.addEvent('Validate&ProcessValueFor all fields'); + const validateResult = await BluebirdPromise.mapSeries(Object.keys(data.data), async fieldName => { + if (bodyData[fieldName] == null) { + const result = await validateAndProcessValueFor({ + meta: metaObject, + fieldName, + value: data.data[fieldName], + actionType: 'update', + objectOriginalValues: record, + objectNewValues: data.data, + idsToUpdate: query._id.$in, + }, dbSession); + if (result.success === false) { + return result; + } + if (result.data !== undefined) { + set(bodyData, fieldName, result.data); + } } return successReturn(); - }, - ); + }); - if (validateLookupsResults.some(result => result.success === false)) { - return errorReturn( - validateLookupsResults - .filter(result => result.success === false) - .map(result => result.errors) - .flat(), - ); - } + if (validateResult.some(result => result.success === false)) { + return errorReturn( + validateResult + .filter(result => result.success === false) + .map(result => result.errors) + .flat(), + ); + } - tracingSpan?.addEvent('Running scriptBeforeValidation'); - const extraData = { - original: existsRecords.find(r => r._id === record._id), - request: data.data, - validated: lookupValues, - }; - const scriptResult = await runScriptBeforeValidation({ - script: metaObject.scriptBeforeValidation, - data: extend({}, record, data.data, lookupValues), - user, - meta: metaObject, - extraData, - }); + if (metaObject.validationScript != null) { + tracingSpan?.addEvent('Running validation script'); + const validationScriptResult = await processValidationScript({ script: metaObject.validationScript, validationData: metaObject.validationData, fullData: extend({}, record, data.data), user }); + if (validationScriptResult.success === false) { + logger.error(validationScriptResult, `Update - Script Validation Error - ${validationScriptResult.reason}`); + return validationScriptResult; + } + } - if (scriptResult.success === false) { - return scriptResult; - } + const updateOperation = Object.keys(bodyData).reduce((acc, key) => { + if (bodyData[key] !== undefined) { + if (bodyData[key] === null) { + set(acc, `$unset.${key}`, 1); + } else { + set(acc, `$set.${key}`, bodyData[key]); + } + } + return acc; + }, {}); - if (scriptResult.data?.result != null && isObject(scriptResult.data.result)) { - Object.assign(bodyData, scriptResult.data.result); - } - if (scriptResult.data?.emailsToSend != null && isArray(scriptResult.data.emailsToSend)) { - emailsToSend.push(...scriptResult.data.emailsToSend); - } - } + const ignoreUpdate = Object.keys(bodyData).every(key => { + if (metaObject.fields == null || metaObject.fields[key] == null) { + return false; + } - tracingSpan?.addEvent('Validate&ProcessValueFor all fields'); - const validateResult = await BluebirdPromise.mapSeries(Object.keys(data.data), async fieldName => { - if (bodyData[fieldName] == null) { - const result = await validateAndProcessValueFor({ - meta: metaObject, - fieldName, - value: data.data[fieldName], - actionType: 'update', - objectOriginalValues: record, - objectNewValues: bodyData, - idsToUpdate: query._id.$in, + return metaObject.fields[key].ignoreHistory === true; }); - if (result.success === false) { - return result; - } - if (result.data !== undefined) { - set(bodyData, fieldName, result.data); + + if (ignoreUpdate === false) { + set(updateOperation, '$set._updatedAt', DateTime.local().toJSDate()); + set( + updateOperation, + '$set._updatedBy', + Object.assign({}, pick(user, ['_id', 'name', 'group']), { + ts: get(updateOperation, '$set._updatedAt'), + }), + ); } - } - return successReturn(); - }); - if (validateResult.some(result => result.success === false)) { - return errorReturn( - validateResult - .filter(result => result.success === false) - .map(result => result.errors) - .flat(), - ); - } + const filter = { + _id: record._id, + }; - if (metaObject.validationScript != null) { - tracingSpan?.addEvent('Running validation script'); - const validationScriptResult = await processValidationScript({ script: metaObject.validationScript, validationData: metaObject.validationData, fullData: extend({}, record, data.data), user }); - if (validationScriptResult.success === false) { - logger.error(validationScriptResult, `Update - Script Validation Error - ${validationScriptResult.reason}`); - return errorReturn(validationScriptResult.reason); - } - } + try { + tracingSpan?.addEvent('Updating record', { filter, updateOperation }); + await collection.updateOne(filter, updateOperation, { session: dbSession }); + return successReturn(record._id); + } catch (e) { + logger.error(e, `Error on update ${MetaObject.Namespace.ns}.${document}: ${e.message}`); + tracingSpan?.addEvent('Error on update', { error: e.message }); + tracingSpan?.setAttribute({ error: e.message }); - const updateOperation = Object.keys(bodyData).reduce((acc, key) => { - if (bodyData[key] !== undefined) { - if (bodyData[key] === null) { - set(acc, `$unset.${key}`, 1); - } else { - set(acc, `$set.${key}`, bodyData[key]); + if (e.code === 11000) { + return errorReturn(`[${document}] Duplicate key error`); + } + return errorReturn(`[${document}] ${e.message}`); } - } - return acc; - }, {}); + }); - const ignoreUpdate = Object.keys(bodyData).every(key => { - if (metaObject.fields == null || metaObject.fields[key] == null) { - return false; + if (updateResults.some(result => result.success === false)) { + return errorReturn( + updateResults + .filter(result => result.success === false) + .map(result => result.errors) + .flat(), + ); } - return metaObject.fields[key].ignoreHistory === true; - }); + const updatedIs = updateResults.map(result => result.data); + + if (updatedIs.length > 0) { + if (MetaObject.Namespace.onUpdate != null) { + const hookRecords = await collection.find({ _id: { $in: updatedIs } }).toArray(); + + const hookData = { + action: 'update', + ns: MetaObject.Namespace.ns, + documentName: document, + user: pick(user, ['_id', 'code', 'name', 'active', 'username', 'nickname', 'group', 'emails', 'locale']), + data: hookRecords, + }; + + const urls = [].concat(MetaObject.Namespace.onUpdate); + tracingSpan?.addEvent('Running onUpdate hooks', { urls }); + + await BluebirdPromise.mapSeries(urls, async url => { + try { + const hookUrl = url.replace('${dataId}', updatedIs.join(',')).replace('${documentId}', `${MetaObject.Namespace.ns}:${document}`); + const hookResponse = await fetch(hookUrl, { + method: 'POST', + body: JSON.stringify(hookData), + }); + if (hookResponse.status === 200) { + logger.info(`Hook ${hookUrl} executed successfully`); + } else { + logger.error(`Error on hook ${url}: ${hookResponse.statusText}`); + } + } catch (e) { + logger.error(e, `Error on hook ${url}: ${e.message}`); + } + }); + } - if (ignoreUpdate === false) { - set(updateOperation, '$set._updatedAt', DateTime.local().toJSDate()); - set( - updateOperation, - '$set._updatedBy', - Object.assign({}, pick(user, ['_id', 'name', 'group']), { - ts: get(updateOperation, '$set._updatedAt'), - }), - ); - } + const updatedQuery = { + _id: { + $in: updatedIs, + }, + }; - const filter = { - _id: record._id, - }; + if (isObject(access.readFilter)) { + const readFilter = parseFilterObject(access.readFilter, metaObject, { user }); - try { - tracingSpan?.addEvent('Updating record', { filter, updateOperation }); - await collection.updateOne(filter, updateOperation, { writeConcern: { w: 'majority', wtimeoutMS: WRITE_TIMEOUT } }); - return successReturn(record._id); - } catch (e) { - logger.error(e, `Error on update ${MetaObject.Namespace.ns}.${document}: ${e.message}`); - tracingSpan?.addEvent('Error on update', { error: e.message }); - tracingSpan?.setAttribute({ error: e.message }); - - if (e.code === 11000) { - return errorReturn(`[${document}] Duplicate key error`); - } - return errorReturn(`[${document}] ${e.message}`); - } - }); + merge(updatedQuery, readFilter); + } - if (updateResults.some(result => result.success === false)) { - return errorReturn( - updateResults - .filter(result => result.success === false) - .map(result => result.errors) - .flat(), - ); - } + const updatedRecords = await collection.find(updatedQuery, { session: dbSession }).toArray(); - const updatedIs = updateResults.map(result => result.data); + if (metaObject.scriptAfterSave != null) { + tracingSpan?.addEvent('Running scriptAfterSave'); + await runScriptAfterSave({ script: metaObject.scriptAfterSave, data: updatedRecords, user, extraData: { original: existsRecords } }); + } - if (updatedIs.length > 0) { - if (MetaObject.Namespace.onUpdate != null) { - const hookRecords = await collection.find({ _id: { $in: updatedIs } }).toArray(); + if (MetaObject.Namespace.plan?.useExternalKonsistent !== true) { + try { + logger.debug('Processing Konsistent'); + tracingSpan?.addEvent('Processing sync Konsistent'); - const hookData = { - action: 'update', - ns: MetaObject.Namespace.ns, - documentName: document, - user: pick(user, ['_id', 'code', 'name', 'active', 'username', 'nickname', 'group', 'emails', 'locale']), - data: hookRecords, - }; + for await (const record of updatedRecords) { + const original = existsRecords.find(r => r._id === record._id); + const newRecord = omit(record, ['_id', '_createdAt', '_createdBy', '_updatedAt', '_updatedBy']); - const urls = [].concat(MetaObject.Namespace.onUpdate); - tracingSpan?.addEvent('Running onUpdate hooks', { urls }); + const changedProps = objectsDiff(original, newRecord); + await processIncomingChange(document, record, 'update', user, changedProps, dbSession); + } + } catch (e) { + logger.error(e, `Error on processIncomingChange ${document}: ${e.message}`); + tracingSpan?.addEvent('Error on Konsistent', { error: e.message }); + await dbSession.abortTransaction(); - await BluebirdPromise.mapSeries(urls, async url => { - try { - const hookUrl = url.replace('${dataId}', updatedIs.join(',')).replace('${documentId}', `${MetaObject.Namespace.ns}:${document}`); - const hookResponse = await fetch(hookUrl, { - method: 'POST', - body: JSON.stringify(hookData), - }); - if (hookResponse.status === 200) { - logger.info(`Hook ${hookUrl} executed successfully`); - } else { - logger.error(`Error on hook ${url}: ${hookResponse.statusText}`); + return errorReturn(`[${document}] Error on Konsistent: ${e.message}`); } - } catch (e) { - logger.error(e, `Error on hook ${url}: ${e.message}`); } - }); - } - const updatedQuery = { - _id: { - $in: updatedIs, - }, - }; + const responseData = updatedRecords.map(record => removeUnauthorizedDataForRead(access, record, user, metaObject)).map(record => dateToString(record)); - if (isObject(access.readFilter)) { - const readFilter = parseFilterObject(access.readFilter, metaObject, { user }); + if (emailsToSend.length > 0) { + tracingSpan?.addEvent('Sending emails'); - merge(updatedQuery, readFilter); - } - - const updatedRecords = await collection.find(updatedQuery).toArray(); - - if (metaObject.scriptAfterSave != null) { - tracingSpan?.addEvent('Running scriptAfterSave'); - await runScriptAfterSave({ script: metaObject.scriptAfterSave, data: updatedRecords, user, extraData: { original: existsRecords } }); - } - - if (MetaObject.Namespace.plan?.useExternalKonsistent !== true) { - try { - logger.debug('Processing Konsistent'); - tracingSpan?.addEvent('Processing sync Konsistent'); - for await (const record of updatedRecords) { - const original = existsRecords.find(r => r._id === record._id); - const newRecord = omit(record, ['_id', '_createdAt', '_createdBy', '_updatedAt', '_updatedBy']); - - const changedProps = objectsDiff(original, newRecord); - await processIncomingChange(document, record, 'update', user, changedProps); + const messagesCollection = MetaObject.Collections['Message']; + const now = DateTime.local().toJSDate(); + await messagesCollection.insertMany( + emailsToSend.map(email => + Object.assign( + {}, + { + _id: randomId(), + _createdAt: now, + _createdBy: pick(user, ['_id', 'name', 'group']), + _updatedAt: now, + _updatedBy: { ...pick(user, ['_id', 'name', 'group']), ts: now }, + }, + email, + ), + ), + { session: dbSession } + ); } - } catch (e) { - logger.error(e, `Error on processIncomingChange ${document}: ${e.message}`); - tracingSpan?.addEvent('Error on Konsistent', { error: e.message }); - } - } - - const responseData = updatedRecords.map(record => removeUnauthorizedDataForRead(access, record, user, metaObject)).map(record => dateToString(record)); - if (emailsToSend.length > 0) { - tracingSpan?.addEvent('Sending emails'); + return successReturn(responseData); + } + }); - const messagesCollection = MetaObject.Collections['Message']; - const now = DateTime.local().toJSDate(); - await messagesCollection.insertMany( - emailsToSend.map(email => - Object.assign( - {}, - { - _id: randomId(), - _createdAt: now, - _createdBy: pick(user, ['_id', 'name', 'group']), - _updatedAt: now, - _updatedBy: { ...pick(user, ['_id', 'name', 'group']), ts: now }, - }, - email, - ), - ), - ); + if (transactionResult != null && transactionResult.success != null) { + tracingSpan?.addEvent('Operation result', omit(transactionResult, ['data'])); + return transactionResult; } - - return successReturn(responseData); + } catch (e) { + tracingSpan?.addEvent('Error on transaction', { error: e.message }); + tracingSpan?.setAttribute({ error: e.message }); + logger.error(e, `Error on update ${MetaObject.Namespace.ns}.${document}: ${e.message}`); + } + finally { + tracingSpan?.addEvent('Ending session'); + dbSession.endSession(); } return errorReturn(`[${document}] Error on update, there is no affected record`); diff --git a/src/imports/konsistent/createHistory.js b/src/imports/konsistent/createHistory.js index 5412258a..81647c32 100644 --- a/src/imports/konsistent/createHistory.js +++ b/src/imports/konsistent/createHistory.js @@ -1,49 +1,36 @@ import { MetaObject } from "@imports/model/MetaObject"; import { logger } from "@imports/utils/logger"; import get from "lodash/get"; +import omitBy from "lodash/omitBy"; import pick from "lodash/pick"; import { v4 as uuidV4 } from 'uuid'; -export default async function createHistory(metaName, action, id, data, updatedBy, updatedAt, changed) { +export default async function createHistory(metaName, action, id, updatedBy, updatedAt, changed, dbSession) { // If data is empty or no update data is avaible then abort - if (Object.keys(data).length === 0 || updatedAt == null || updatedBy == null) { + if (Object.keys(changed).length === 0 || updatedAt == null || updatedBy == null) { return; } const changeId = uuidV4(); - const historyData = {}; - const meta = MetaObject.Meta[metaName]; - // Remove fields marked to ignore history - for (let key in changed) { - const value = data[key]; - const field = meta.fields[key]; - if (get(field, 'ignoreHistory', false) !== true) { - historyData[key] = value; - } - } - - // Get history collection const history = MetaObject.Collections[`${metaName}.History`]; if (!history) { return logger.error(`Can't get History collection from ${metaName}`); } - const historyQuery = { _id: changeId }; - - const userDetailFields = ["_id"].concat(get(meta, "fields._user.detailFields", ["name", "active"])); - + const userDetailFields = ["_id"].concat(get(meta, "fields._user.descriptionFields", ["name", "active"])); const historyItem = { dataId: id, createdAt: updatedAt, createdBy: pick(updatedBy, userDetailFields), - data: historyData, + data: omitBy(changed, (value, key) => meta.fields[key]?.ignoreHistory), type: action, }; try { - await history.updateOne(historyQuery, { $set: historyItem, $setOnInsert: historyQuery }, { upsert: true }); + const historyQuery = { _id: changeId }; + await history.updateOne(historyQuery, { $set: historyItem, $setOnInsert: historyQuery }, { upsert: true, session: dbSession }); } catch (e) { logger.error(e, 'Error on create history'); } diff --git a/src/imports/konsistent/processIncomingChange.ts b/src/imports/konsistent/processIncomingChange.ts index 0ce54e64..aa5d904b 100644 --- a/src/imports/konsistent/processIncomingChange.ts +++ b/src/imports/konsistent/processIncomingChange.ts @@ -5,6 +5,7 @@ import * as References from './updateReferences'; import { DataDocument } from '@imports/types/data'; import omit from 'lodash/omit'; +import { ClientSession, MongoServerError } from 'mongodb'; type Action = 'create' | 'update' | 'delete'; @@ -13,26 +14,37 @@ const logTimeSpent = (startTime: [number, number], message: string) => { logger.debug(`${totalTime[0]}s ${totalTime[1] / 1000000}ms => ${message}`); }; -export default async function processIncomingChange(metaName: string, incomingChange: DataDocument, action: Action, user: object, changedProps: Record) { - try { - const keysToIgnore = ['_updatedAt', '_createdAt', '_deletedAt', '_updatedBy', '_createdBy', '_deletedBy']; - let startTime = process.hrtime(); +export default async function processIncomingChange( + metaName: string, + incomingChange: DataDocument, + action: Action, + user: object, + changedProps: Record, + dbSession?: ClientSession, +) { + const keysToIgnore = ['_updatedAt', '_createdAt', '_deletedAt', '_updatedBy', '_createdBy', '_deletedBy']; + let startTime = process.hrtime(); + try { if (action === 'update') { - await References.updateLookups(metaName, incomingChange._id, changedProps); + await References.updateLookups(metaName, incomingChange._id, changedProps, dbSession); logTimeSpent(startTime, `Updated lookup references for ${metaName}`); } await processReverseLookups(metaName, incomingChange._id, incomingChange, action); logTimeSpent(startTime, `Process'd reverse lookups for ${metaName}`); - await References.updateRelations(metaName, action, incomingChange._id, incomingChange); + await References.updateRelations(metaName, action, incomingChange._id, incomingChange, dbSession); logTimeSpent(startTime, `Updated relation references for ${metaName}`); - await createHistory(metaName, action, incomingChange._id, omit(incomingChange, keysToIgnore), user, new Date(), changedProps); + await createHistory(metaName, action, incomingChange._id, user, new Date(), omit(changedProps, keysToIgnore), dbSession); logTimeSpent(startTime, `Created history for ${metaName}`); - } catch (err) { - const error = err as Error; - logger.error(`Error processing incoming change for ${metaName}: ${error.message}`); + } catch (e) { + if ((e as MongoServerError).codeName === 'NoSuchTransaction') { + logger.trace('Transaction was already closed'); + return; + } + + throw e; } } diff --git a/src/imports/konsistent/processReverseLookups.js b/src/imports/konsistent/processReverseLookups.js index 0accb530..219b9799 100644 --- a/src/imports/konsistent/processReverseLookups.js +++ b/src/imports/konsistent/processReverseLookups.js @@ -5,7 +5,7 @@ import { logger } from '@imports/utils/logger'; import { MetaObject } from '@imports/model/MetaObject'; -export default async function processReverseLookups(metaName, id, data, action) { +export default async function processReverseLookups(metaName, id, data, action, dbSession) { let field; if (action === 'delete') { return; @@ -29,7 +29,7 @@ export default async function processReverseLookups(metaName, id, data, action) // Get all data to copty into lookups const query = { _id: id }; - const record = await collection.findOne(query); + const record = await collection.findOne(query, { session: dbSession }); if (!record) { return logger.error(`Record not found with _id [${id.valueOf()}] on document [${metaName}]`); @@ -74,7 +74,7 @@ export default async function processReverseLookups(metaName, id, data, action) reverseLookupUpdate.$pull[`${field.reverseLookup}`] = { _id: id }; } - const updateResult = await reverseLookupCollection.updateMany(reverseLookupQuery, reverseLookupUpdate); + const updateResult = await reverseLookupCollection.updateMany(reverseLookupQuery, reverseLookupUpdate, { session: dbSession }); if (updateResult.modifiedCount > 0) { logger.info(`∞ ${field.document}.${field.reverseLookup} - ${metaName} (${updateResult.modifiedCount})`); @@ -86,17 +86,15 @@ export default async function processReverseLookups(metaName, id, data, action) const value = {}; value[field.reverseLookup] = { _id: id }; - await copyDescriptionAndInheritedFields( - reverseLookupMeta.fields[field.reverseLookup], - value[field.reverseLookup], - record, - reverseLookupMeta, - action, - reverseLookupCollection, - value, - value, - [data[field.name]._id], - ); + await copyDescriptionAndInheritedFields({ + field: reverseLookupMeta.fields[field.reverseLookup], + record: record, + meta: reverseLookupMeta, + actionType: action, + objectOriginalValues: value, + objectNewValues: value, + idsToUpdate: [data[field.name]._id], + }, dbSession); // Mount query and update to create the reverse lookup reverseLookupQuery = { _id: data[field.name]._id }; @@ -113,7 +111,7 @@ export default async function processReverseLookups(metaName, id, data, action) } } - const reverseUpdateResult = await reverseLookupCollection.updateMany(reverseLookupQuery, reverseLookupUpdate); + const reverseUpdateResult = await reverseLookupCollection.updateMany(reverseLookupQuery, reverseLookupUpdate, { session: dbSession }); if (reverseUpdateResult.modifiedCount > 0) { logger.info(`∞ ${field.document}.${field.reverseLookup} < ${metaName} (${reverseUpdateResult.modifiedCount})`); diff --git a/src/imports/konsistent/updateReferences/lookupReference.js b/src/imports/konsistent/updateReferences/lookupReference.js index f60a82f7..0def97f6 100644 --- a/src/imports/konsistent/updateReferences/lookupReference.js +++ b/src/imports/konsistent/updateReferences/lookupReference.js @@ -11,7 +11,7 @@ import { logger } from '@imports/utils/logger'; import { getFieldNamesOfPaths } from '../utils'; import updateLookupReferences from './lookupReferences'; -async function getDescriptionAndInheritedFieldsToUpdate({ record, metaField, meta }) { +async function getDescriptionAndInheritedFieldsToUpdate({ record, metaField, meta, dbSession }) { const fieldsToUpdate = {} if (isArray(metaField.descriptionFields) && metaField.descriptionFields.length > 0) { @@ -37,10 +37,10 @@ async function getDescriptionAndInheritedFieldsToUpdate({ record, metaField, met const projection = convertStringOfFieldsSeparatedByCommaIntoObjectToFind(keysToFind); const Collection = MetaObject.Collections[lookupField.document]; - const lookupRecord = await Collection.find({ _id: { $in: [].concat(record[lookupField.name]).map(v => v._id) } }, { projection }).toArray(); + const lookupRecord = await Collection.find({ _id: { $in: [].concat(record[lookupField.name]).map(v => v._id) } }, { projection, session: dbSession }).toArray(); for await (const lookupRec of lookupRecord) { - const result = await getDescriptionAndInheritedFieldsToUpdate({ record: lookupRec, metaField: lookupField, meta }); + const result = await getDescriptionAndInheritedFieldsToUpdate({ record: lookupRec, metaField: lookupField, meta, dbSession }); if (lookupField.isList) { mergeWith(fieldsToUpdate, result, (objValue = [], srcValue = [], key) => /\$$/.test(key) ? [].concat(objValue, srcValue) : undefined); } else { @@ -58,7 +58,11 @@ async function getDescriptionAndInheritedFieldsToUpdate({ record, metaField, met return fieldsToUpdate; } -export default async function updateLookupReference(metaName, fieldName, field, record, relatedMetaName) { +export default async function updateLookupReference(metaName, fieldName, field, record, relatedMetaName, dbSession) { + if (dbSession?.hasEnded) { + return; + } + const meta = MetaObject.Meta[metaName]; if (!meta) { return logger.error(`MetaObject.Meta ${metaName} does not exists`); @@ -76,7 +80,7 @@ export default async function updateLookupReference(metaName, fieldName, field, } const query = { [`${fieldName}._id`]: record._id }; - const updateResult = await collection.updateMany(query, { $set: updateData }); + const updateResult = await collection.updateMany(query, { $set: updateData }, { session: dbSession }); if (updateResult.modifiedCount > 0) { logger.debug(`🔗 ${relatedMetaName} > ${metaName}.${fieldName} (${updateResult.modifiedCount})`); @@ -84,7 +88,7 @@ export default async function updateLookupReference(metaName, fieldName, field, const modified = await collection.find(query, { projection }).toArray(); await Promise.all(modified.map(async (modifiedRecord) => - updateLookupReferences(metaName, modifiedRecord._id, modifiedRecord) + updateLookupReferences(metaName, modifiedRecord._id, modifiedRecord, dbSession) )); } diff --git a/src/imports/konsistent/updateReferences/lookupReferences.js b/src/imports/konsistent/updateReferences/lookupReferences.js index 3c00ef58..368bd9b6 100644 --- a/src/imports/konsistent/updateReferences/lookupReferences.js +++ b/src/imports/konsistent/updateReferences/lookupReferences.js @@ -19,7 +19,7 @@ import { getFieldNamesOfPaths } from '../utils'; * @param {object} data * @returns {Promise} */ -export default async function updateLookupReferences(metaName, id, data) { +export default async function updateLookupReferences(metaName, id, data, dbSession) { const references = MetaObject.References[metaName]; if (!isObject(references) || size(keys(references.from)) === 0) { @@ -41,12 +41,10 @@ export default async function updateLookupReferences(metaName, id, data) { const field = fields[fieldName]; let keysToUpdate = [].concat(field.descriptionFields || [], field.inheritedFields || []).map(getFieldNamesOfPaths); - // Remove duplicated fields + // Remove duplicated fields & get only fields that were updated keysToUpdate = uniq(keysToUpdate); - // Get only keys that exists in references and list of updated keys keysToUpdate = intersection(keysToUpdate, updatedKeys); - // If there are common fields, add field to list of relations to be processed if (keysToUpdate.length > 0) { if (!referencesToUpdate[referenceDocumentName]) { referencesToUpdate[referenceDocumentName] = {}; @@ -60,18 +58,18 @@ export default async function updateLookupReferences(metaName, id, data) { return; } - const record = await collection.findOne({ _id: id }); + const record = await collection.findOne({ _id: id }, { session: dbSession }); if (!record) { return logger.error(`Can't find record ${id} from ${metaName}`); } logger.debug(`Updating references for ${metaName} - ${Object.keys(referencesToUpdate).join(", ")}`); - await BluebirdPromise.mapSeries(Object.keys(referencesToUpdate), async referenceDocumentName => { + await BluebirdPromise.map(Object.keys(referencesToUpdate), async referenceDocumentName => { const fields = referencesToUpdate[referenceDocumentName]; - await BluebirdPromise.mapSeries(Object.keys(fields), async fieldName => { + await BluebirdPromise.map(Object.keys(fields), async fieldName => { const field = fields[fieldName]; - return updateLookupReference(referenceDocumentName, fieldName, field, record, metaName); - }); - }); + return updateLookupReference(referenceDocumentName, fieldName, field, record, metaName, dbSession); + }, { concurrency: 5 }); + }, { concurrency: 5 }); } diff --git a/src/imports/konsistent/updateReferences/relationReference.ts b/src/imports/konsistent/updateReferences/relationReference.ts index f4d10a36..e4a10212 100644 --- a/src/imports/konsistent/updateReferences/relationReference.ts +++ b/src/imports/konsistent/updateReferences/relationReference.ts @@ -1,5 +1,5 @@ import BluebirdPromise from 'bluebird'; -import { Collection, Filter, UpdateFilter } from 'mongodb'; +import { ClientSession, Collection, Filter, UpdateFilter } from 'mongodb'; import isArray from 'lodash/isArray'; import isObject from 'lodash/isObject'; @@ -13,11 +13,15 @@ import { Relation } from '@imports/model/Relation'; import type { AggregatePipeline } from '@imports/types/mongo'; import { logger } from '@imports/utils/logger'; -export default async function updateRelationReference(metaName: string, relation: Relation, lookupId: string, documentName: string) { +export default async function updateRelationReference(metaName: string, relation: Relation, lookupId: string, documentName: string, dbSession?: ClientSession) { // Try to get metadata let e, query: Filter | undefined; const meta = MetaObject.Meta[metaName]; + if (dbSession?.hasEnded) { + return; + } + if (!meta) { logger.error(`Can't get meta of document ${metaName}`); return 0; @@ -89,7 +93,9 @@ export default async function updateRelationReference(metaName: string, relation const MetaObj = MetaObject.Meta[relation.document]; if (MetaObj.type !== 'document') return; - const aggregatorField = MetaObj.fields[Number(aggregator.field?.split('.')[0])]; + const aggregatorField = MetaObj.fields[aggregator.field?.split('.')[0] ?? '']; + if (aggregatorField == null) return; + ({ type } = aggregatorField); // If type is money ensure that field has .value @@ -125,11 +131,15 @@ export default async function updateRelationReference(metaName: string, relation group.$group.value[`$${aggregator.aggregator}`] = `$${aggregator.field}`; } + if (group.$group.currency == null) { + delete group.$group.currency; + } + pipeline.push(group); // Try to execute agg and log error if fails try { - const result = await collection.aggregate(pipeline, { cursor: { batchSize: 1 } }).toArray(); + const result = await collection.aggregate(pipeline, { cursor: { batchSize: 1 }, session: dbSession }).toArray(); // If result was an array with one item cotaining a property value if (isArray(result) && isObject(result[0]) && result[0].value) { @@ -148,6 +158,7 @@ export default async function updateRelationReference(metaName: string, relation } catch (error) { e = error as Error; logger.error(e, `Error on aggregate relation ${relation.document} on document ${metaName}: ${e.message}`); + throw e; } }); @@ -177,28 +188,23 @@ export default async function updateRelationReference(metaName: string, relation const updateQuery: Filter<{ _id: string }> = { _id: lookupId }; // Try to execute update query - try { - const { modifiedCount: affected } = await referenceCollection.updateOne(updateQuery, valuesToUpdate); - - // If there are affected records - if (affected > 0) { - // Log Status - logger.info(`∑ ${documentName} < ${metaName} (${affected})`); - - // And log all aggregatores for this status - Object.entries(relation.aggregators).forEach(([fieldName, aggregator]) => { - if (aggregator.field) { - logger.info(` ${documentName}.${fieldName} < ${aggregator.aggregator} ${metaName}.${aggregator.field}`); - } else { - logger.info(` ${documentName}.${fieldName} < ${aggregator.aggregator} ${metaName}`); - } - }); - } - return affected; - } catch (error1) { - logger.error(error1, 'Error on updateRelationReference'); + const { modifiedCount: affected } = await referenceCollection.updateOne(updateQuery, valuesToUpdate, { session: dbSession }); + + // If there are affected records + if (affected > 0) { + // Log Status + logger.info(`∑ ${documentName} < ${metaName} (${affected})`); + + // And log all aggregatores for this status + Object.entries(relation.aggregators).forEach(([fieldName, aggregator]) => { + if (aggregator.field) { + logger.info(` ${documentName}.${fieldName} < ${aggregator.aggregator} ${metaName}.${aggregator.field}`); + } else { + logger.info(` ${documentName}.${fieldName} < ${aggregator.aggregator} ${metaName}`); + } + }); } - return 0; + return affected; } diff --git a/src/imports/konsistent/updateReferences/relationReferences.ts b/src/imports/konsistent/updateReferences/relationReferences.ts index 2ca4c46f..06a6a67e 100644 --- a/src/imports/konsistent/updateReferences/relationReferences.ts +++ b/src/imports/konsistent/updateReferences/relationReferences.ts @@ -13,12 +13,13 @@ import { MetaObject } from '@imports/model/MetaObject'; import type { Relation } from '@imports/model/Relation'; import { DataDocument, HistoryDocument } from '@imports/types/data'; import { logger } from '@imports/utils/logger'; -import { Collection, FindOptions } from 'mongodb'; +import { ClientSession, Collection, FindOptions } from 'mongodb'; import updateRelationReference from './relationReference'; type Action = 'update' | 'create' | 'delete'; +const CONCURRENCY = 5; -export default async function updateRelationReferences(metaName: string, action: Action, id: string, data: Record) { +export default async function updateRelationReferences(metaName: string, action: Action, id: string, data: Record, dbSession?: ClientSession) { // Get references from meta let relation, relations, relationsFromDocumentName; const references = MetaObject.References[metaName]; @@ -90,7 +91,7 @@ export default async function updateRelationReferences(metaName: string, action: } // Find record with all information, not only udpated data, to calc aggregations - const record = await (collection as unknown as Collection).findOne({ _id: id }); + const record = await (collection as unknown as Collection).findOne({ _id: id }, { session: dbSession }); // If no record was found log error and abort if (!record) { @@ -98,63 +99,76 @@ export default async function updateRelationReferences(metaName: string, action: } // # Iterate over relations to process - await BluebirdPromise.mapSeries(Object.keys(referencesToUpdate), async referenceDocumentName => { - relations = referencesToUpdate[referenceDocumentName]; - await BluebirdPromise.mapSeries(relations, async relation => { - var value; - const relationLookupMeta = MetaObject.Meta[relation.document]; - // Get lookup id from record - const lookupId: string[] = []; - if (has(record, `${relation.lookup}._id`)) { - lookupId.push(get(record, `${relation.lookup}._id`, '') as string); - } else if (get(relationLookupMeta, `fields.${relation.lookup}.isList`) === true && Array.isArray(record[relation.lookup])) { - for (value of record[relation.lookup] as Array>) { - if (value != null && value._id != null) { - lookupId.push(value._id); + await BluebirdPromise.map( + Object.keys(referencesToUpdate), + async referenceDocumentName => { + relations = referencesToUpdate[referenceDocumentName]; + await BluebirdPromise.map( + relations, + async relation => { + var value; + const relationLookupMeta = MetaObject.Meta[relation.document]; + // Get lookup id from record + const lookupId: string[] = []; + if (has(record, `${relation.lookup}._id`)) { + lookupId.push(get(record, `${relation.lookup}._id`, '') as string); + } else if (get(relationLookupMeta, `fields.${relation.lookup}.isList`) === true && Array.isArray(record[relation.lookup])) { + for (value of record[relation.lookup] as Array>) { + if (value != null && value._id != null) { + lookupId.push(value._id); + } + } } - } - } - - // If action is update and the lookup field of relation was updated go to hitory to update old relation - if (lookupId.length > 0 && action === 'update' && has(data, `${relation.lookup}._id`)) { - // Try to get history model - const historyCollection = MetaObject.Collections[`${metaName}.History`]; - - if (historyCollection == null) { - logger.error(`Can't get model for document ${metaName}.History`); - } - // Define query of history with data id - const historyQuery: Record = { dataId: id.toString() }; + // If action is update and the lookup field of relation was updated go to hitory to update old relation + if (lookupId.length > 0 && action === 'update' && has(data, `${relation.lookup}._id`)) { + // Try to get history model + const historyCollection = MetaObject.Collections[`${metaName}.History`]; - // Add condition to get aonly data with changes on lookup field - historyQuery[`data.${relation.lookup}`] = { $exists: true }; - - // And sort DESC to get only last data - const historyOptions: FindOptions = { sort: { createdAt: -1 } }; - - // User findOne to get only one data - const historyRecord = await historyCollection.findOne(historyQuery, historyOptions); + if (historyCollection == null) { + logger.error(`Can't get model for document ${metaName}.History`); + } - // If there are record - if (historyRecord) { - // Then get lookupid to execute update on old relation - let historyLookupId: string[] = new Array().concat(get(historyRecord, `data.${relation.lookup}._id`, [])); - if (get(relationLookupMeta, `fields.${relation.lookup}.isList`) === true && isArray(historyRecord.data[relation.lookup])) { - historyLookupId = []; - for (value of historyRecord.data[relation.lookup] as Array>) { - value._id != null && historyLookupId.push(value._id); + // Define query of history with data id + const historyQuery: Record = { dataId: id.toString() }; + + // Add condition to get aonly data with changes on lookup field + historyQuery[`data.${relation.lookup}`] = { $exists: true }; + + // And sort DESC to get only last data + const historyOptions: FindOptions = { sort: { createdAt: -1 }, session: dbSession }; + + const historyRecord = await historyCollection.findOne(historyQuery, historyOptions); + + // If there are record + if (historyRecord) { + // Then get lookupid to execute update on old relation + let historyLookupId: string[] = new Array().concat(get(historyRecord, `data.${relation.lookup}._id`, [])); + if (get(relationLookupMeta, `fields.${relation.lookup}.isList`) === true && isArray(historyRecord.data[relation.lookup])) { + historyLookupId = []; + for (value of historyRecord.data[relation.lookup] as Array>) { + value._id != null && historyLookupId.push(value._id); + } + } + + await BluebirdPromise.map( + historyLookupId, + async historyLookupIdItem => { + return updateRelationReference(metaName, relation, historyLookupIdItem, referenceDocumentName, dbSession); + }, + { concurrency: CONCURRENCY }, + ); } } - await BluebirdPromise.mapSeries(historyLookupId, async historyLookupIdItem => { - return updateRelationReference(metaName, relation, historyLookupIdItem, referenceDocumentName); + // Execute update of relations into new value + await BluebirdPromise.map(lookupId, lookupIdItem => updateRelationReference(metaName, relation, lookupIdItem, referenceDocumentName, dbSession), { + concurrency: CONCURRENCY, }); - } - } - - // Execute update of relations into new value - await BluebirdPromise.mapSeries(lookupId, lookupIdItem => updateRelationReference(metaName, relation, lookupIdItem, referenceDocumentName)); - }); - }); + }, + { concurrency: CONCURRENCY }, + ); + }, + { concurrency: CONCURRENCY }, + ); } diff --git a/src/imports/meta/copyDescriptionAndInheritedFields.js b/src/imports/meta/copyDescriptionAndInheritedFields.js index ccd1c7b2..a816f15e 100644 --- a/src/imports/meta/copyDescriptionAndInheritedFields.js +++ b/src/imports/meta/copyDescriptionAndInheritedFields.js @@ -2,7 +2,7 @@ import isArray from 'lodash/isArray'; import pick from 'lodash/pick'; import { validateAndProcessValueFor } from '../meta/validateAndProcessValueFor'; -export async function copyDescriptionAndInheritedFields({ field, record, meta, actionType, objectOriginalValues, objectNewValues, idsToUpdate }) { +export async function copyDescriptionAndInheritedFields({ field, record, meta, actionType, objectOriginalValues, objectNewValues, idsToUpdate }, dbSession) { const value = { _id: record._id, }; @@ -42,7 +42,7 @@ export async function copyDescriptionAndInheritedFields({ field, record, meta, a objectOriginalValues, objectNewValues, idsToUpdate, - }); + }, dbSession); if (validateResult.success === true) { Object.assign(objectNewValues, { [inheritedField.fieldName]: validateResult.data }); } @@ -59,7 +59,7 @@ export async function copyDescriptionAndInheritedFields({ field, record, meta, a objectOriginalValues, objectNewValues, idsToUpdate, - }); + }, dbSession); if (validateResult.success === true) { Object.assign(objectNewValues, { [inheritedField.fieldName]: validateResult.data }); } diff --git a/src/imports/meta/getNextCode/index.ts b/src/imports/meta/getNextCode/index.ts index 2e4ef4f7..19e0057f 100644 --- a/src/imports/meta/getNextCode/index.ts +++ b/src/imports/meta/getNextCode/index.ts @@ -1,12 +1,12 @@ import get from 'lodash/get'; import { MetaObject } from '@imports/model/MetaObject'; -import { logger } from '../../utils/logger'; -import { Filter } from 'mongodb'; +import { logger } from '@imports/utils/logger'; +import { ClientSession, Filter, FindOneAndUpdateOptions } from 'mongodb'; const GENERATE_CODE_MAX_DEPTH = 1000; -export async function getNextCode(documentName: string, fieldName: string) { +export async function getNextCode(documentName: string, fieldName: string, dbSession?: ClientSession) { if (!fieldName) { fieldName = 'code'; } @@ -23,9 +23,10 @@ export async function getNextCode(documentName: string, fieldName: string) { }, }; - const options = { + const options: FindOneAndUpdateOptions = { upsert: true, - returnNewDocument: true, + returnDocument: 'after', + session: dbSession, }; // Try to get next code @@ -39,7 +40,7 @@ export async function getNextCode(documentName: string, fieldName: string) { }> => { const autoNumberResult = await autoNumberCollection.findOneAndUpdate(query as any, update, options); const nextVal = get(autoNumberResult, 'next_val', 1); - const existingCodes = await documentCollection.countDocuments({ [fieldName]: nextVal }); + const existingCodes = await documentCollection.countDocuments({ [fieldName]: nextVal }, { session: dbSession }); if (existingCodes === 0) { return { success: true, diff --git a/src/imports/meta/loadMetaObjects.ts b/src/imports/meta/loadMetaObjects.ts index 986465ff..98027ab5 100644 --- a/src/imports/meta/loadMetaObjects.ts +++ b/src/imports/meta/loadMetaObjects.ts @@ -46,13 +46,11 @@ async function registerMeta(meta: MetaObjectType) { }; } - if (MetaObject.Collections[meta.name] == null) { - MetaObject.Collections[meta.name] = db.collection(`${meta.collection ?? meta.name}`); - MetaObject.Collections[`${meta.name}.Comment`] = db.collection(`${meta.collection ?? meta.name}.Comment`); - MetaObject.Collections[`${meta.name}.History`] = db.collection(`${meta.collection ?? meta.name}.History`); - MetaObject.Collections[`${meta.name}.Trash`] = db.collection(`${meta.collection ?? meta.name}.Trash`); - MetaObject.Collections[`${meta.name}.AutoNumber`] = db.collection(`${meta.collection ?? meta.name}.AutoNumber`); - } + MetaObject.Collections[meta.name] = db.collection(`${meta.collection ?? meta.name}`); + MetaObject.Collections[`${meta.name}.Comment`] = db.collection(`${meta.collection ?? meta.name}.Comment`); + MetaObject.Collections[`${meta.name}.History`] = db.collection(`${meta.collection ?? meta.name}.History`); + MetaObject.Collections[`${meta.name}.Trash`] = db.collection(`${meta.collection ?? meta.name}.Trash`); + MetaObject.Collections[`${meta.name}.AutoNumber`] = db.collection(`${meta.collection ?? meta.name}.AutoNumber`); await applyIndexes(meta); } diff --git a/src/imports/meta/validateAndProcessValueFor.js b/src/imports/meta/validateAndProcessValueFor.js index 8143dfd7..54bc7491 100644 --- a/src/imports/meta/validateAndProcessValueFor.js +++ b/src/imports/meta/validateAndProcessValueFor.js @@ -29,7 +29,8 @@ import { copyDescriptionAndInheritedFields } from '../meta/copyDescriptionAndInh import { removeInheritedFields } from '../meta/removeInheritedFields'; import { getNextCode } from './getNextCode'; -import { errorReturn } from '@imports/utils/return'; +import { errorReturn, successReturn } from '@imports/utils/return'; +import Bluebird from 'bluebird'; import { BCRYPT_SALT_ROUNDS } from '../consts'; const regexUtils = { @@ -73,7 +74,7 @@ const VALID_OPERATORS = [ const ALLOWED_CURRENCIES = ['BRL']; -export async function validateAndProcessValueFor({ meta, fieldName, value, actionType, objectOriginalValues, objectNewValues, idsToUpdate }) { +export async function validateAndProcessValueFor({ meta, fieldName, value, actionType, objectOriginalValues, objectNewValues, idsToUpdate }, dbSession) { if (meta == null) { return errorReturn(`MetaObject.Meta does not exists`); } @@ -352,13 +353,7 @@ export async function validateAndProcessValueFor({ meta, fieldName, value, actio switch (field.type) { case 'boolean': const booleanResult = mustBeBoolean(value); - if (booleanResult.success === false) { - return booleanResult; - } - return { - success: true, - data: value, - }; + return booleanResult.success ? successReturn(value) : booleanResult; case 'number': case 'percentage': @@ -368,31 +363,14 @@ export async function validateAndProcessValueFor({ meta, fieldName, value, actio } if (isNumber(field.maxValue) && value > field.maxValue) { - return { - success: false, - errors: [ - { - message: `Value for field ${fieldName} must be less than ${field.maxValue}`, - }, - ], - }; + return errorReturn(`Value for field ${fieldName} must be less than ${field.maxValue}`) } if (isNumber(field.minValue) && value < field.minValue) { - return { - success: false, - errors: [ - { - message: `Value for field ${fieldName} must be greater than ${field.minValue}`, - }, - ], - }; + return errorReturn(`Value for field ${fieldName} must be greater than ${field.minValue}`) } - return { - success: true, - data: value, - }; + return successReturn(value); case 'picklist': if (isNumber(field.maxSelected) && field.maxSelected > 1) { @@ -401,45 +379,21 @@ export async function validateAndProcessValueFor({ meta, fieldName, value, actio return pickListResult; } if (value.length > field.maxSelected) { - return { - success: false, - errors: [ - { - message: `Value for field ${fieldName} must be an array with max of ${field.maxSelected} item(s)`, - }, - ], - }; + return errorReturn(`Value for field ${fieldName} must be an array with max of ${field.maxSelected} item(s)`); } } if (isNumber(field.minSelected) && field.minSelected > 0) { if (value.length < field.minSelected) { - return { - success: false, - errors: [ - { - message: `Value for field ${fieldName} must be an array with min of ${field.minSelected} item(s)`, - }, - ], - }; + return errorReturn(`Value for field ${fieldName} must be an array with min of ${field.minSelected} item(s)`); } } if ([].concat(value).some(v => !Object.keys(field.options).includes(v))) { - return { - success: false, - errors: [ - { - message: `Value ${value} for field ${fieldName} is invalid`, - }, - ], - }; + return errorReturn(`Value ${value} for field ${fieldName} is invalid`) } - return { - success: true, - data: value, - }; + return successReturn(value); case 'text': case 'richText': @@ -778,10 +732,7 @@ export async function validateAndProcessValueFor({ meta, fieldName, value, actio }; } - return { - success: true, - data: value, - }; + return successReturn(value); case 'password': const passwordStringResult = mustBeString(value); @@ -793,10 +744,7 @@ export async function validateAndProcessValueFor({ meta, fieldName, value, actio const hashPassword = await bcryptHash(hashedPassword, BCRYPT_SALT_ROUNDS); - return { - success: true, - data: hashPassword, - }; + return successReturn(hashPassword); case 'encrypted': const encryptedStringResult = mustBeString(value); @@ -806,26 +754,20 @@ export async function validateAndProcessValueFor({ meta, fieldName, value, actio value = createHash('md5').update(value).digest('hex'); - return { - success: true, - data: value, - }; + return successReturn(value); case 'autoNumber': if (actionType === 'update') { value = undefined; } else { - const nextCodeResult = await getNextCode(meta.name, fieldName); + const nextCodeResult = await getNextCode(meta.name, fieldName, dbSession); if (nextCodeResult.success === false) { return nextCodeResult; } value = nextCodeResult.data; } - return { - success: true, - data: value, - }; + return successReturn(value); case 'address': const addressObjectResult = mustBeObject(value); @@ -886,10 +828,7 @@ export async function validateAndProcessValueFor({ meta, fieldName, value, actio }; } - return { - success: true, - data: value, - }; + return successReturn(value); case 'filter': const objectFilterResult = mustBeObject(value); @@ -904,10 +843,7 @@ export async function validateAndProcessValueFor({ meta, fieldName, value, actio value = stringToDate(value); - return { - success: true, - data: value, - }; + return successReturn(value); case 'composite': const compositeObjectResult = mustBeObject(value); @@ -919,14 +855,7 @@ export async function validateAndProcessValueFor({ meta, fieldName, value, actio const referenceMeta = MetaObject.Meta[field.objectRefId]; if (referenceMeta == null) { - return { - success: false, - errors: [ - { - message: `Document ${field.objectRefId} not found`, - }, - ], - }; + return errorReturn(`Document ${field.objectRefId} not found`); } const referenceDataValidationResults = await Promise.all( @@ -942,14 +871,11 @@ export async function validateAndProcessValueFor({ meta, fieldName, value, actio idsToUpdate }; - const validationResult = await validateAndProcessValueFor(params); + const validationResult = await validateAndProcessValueFor(params, dbSession); if (validationResult.success === false) { return validationResult; } - return { - success: true, - data: { key, value: validationResult.data }, - }; + return successReturn({ key, value: validationResult.data }); }), ); @@ -963,10 +889,7 @@ export async function validateAndProcessValueFor({ meta, fieldName, value, actio }, {}); } - return { - success: true, - data: value, - }; + return successReturn(value); case 'lookup': case 'inheritLookup': @@ -985,7 +908,7 @@ export async function validateAndProcessValueFor({ meta, fieldName, value, actio return errorReturn(`Collection ${field.document} not found`); } - const record = await lookupCollection.findOne({ _id: value._id }); + const record = await lookupCollection.findOne({ _id: value._id }, { session: dbSession }); if (record == null) { return errorReturn(`Record not found for field ${fieldName} with _id [${value._id}] on document [${field.document}]`); @@ -999,7 +922,7 @@ export async function validateAndProcessValueFor({ meta, fieldName, value, actio objectOriginalValues, objectNewValues, idsToUpdate, - }); + }, dbSession); return inheritedFieldsResult; @@ -1012,22 +935,12 @@ export async function validateAndProcessValueFor({ meta, fieldName, value, actio } const unauthorizedKeys = ['key', 'name', 'size', 'created', 'etag', 'headers', 'kind', 'last_modified', 'description', 'label', 'wildcard']; - return { - success: true, - data: removeUnauthorizedKeys(value, unauthorizedKeys), - }; + return successReturn(removeUnauthorizedKeys(value, unauthorizedKeys)); default: logger.error(`Field ${fieldName} of type ${field.type} can not be validated`); - return { - success: false, - errors: [ - { - message: `Field ${fieldName} of type ${field.type} can not be validated`, - }, - ], - }; + return errorReturn(`Field ${fieldName} of type ${field.type} can not be validated`); } }; @@ -1036,24 +949,14 @@ export async function validateAndProcessValueFor({ meta, fieldName, value, actio } if (value == null) { - return { - success: false, - errors: [ - { - message: `Value for field ${fieldName} must be array`, - }, - ], - }; + return errorReturn(`Value for field ${fieldName} must be array`); } - const arrayResult = await Promise.all([].concat(value).map(validate)); + const arrayResult = await Bluebird.map([].concat(value), validate, { concurrency: 10 }); if (arrayResult.some(v => v.success === false)) { return arrayResult.find(v => v.success === false); } - return { - success: true, - data: arrayResult.map(v => v.data), - }; + return successReturn(arrayResult.map(v => v.data)); } From 8bf9aa2ef13c5d1f8b609e6b6e4fdb86856fa678 Mon Sep 17 00:00:00 2001 From: Leonardo Viva Date: Mon, 8 Jul 2024 17:37:07 -0300 Subject: [PATCH 08/20] feat: accept ids array & fetch all together if possible --- .../updateReferences/lookupReference.js | 4 +--- .../updateReferences/lookupReferences.js | 17 +++++++++++++---- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/imports/konsistent/updateReferences/lookupReference.js b/src/imports/konsistent/updateReferences/lookupReference.js index 0def97f6..4fa22e46 100644 --- a/src/imports/konsistent/updateReferences/lookupReference.js +++ b/src/imports/konsistent/updateReferences/lookupReference.js @@ -87,9 +87,7 @@ export default async function updateLookupReference(metaName, fieldName, field, const projection = convertStringOfFieldsSeparatedByCommaIntoObjectToFind(Object.keys(updateData).join()); const modified = await collection.find(query, { projection }).toArray(); - await Promise.all(modified.map(async (modifiedRecord) => - updateLookupReferences(metaName, modifiedRecord._id, modifiedRecord, dbSession) - )); + await updateLookupReferences(metaName, modified.map(m => m._id), updateData, dbSession); } return updateResult.modifiedCount; diff --git a/src/imports/konsistent/updateReferences/lookupReferences.js b/src/imports/konsistent/updateReferences/lookupReferences.js index 368bd9b6..397df01f 100644 --- a/src/imports/konsistent/updateReferences/lookupReferences.js +++ b/src/imports/konsistent/updateReferences/lookupReferences.js @@ -23,6 +23,7 @@ export default async function updateLookupReferences(metaName, id, data, dbSessi const references = MetaObject.References[metaName]; if (!isObject(references) || size(keys(references.from)) === 0) { + logger.debug(`No references from ${metaName}`); return; } @@ -55,21 +56,29 @@ export default async function updateLookupReferences(metaName, id, data, dbSessi } if (Object.keys(referencesToUpdate).length === 0) { + logger.debug(`No references to update for ${metaName}`); return; } - const record = await collection.findOne({ _id: id }, { session: dbSession }); + if (Array.isArray(id) && id.length > 1) { + const records = await collection.find({ _id: { $in: id } }, { session: dbSession }).toArray(); + return await Promise.all(records.map(record => processReferences({ referencesToUpdate, metaName, record, dbSession }))); + } + + const record = await collection.findOne({ _id: [].concat(id)[0] }, { session: dbSession }); if (!record) { return logger.error(`Can't find record ${id} from ${metaName}`); } logger.debug(`Updating references for ${metaName} - ${Object.keys(referencesToUpdate).join(", ")}`); - - await BluebirdPromise.map(Object.keys(referencesToUpdate), async referenceDocumentName => { + return await processReferences({ referencesToUpdate, metaName, record, dbSession }); +} +const processReferences = async ({ referencesToUpdate, metaName, record, dbSession }) => { + return await BluebirdPromise.map(Object.keys(referencesToUpdate), async referenceDocumentName => { const fields = referencesToUpdate[referenceDocumentName]; await BluebirdPromise.map(Object.keys(fields), async fieldName => { const field = fields[fieldName]; return updateLookupReference(referenceDocumentName, fieldName, field, record, metaName, dbSession); }, { concurrency: 5 }); }, { concurrency: 5 }); -} +} \ No newline at end of file From 73f6fa3a71f5209241e34ab3c4650540c535a166 Mon Sep 17 00:00:00 2001 From: Leonardo Viva Date: Wed, 10 Jul 2024 12:30:56 -0300 Subject: [PATCH 09/20] chore: param type --- src/imports/konsistent/updateReferences/lookupReferences.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/imports/konsistent/updateReferences/lookupReferences.js b/src/imports/konsistent/updateReferences/lookupReferences.js index 397df01f..38b1ba5a 100644 --- a/src/imports/konsistent/updateReferences/lookupReferences.js +++ b/src/imports/konsistent/updateReferences/lookupReferences.js @@ -15,7 +15,7 @@ import { getFieldNamesOfPaths } from '../utils'; * When some document changes, verify if it's a lookup in some other document. * If it is, update description & inherited fields in all related documents. * @param {string} metaName - * @param {string} id + * @param {string | string[]} id * @param {object} data * @returns {Promise} */ From ae15ce5022497b4579ee80f171647c0654ac9003 Mon Sep 17 00:00:00 2001 From: Leonardo Viva Date: Wed, 9 Oct 2024 08:42:57 -0300 Subject: [PATCH 10/20] fix: reverse lookup count --- src/imports/konsistent/processReverseLookups.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/imports/konsistent/processReverseLookups.js b/src/imports/konsistent/processReverseLookups.js index 219b9799..7682f05f 100644 --- a/src/imports/konsistent/processReverseLookups.js +++ b/src/imports/konsistent/processReverseLookups.js @@ -17,7 +17,7 @@ export default async function processReverseLookups(metaName, id, data, action, let reverseLookupCount = 0; for (var fieldName in meta.fields) { field = meta.fields[fieldName]; - if (field.type === 'lookup' && !field.reverseLookup && data[field.name] !== undefined) { + if (field.type === 'lookup' && field.reverseLookup && data[field.name] !== undefined) { reverseLookupCount++; } } From 099b3df8ec85d11a0d0c87196aee8886668958d1 Mon Sep 17 00:00:00 2001 From: Leonardo Viva Date: Thu, 10 Oct 2024 15:52:02 -0300 Subject: [PATCH 11/20] fix: ccorrectly inherit fields --- src/imports/data/data.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/imports/data/data.js b/src/imports/data/data.js index e78dc278..194d1ea1 100644 --- a/src/imports/data/data.js +++ b/src/imports/data/data.js @@ -1204,7 +1204,7 @@ export async function update({ authTokenId, document, data, contextUser, tracing value: data.data[fieldName], actionType: 'update', objectOriginalValues: record, - objectNewValues: data.data, + objectNewValues: bodyData, idsToUpdate: query._id.$in, }, dbSession); if (result.success === false) { @@ -1228,7 +1228,7 @@ export async function update({ authTokenId, document, data, contextUser, tracing if (metaObject.validationScript != null) { tracingSpan?.addEvent('Running validation script'); - const validationScriptResult = await processValidationScript({ script: metaObject.validationScript, validationData: metaObject.validationData, fullData: extend({}, record, data.data), user }); + const validationScriptResult = await processValidationScript({ script: metaObject.validationScript, validationData: metaObject.validationData, fullData: extend({}, record, bodyData), user }); if (validationScriptResult.success === false) { logger.error(validationScriptResult, `Update - Script Validation Error - ${validationScriptResult.reason}`); return validationScriptResult; From 463058de5388219182f4040fd60ba748932ae746 Mon Sep 17 00:00:00 2001 From: Cadu Gomes Date: Tue, 22 Oct 2024 16:48:21 -0300 Subject: [PATCH 12/20] feat: apply meta route --- src/server/routes/api/document/index.ts | 49 +++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/src/server/routes/api/document/index.ts b/src/server/routes/api/document/index.ts index face656e..dde12a1b 100644 --- a/src/server/routes/api/document/index.ts +++ b/src/server/routes/api/document/index.ts @@ -4,6 +4,8 @@ import fp from 'fastify-plugin'; import { getUserFromRequest } from '@imports/auth/getUser'; import { getDocument } from '@imports/document'; import { logger } from '@imports/utils/logger'; +import { db } from '@imports/database'; +import { ObjectId } from 'mongodb'; const documentAPi: FastifyPluginCallback = async fastify => { fastify.get<{ Params: { name: string } }>('/api/document/:name', async (req, reply) => { @@ -40,6 +42,53 @@ const documentAPi: FastifyPluginCallback = async fastify => { return reply.status(500).send('Internal server error'); } }); + + fastify.put<{ Params: { id: string } }>('/api/document/:id', async (req, reply) => { + if (req.originalUrl == null || req.params == null) { + return reply.status(404).send('Not found'); + } + + const id = req.params.id; + + if (id == null) { + return reply.status(400).send('Bad request'); + } + + try { + const user = await getUserFromRequest(req); + + if (user == null || user.admin !== true) { + return reply.status(401).send('Unauthorized'); + } + + const document = req.body as object; + + if (document == null) { + return reply.status(400).send('Bad request'); + } + + const result = await db.collection('MetaObjects').replaceOne({ _id: new ObjectId(id) }, document, { upsert: true }); + + if (result.modifiedCount === 0) { + return reply.status(404).send('Invalid meta object'); + } + + if (result.upsertedCount === 1) { + return reply.status(201).send('Created'); + } + + if (result.matchedCount === 1) { + return reply.send('Updated'); + } + } catch (error) { + if (/^\[get-user\]/.test((error as Error).message)) { + return reply.status(401).send('Unauthorized'); + } + + logger.error(error, `Error getting document for ${id}`); + } + return reply.status(500).send('Internal server error'); + }); }; export default fp(documentAPi); From 13470c20c9e81f5d7959ea5c8f8211ed99c31f16 Mon Sep 17 00:00:00 2001 From: Cadu Gomes Date: Tue, 29 Oct 2024 13:21:41 -0300 Subject: [PATCH 13/20] feat: apply metas and config filter conditions --- .eslintrc.js | 1 + src/imports/meta/loadMetaObjects.ts | 28 ++++++-- src/imports/model/Filter.ts | 19 +++++- src/imports/model/Form.ts | 2 + src/imports/model/List.ts | 4 +- src/server/routes/api/document/index.ts | 86 ++++++++++++++++++++++--- 6 files changed, 123 insertions(+), 17 deletions(-) diff --git a/.eslintrc.js b/.eslintrc.js index 6c267991..a8cd937c 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -13,6 +13,7 @@ module.exports = { '@typescript-eslint/no-unsafe-call': 'off', 'no-case-declarations': 'off', '@typescript-eslint/no-explicit-any': 'warn', + '@typescript-eslint/ban-types': 'warn', }, settings: { 'import/parsers': { diff --git a/src/imports/meta/loadMetaObjects.ts b/src/imports/meta/loadMetaObjects.ts index 986465ff..1aef7075 100644 --- a/src/imports/meta/loadMetaObjects.ts +++ b/src/imports/meta/loadMetaObjects.ts @@ -95,18 +95,38 @@ async function dbLoad() { function dbWatch() { MetaObject.MetaObject.watch().on('change', async (change: any) => { if (change.operationType === 'delete') { - switch (change.fullDocumentBeforeChange.type) { + const id = change.documentKey._id as string; + + let meta: any = null; + + if (id in MetaObject.DisplayMeta) { + meta = MetaObject.DisplayMeta[id]; + } + + if (id in MetaObject.Meta) { + meta = MetaObject.Meta[id]; + } + + if (id in MetaObject.Access) { + meta = MetaObject.Access[id]; + } + + if (id in MetaObject.MetaByCollection) { + meta = MetaObject.MetaByCollection[id]; + } + + switch (meta?.type) { case 'access': - unset(MetaObject.Access, change.fullDocumentBeforeChange._id); + unset(MetaObject.Access, id); break; case 'document': case 'composite': - deregisterMeta(change.fullDocumentBeforeChange); + deregisterMeta(meta); break; case 'pivot': case 'view': case 'list': - unset(MetaObject.DisplayMeta, change.fullDocumentBeforeChange._id); + unset(MetaObject.DisplayMeta, id); break; } } else if (change.operationType === 'insert') { diff --git a/src/imports/model/Filter.ts b/src/imports/model/Filter.ts index c35f53a2..3a380fe0 100644 --- a/src/imports/model/Filter.ts +++ b/src/imports/model/Filter.ts @@ -3,20 +3,33 @@ import { z } from 'zod'; export const Condition = z.object({ term: z.string(), operator: z.string(), - value: z.union([z.string(), z.number(), z.boolean(), z.array(z.union([z.string(), z.number(), z.boolean()]))]), + value: z + .union([z.string(), z.number(), z.boolean(), z.any(), z.array(z.union([z.string(), z.number(), z.boolean()]))]) + .optional() + .nullable(), editable: z.boolean().optional(), disabled: z.boolean().optional(), + sort: z.number().optional(), + style: z + .object({ + // renderAs: z.enum(['checbox', 'lookupfield', 'datetimefield', 'textfield', 'radiobox']).optional(), + renderAs: z.string().optional(), // o ideal seria mapear todos as possibilidades + columns: z.number().optional(), + hideOnDisable: z.boolean().optional(), + customLabel: z.string().optional(), + }) + .optional(), }); export const KonFilter = z.object({ match: z.literal('and').or(z.literal('or')), - conditions: z.array(Condition).optional(), + conditions: z.array(Condition).optional().or(z.record(Condition).optional()), textSearch: z.string().optional(), filters: z .array( z.object({ match: z.literal('and').or(z.literal('or')).optional(), - conditions: z.array(Condition).optional(), + conditions: z.array(Condition).optional().or(z.record(Condition).optional()), textSearch: z.string().optional(), }), ) diff --git a/src/imports/model/Form.ts b/src/imports/model/Form.ts index 0f0264cf..832b1898 100644 --- a/src/imports/model/Form.ts +++ b/src/imports/model/Form.ts @@ -12,6 +12,8 @@ export const FormSchema = z.object({ icon: z.string().optional(), visuals: z.array(VisualsSchema).optional(), collection: z.string().optional(), + namespace: z.array(z.string()).optional(), + parent: z.string().optional(), }); export type Form = z.infer; diff --git a/src/imports/model/List.ts b/src/imports/model/List.ts index b061d550..3b71b4ac 100644 --- a/src/imports/model/List.ts +++ b/src/imports/model/List.ts @@ -1,6 +1,6 @@ import { z } from 'zod'; -import { KonFilter } from './Filter'; import { LabelSchema } from './Label'; +import { KonFilter } from './Filter'; export const ListSchema = z.object({ type: z.literal('list'), @@ -11,6 +11,8 @@ export const ListSchema = z.object({ name: z.string(), label: LabelSchema, plurals: LabelSchema, + loadDataAtOpen: z.boolean().optional(), + namespace: z.array(z.string()).optional(), icon: z.string().optional(), refreshRate: z.object({ options: z.array(z.number()), diff --git a/src/server/routes/api/document/index.ts b/src/server/routes/api/document/index.ts index dde12a1b..7567221f 100644 --- a/src/server/routes/api/document/index.ts +++ b/src/server/routes/api/document/index.ts @@ -4,8 +4,10 @@ import fp from 'fastify-plugin'; import { getUserFromRequest } from '@imports/auth/getUser'; import { getDocument } from '@imports/document'; import { logger } from '@imports/utils/logger'; -import { db } from '@imports/database'; -import { ObjectId } from 'mongodb'; +import { WithoutId } from 'mongodb'; +import { loadMetaObjects } from '@imports/meta/loadMetaObjects'; +import { MetaObject } from '@imports/model/MetaObject'; +import { MetaObjectSchema, MetaObjectType } from '@imports/types/metadata'; const documentAPi: FastifyPluginCallback = async fastify => { fastify.get<{ Params: { name: string } }>('/api/document/:name', async (req, reply) => { @@ -43,7 +45,7 @@ const documentAPi: FastifyPluginCallback = async fastify => { } }); - fastify.put<{ Params: { id: string } }>('/api/document/:id', async (req, reply) => { + fastify.post<{ Params: { id: string } }>('/api/document/:id', async (req, reply) => { if (req.originalUrl == null || req.params == null) { return reply.status(404).send('Not found'); } @@ -61,31 +63,97 @@ const documentAPi: FastifyPluginCallback = async fastify => { return reply.status(401).send('Unauthorized'); } - const document = req.body as object; + const document = req.body as MetaObjectType; if (document == null) { return reply.status(400).send('Bad request'); } - const result = await db.collection('MetaObjects').replaceOne({ _id: new ObjectId(id) }, document, { upsert: true }); + const parsed = MetaObjectSchema.safeParse(document); - if (result.modifiedCount === 0) { - return reply.status(404).send('Invalid meta object'); + if (parsed.success === false) { + logger.error(`Error parsing document: ${parsed.error.errors.map(e => `${e.path}, ${e.code}: ${e.message}`).join('| ')}`); + return reply.status(400).send('Bad request'); } + const result = await MetaObject.MetaObject.replaceOne({ _id: id }, document as WithoutId, { upsert: true }); + if (result.upsertedCount === 1) { return reply.status(201).send('Created'); } - if (result.matchedCount === 1) { + if (result.modifiedCount === 1) { return reply.send('Updated'); } + + if (result.modifiedCount === 0) { + return reply.send('Not modified'); + } + } catch (error) { + if (/^\[get-user\]/.test((error as Error).message)) { + return reply.status(401).send('Unauthorized'); + } + + logger.error(error, `Error updating document with id ${id}`); + } + return reply.status(500).send('Internal server error'); + }); + + fastify.delete<{ Params: { id: string } }>('/api/document/:id', async (req, reply) => { + if (req.originalUrl == null || req.params == null) { + return reply.status(404).send('Not found'); + } + + const id = req.params.id; + + if (id == null) { + return reply.status(400).send('Bad request'); + } + + try { + const user = await getUserFromRequest(req); + + if (user == null || user.admin !== true) { + return reply.status(401).send('Unauthorized'); + } + + const result = await MetaObject.MetaObject.deleteOne({ _id: id }); + + if (result.deletedCount === 0) { + return reply.status(404).send('Not found'); + } + return reply.status(200).send('Deleted'); + } catch (error) { + if (/^\[get-user\]/.test((error as Error).message)) { + return reply.status(401).send('Unauthorized'); + } + + logger.error(error, `Error deleting document with id ${id}`); + } + return reply.status(500).send('Internal server error'); + }); + + fastify.get<{ Params: { id: string } }>('/api/document/rebuild-references', async (req, reply) => { + if (req.originalUrl == null || req.params == null) { + return reply.status(404).send('Not found'); + } + + try { + const user = await getUserFromRequest(req); + + if (user == null || user.admin !== true) { + return reply.status(401).send('Unauthorized'); + } + + await loadMetaObjects(); + + return reply.status(200).send('Rebuilt'); } catch (error) { if (/^\[get-user\]/.test((error as Error).message)) { return reply.status(401).send('Unauthorized'); } - logger.error(error, `Error getting document for ${id}`); + logger.error(error, `Error rebuilding references`); } return reply.status(500).send('Internal server error'); }); From 8b52ace699c35c8ea95daed5a62f579ac635be9d Mon Sep 17 00:00:00 2001 From: Leonardo Viva Date: Mon, 18 Nov 2024 15:20:15 -0300 Subject: [PATCH 14/20] fix: login error message --- src/imports/auth/login/index.ts | 4 ++-- src/private/templates/login/login.js | 18 +++--------------- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/src/imports/auth/login/index.ts b/src/imports/auth/login/index.ts index 01bec29c..be2bdb13 100644 --- a/src/imports/auth/login/index.ts +++ b/src/imports/auth/login/index.ts @@ -97,7 +97,7 @@ export async function login({ ip, user, password, password_SHA256, geolocation, return { success: false, logged: false, - errors: [{ message: 'Wrong user or password' }], + errors: [{ message: 'Usuário ou senha inválidos' }], }; } @@ -121,7 +121,7 @@ export async function login({ ip, user, password, password_SHA256, geolocation, return { success: false, logged: false, - errors: [{ message: 'Wrong user or password' }], + errors: [{ message: 'Usuário ou senha inválidos' }], }; } diff --git a/src/private/templates/login/login.js b/src/private/templates/login/login.js index b7e34da2..3f5354f0 100644 --- a/src/private/templates/login/login.js +++ b/src/private/templates/login/login.js @@ -468,7 +468,7 @@ function clearCookie(cookie, clearMyKonecty) { $('.login-panel').show(); $('.login-panel').find('input:first').focus(); var json = JSON.parse(r.responseText); - $('.login-panel .alert-danger').html(json.errors[0].errors[0].msg); + $('.login-panel .alert-danger').html(json.errors[0].message); $('.login-panel .alert-danger').removeClass('hidden'); } }, @@ -480,20 +480,8 @@ function clearCookie(cookie, clearMyKonecty) { }, 8000); window.navigator.geolocation.getCurrentPosition(continueLogin, function () { - // $( '.loading-panel' ).hide(); - clearTimeout(timeout); continueLogin(); - - // if ( $.browser.chrome === true ) { - // $( '.geolocation-denied-chrome-panel' ).show(); - // } - // else if ( $.browser.safari === true ) { - // $( '.geolocation-denied-safari-panel' ).show(); - // } - // else if ( $.browser.mozilla === true ) { - // $( '.geolocation-denied-mozilla-panel' ).show(); - // } }); return false; @@ -521,11 +509,11 @@ function clearCookie(cookie, clearMyKonecty) { complete: function (r) { var json = JSON.parse(r.responseText); - if (json && json.errors && json.errors.length > 0 && json.errors[0].errors && json.errors[0].errors.length > 0 && json.errors[0].errors[0].msg) { + if (json && json.errors && json.errors.length > 0 && json.errors[0].message) { $('.loading-panel').hide(); $('.reset-panel').show(); $('.reset-panel').find('input:first').focus(); - $('.reset-panel .alert-danger').html(json.errors[0].errors[0].msg); + $('.reset-panel .alert-danger').html(json.errors[0].message); $('.reset-panel .alert-danger').removeClass('hidden'); return; } From 4b0e6f22bbf6a9563985a44ba3e18673b2c696a1 Mon Sep 17 00:00:00 2001 From: Leonardo Viva Date: Mon, 18 Nov 2024 16:25:04 -0300 Subject: [PATCH 15/20] feat: collect device fingerprint --- src/imports/auth/login/index.ts | 9 ++++++--- src/imports/model/Namespace.ts | 1 + src/private/templates/fingerprint.js | 26 ++++++++++++++++++++++++++ src/private/templates/index.hbs | 4 ++++ src/private/templates/login/login.hbs | 4 ++++ src/private/templates/login/login.js | 2 ++ src/server/routes/rest/auth/authApi.ts | 4 +++- src/server/routes/rest/view/view.ts | 21 +++++++++++++++------ 8 files changed, 61 insertions(+), 10 deletions(-) create mode 100644 src/private/templates/fingerprint.js diff --git a/src/imports/auth/login/index.ts b/src/imports/auth/login/index.ts index be2bdb13..55c66cba 100644 --- a/src/imports/auth/login/index.ts +++ b/src/imports/auth/login/index.ts @@ -16,9 +16,10 @@ interface LoginParams { resolution?: { width: number; height: number } | string; userAgent?: string; source?: string; + fingerprint?: string; } -interface accessLog { +interface AccessLog { _createdAt: Date; _updatedAt: Date; ip?: string | string[]; @@ -31,6 +32,7 @@ interface accessLog { resolution?: { width: number; height: number }; reason?: string; source?: string; + fingerprint?: string; __from?: string; _user?: [ { @@ -41,10 +43,10 @@ interface accessLog { ]; } -export async function login({ ip, user, password, password_SHA256, geolocation, resolution, userAgent, source }: LoginParams) { +export async function login({ ip, user, password, password_SHA256, geolocation, resolution, userAgent, source, fingerprint }: LoginParams) { const ua = new UAParser(userAgent ?? 'API Call').getResult(); - const accessLog: accessLog = { + const accessLog: AccessLog = { _createdAt: new Date(), _updatedAt: new Date(), ip, @@ -54,6 +56,7 @@ export async function login({ ip, user, password, password_SHA256, geolocation, os: ua.os.name, platform: ua.device.type, source, + fingerprint, __from: 'login', }; diff --git a/src/imports/model/Namespace.ts b/src/imports/model/Namespace.ts index 76c5ec4f..131490ca 100644 --- a/src/imports/model/Namespace.ts +++ b/src/imports/model/Namespace.ts @@ -40,6 +40,7 @@ export const NamespaceSchema = z .object({ type: z.literal('Namespace'), trackUserGeolocation: z.boolean().optional(), + trackUserFingerprint: z.boolean().optional(), loginExpiration: z.number().optional(), dateFormat: z.string().optional(), logoURL: z.string().optional(), diff --git a/src/private/templates/fingerprint.js b/src/private/templates/fingerprint.js new file mode 100644 index 00000000..a5bae741 --- /dev/null +++ b/src/private/templates/fingerprint.js @@ -0,0 +1,26 @@ +(function () { + if (window.localStorage) { + const MAX_FP_AGE = 10 * 24 * 60 * 60 * 1000; // 10 days + window.fingerprint = localStorage.getItem('_k.fp'); + + if (window.fingerprint) { + var fpDate = localStorage.getItem('_k.fp.date'); + + if (fpDate && Date.now() - Number(fpDate) > MAX_FP_AGE) { + localStorage.removeItem('_k.fp'); + localStorage.removeItem('_k.fp.date'); + window.fingerprint = null; + } + } + + if (!window.fingerprint) { + const fpPromise = window.FingerprintJS.load(); + + fpPromise.then((fp) => fp.get()).then((result) => { + window.fingerprint = result.visitorId; + localStorage.setItem('_k.fp', result.visitorId); + localStorage.setItem('_k.fp.date', new Date().getTime().toString()); + }); + } + } +})(); diff --git a/src/private/templates/index.hbs b/src/private/templates/index.hbs index f2bbf116..0ea5dbc3 100644 --- a/src/private/templates/index.hbs +++ b/src/private/templates/index.hbs @@ -64,6 +64,10 @@ integrity="sha512-Kq3/MTxphzXJIRDWtrpLhhNnLDPiBXPMKkx/KogMYZO92Geor9j8sJguZ1OozBS+YVmVKo2HEx2gZfGOQBFM8A==" crossorigin="anonymous" > + {{#if collectFingerprint}} + + + {{/if}} + + {{/if}}