From 7c361084c9b2a5c9c80f2743903f755056bf21e3 Mon Sep 17 00:00:00 2001 From: Yazeed Loonat Date: Fri, 22 Mar 2024 14:18:41 -0700 Subject: [PATCH] feat: better memory management on app export (#3965) --- .../application-csv-export.service.ts | 161 ++-- api/src/services/application.service.ts | 211 ++++- .../unit/services/application.service.spec.ts | 722 ++++++++++++++---- 3 files changed, 853 insertions(+), 241 deletions(-) diff --git a/api/src/services/application-csv-export.service.ts b/api/src/services/application-csv-export.service.ts index 5ae32d2d4d..495dfaad0d 100644 --- a/api/src/services/application-csv-export.service.ts +++ b/api/src/services/application-csv-export.service.ts @@ -23,7 +23,11 @@ import { mapTo } from '../utilities/mapTo'; view.csv = { ...view.details, - applicationFlaggedSet: true, + applicationFlaggedSet: { + select: { + id: true, + }, + }, listings: false, }; @@ -81,12 +85,10 @@ export class ApplicationCsvExporterService filename: string, queryParams: QueryParams, ): Promise { - if (queryParams.includeDemographics) { - view.csv.demographics = true; - } - const applications = await this.prisma.applications.findMany({ - include: view.csv, + select: { + id: true, + }, where: { listingId: queryParams.listingId, deletedAt: null, @@ -110,7 +112,7 @@ export class ApplicationCsvExporterService queryParams.includeDemographics, ); - return new Promise((resolve, reject) => { + return new Promise(async (resolve, reject) => { // create stream const writableStream = fs.createWriteStream(`${filename}`); writableStream @@ -122,80 +124,109 @@ export class ApplicationCsvExporterService .on('close', () => { resolve(); }) - .on('open', () => { + .on('open', async () => { writableStream.write( csvHeaders .map((header) => `"${header.label.replace(/"/g, `""`)}"`) .join(',') + '\n', ); - // now loop over applications and write them to file - applications.forEach((app) => { - let row = ''; - let preferences: ApplicationMultiselectQuestion[]; - csvHeaders.forEach((header, index) => { - let multiselectQuestionValue = false; - let parsePreference = false; - let value = header.path.split('.').reduce((acc, curr) => { - // return preference/program as value for the format function to accept - if (multiselectQuestionValue) { - return acc; - } + for (let i = 0; i < applications.length / 1000 + 1; i++) { + // grab applications 1k at a time + const paginatedApplications = + await this.prisma.applications.findMany({ + include: { + ...view.csv, + demographics: queryParams.includeDemographics + ? { + select: { + id: true, + createdAt: true, + updatedAt: true, + ethnicity: true, + gender: true, + sexualOrientation: true, + howDidYouHear: true, + race: true, + }, + } + : false, + }, + where: { + listingId: queryParams.listingId, + deletedAt: null, + }, + skip: i * 1000, + take: 1000, + }); - if (parsePreference) { - // curr should equal the preference id we're pulling from - if (!preferences) { - preferences = - app.preferences as unknown as ApplicationMultiselectQuestion[]; + // now loop over applications and write them to file + paginatedApplications.forEach((app) => { + let row = ''; + let preferences: ApplicationMultiselectQuestion[]; + csvHeaders.forEach((header, index) => { + let multiselectQuestionValue = false; + let parsePreference = false; + let value = header.path.split('.').reduce((acc, curr) => { + // return preference/program as value for the format function to accept + if (multiselectQuestionValue) { + return acc; } - parsePreference = false; - // there aren't typically many preferences, but if there, then a object map should be created and used - const preference = preferences.find( - (preference) => preference.multiselectQuestionId === curr, - ); - multiselectQuestionValue = true; - return preference; - } - // sets parsePreference to true, for the next iteration - if (curr === 'preferences') { - parsePreference = true; - } + if (parsePreference) { + // curr should equal the preference id we're pulling from + if (!preferences) { + preferences = + app.preferences as unknown as ApplicationMultiselectQuestion[]; + } + parsePreference = false; + // there aren't typically many preferences, but if there, then a object map should be created and used + const preference = preferences.find( + (preference) => preference.multiselectQuestionId === curr, + ); + multiselectQuestionValue = true; + return preference; + } - if (acc === null || acc === undefined) { - return ''; - } + // sets parsePreference to true, for the next iteration + if (curr === 'preferences') { + parsePreference = true; + } - // handles working with arrays, e.g. householdMember.0.firstName - if (!isNaN(Number(curr))) { - const index = Number(curr); - return acc[index]; - } + if (acc === null || acc === undefined) { + return ''; + } - return acc[curr]; - }, app); - value = value === undefined ? '' : value === null ? '' : value; - if (header.format) { - value = header.format(value); - } + // handles working with arrays, e.g. householdMember.0.firstName + if (!isNaN(Number(curr))) { + const index = Number(curr); + return acc[index]; + } - row += value ? `"${value.toString().replace(/"/g, `""`)}"` : ''; - if (index < csvHeaders.length - 1) { - row += ','; - } - }); + return acc[curr]; + }, app); + value = value === undefined ? '' : value === null ? '' : value; + if (header.format) { + value = header.format(value); + } - try { - writableStream.write(row + '\n'); - } catch (e) { - console.log('writeStream write error = ', e); - writableStream.once('drain', () => { - console.log('drain buffer'); - writableStream.write(row + '\n'); + row += value ? `"${value.toString().replace(/"/g, `""`)}"` : ''; + if (index < csvHeaders.length - 1) { + row += ','; + } }); - } - }); + try { + writableStream.write(row + '\n'); + } catch (e) { + console.log('writeStream write error = ', e); + writableStream.once('drain', () => { + console.log('drain buffer'); + writableStream.write(row + '\n'); + }); + } + }); + } writableStream.end(); }); }); diff --git a/api/src/services/application.service.ts b/api/src/services/application.service.ts index 489160d929..952c10c485 100644 --- a/api/src/services/application.service.ts +++ b/api/src/services/application.service.ts @@ -33,18 +33,125 @@ export const view: Partial< > = { partnerList: { applicant: { - include: { - applicantAddress: true, - applicantWorkAddress: true, + select: { + id: true, + firstName: true, + middleName: true, + lastName: true, + birthMonth: true, + birthDay: true, + birthYear: true, + emailAddress: true, + noEmail: true, + phoneNumber: true, + phoneNumberType: true, + noPhone: true, + workInRegion: true, + applicantAddress: { + select: { + id: true, + placeName: true, + city: true, + county: true, + state: true, + street: true, + street2: true, + zipCode: true, + latitude: true, + longitude: true, + }, + }, + applicantWorkAddress: { + select: { + id: true, + placeName: true, + city: true, + county: true, + state: true, + street: true, + street2: true, + zipCode: true, + latitude: true, + longitude: true, + }, + }, + }, + }, + householdMember: { + select: { + id: true, + orderId: true, + firstName: true, + middleName: true, + lastName: true, + birthMonth: true, + birthDay: true, + birthYear: true, + sameAddress: true, + relationship: true, + workInRegion: true, + }, + }, + accessibility: { + select: { + id: true, + mobility: true, + vision: true, + hearing: true, + }, + }, + applicationsMailingAddress: { + select: { + id: true, + placeName: true, + city: true, + county: true, + state: true, + street: true, + street2: true, + zipCode: true, + latitude: true, + longitude: true, + }, + }, + applicationsAlternateAddress: { + select: { + id: true, + placeName: true, + city: true, + county: true, + state: true, + street: true, + street2: true, + zipCode: true, + latitude: true, + longitude: true, }, }, - householdMember: true, - accessibility: true, - applicationsMailingAddress: true, - applicationsAlternateAddress: true, alternateContact: { - include: { - address: true, + select: { + id: true, + type: true, + otherType: true, + firstName: true, + lastName: true, + agency: true, + phoneNumber: true, + emailAddress: true, + address: { + select: { + id: true, + placeName: true, + city: true, + county: true, + state: true, + street: true, + street2: true, + zipCode: true, + latitude: true, + longitude: true, + }, + }, }, }, listings: { @@ -57,20 +164,92 @@ export const view: Partial< view.base = { ...view.partnerList, - demographics: true, - preferredUnitTypes: true, - listings: true, + demographics: { + select: { + id: true, + createdAt: true, + updatedAt: true, + ethnicity: true, + gender: true, + sexualOrientation: true, + howDidYouHear: true, + race: true, + }, + }, + preferredUnitTypes: { + select: { + id: true, + name: true, + numBedrooms: true, + }, + }, + listings: { + select: { + id: true, + name: true, + jurisdictions: { + select: { + id: true, + name: true, + }, + }, + }, + }, householdMember: { - include: { - householdMemberAddress: true, - householdMemberWorkAddress: true, + select: { + id: true, + orderId: true, + firstName: true, + middleName: true, + lastName: true, + birthMonth: true, + birthDay: true, + birthYear: true, + sameAddress: true, + relationship: true, + workInRegion: true, + householdMemberAddress: { + select: { + id: true, + placeName: true, + city: true, + county: true, + state: true, + street: true, + street2: true, + zipCode: true, + latitude: true, + longitude: true, + }, + }, + householdMemberWorkAddress: { + select: { + id: true, + placeName: true, + city: true, + county: true, + state: true, + street: true, + street2: true, + zipCode: true, + latitude: true, + longitude: true, + }, + }, }, }, }; view.details = { ...view.base, - userAccounts: true, + userAccounts: { + select: { + id: true, + firstName: true, + lastName: true, + email: true, + }, + }, }; /* diff --git a/api/test/unit/services/application.service.spec.ts b/api/test/unit/services/application.service.spec.ts index e0d066e4da..5a0c78d1d1 100644 --- a/api/test/unit/services/application.service.spec.ts +++ b/api/test/unit/services/application.service.spec.ts @@ -233,6 +233,382 @@ export const mockCreateApplicationData = ( } as ApplicationCreate; }; +const detailView = { + applicant: { + select: { + id: true, + firstName: true, + middleName: true, + lastName: true, + birthMonth: true, + birthDay: true, + birthYear: true, + emailAddress: true, + noEmail: true, + phoneNumber: true, + phoneNumberType: true, + noPhone: true, + workInRegion: true, + applicantAddress: { + select: { + id: true, + placeName: true, + city: true, + county: true, + state: true, + street: true, + street2: true, + zipCode: true, + latitude: true, + longitude: true, + }, + }, + applicantWorkAddress: { + select: { + id: true, + placeName: true, + city: true, + county: true, + state: true, + street: true, + street2: true, + zipCode: true, + latitude: true, + longitude: true, + }, + }, + }, + }, + accessibility: { + select: { + id: true, + mobility: true, + vision: true, + hearing: true, + }, + }, + applicationsMailingAddress: { + select: { + id: true, + placeName: true, + city: true, + county: true, + state: true, + street: true, + street2: true, + zipCode: true, + latitude: true, + longitude: true, + }, + }, + applicationsAlternateAddress: { + select: { + id: true, + placeName: true, + city: true, + county: true, + state: true, + street: true, + street2: true, + zipCode: true, + latitude: true, + longitude: true, + }, + }, + alternateContact: { + select: { + id: true, + type: true, + otherType: true, + firstName: true, + lastName: true, + agency: true, + phoneNumber: true, + emailAddress: true, + address: { + select: { + id: true, + placeName: true, + city: true, + county: true, + state: true, + street: true, + street2: true, + zipCode: true, + latitude: true, + longitude: true, + }, + }, + }, + }, + demographics: { + select: { + id: true, + createdAt: true, + updatedAt: true, + ethnicity: true, + gender: true, + sexualOrientation: true, + howDidYouHear: true, + race: true, + }, + }, + preferredUnitTypes: { + select: { + id: true, + name: true, + numBedrooms: true, + }, + }, + listings: { + select: { + id: true, + name: true, + jurisdictions: { + select: { + id: true, + name: true, + }, + }, + }, + }, + householdMember: { + select: { + id: true, + orderId: true, + firstName: true, + middleName: true, + lastName: true, + birthMonth: true, + birthDay: true, + birthYear: true, + sameAddress: true, + relationship: true, + workInRegion: true, + householdMemberAddress: { + select: { + id: true, + placeName: true, + city: true, + county: true, + state: true, + street: true, + street2: true, + zipCode: true, + latitude: true, + longitude: true, + }, + }, + householdMemberWorkAddress: { + select: { + id: true, + placeName: true, + city: true, + county: true, + state: true, + street: true, + street2: true, + zipCode: true, + latitude: true, + longitude: true, + }, + }, + }, + }, + userAccounts: { + select: { + id: true, + firstName: true, + lastName: true, + email: true, + }, + }, +}; + +const baseView = { + applicant: { + select: { + id: true, + firstName: true, + middleName: true, + lastName: true, + birthMonth: true, + birthDay: true, + birthYear: true, + emailAddress: true, + noEmail: true, + phoneNumber: true, + phoneNumberType: true, + noPhone: true, + workInRegion: true, + applicantAddress: { + select: { + id: true, + placeName: true, + city: true, + county: true, + state: true, + street: true, + street2: true, + zipCode: true, + latitude: true, + longitude: true, + }, + }, + applicantWorkAddress: { + select: { + id: true, + placeName: true, + city: true, + county: true, + state: true, + street: true, + street2: true, + zipCode: true, + latitude: true, + longitude: true, + }, + }, + }, + }, + accessibility: { + select: { + id: true, + mobility: true, + vision: true, + hearing: true, + }, + }, + applicationsMailingAddress: { + select: { + id: true, + placeName: true, + city: true, + county: true, + state: true, + street: true, + street2: true, + zipCode: true, + latitude: true, + longitude: true, + }, + }, + applicationsAlternateAddress: { + select: { + id: true, + placeName: true, + city: true, + county: true, + state: true, + street: true, + street2: true, + zipCode: true, + latitude: true, + longitude: true, + }, + }, + alternateContact: { + select: { + id: true, + type: true, + otherType: true, + firstName: true, + lastName: true, + agency: true, + phoneNumber: true, + emailAddress: true, + address: { + select: { + id: true, + placeName: true, + city: true, + county: true, + state: true, + street: true, + street2: true, + zipCode: true, + latitude: true, + longitude: true, + }, + }, + }, + }, + demographics: { + select: { + id: true, + createdAt: true, + updatedAt: true, + ethnicity: true, + gender: true, + sexualOrientation: true, + howDidYouHear: true, + race: true, + }, + }, + preferredUnitTypes: { + select: { + id: true, + name: true, + numBedrooms: true, + }, + }, + listings: { + select: { + id: true, + name: true, + jurisdictions: { + select: { + id: true, + name: true, + }, + }, + }, + }, + householdMember: { + select: { + id: true, + orderId: true, + firstName: true, + middleName: true, + lastName: true, + birthMonth: true, + birthDay: true, + birthYear: true, + sameAddress: true, + relationship: true, + workInRegion: true, + householdMemberAddress: { + select: { + id: true, + placeName: true, + city: true, + county: true, + state: true, + street: true, + street2: true, + zipCode: true, + latitude: true, + longitude: true, + }, + }, + householdMemberWorkAddress: { + select: { + id: true, + placeName: true, + city: true, + county: true, + state: true, + street: true, + street2: true, + zipCode: true, + latitude: true, + longitude: true, + }, + }, + }, + }, +}; + describe('Testing application service', () => { let service: ApplicationService; let prisma: PrismaService; @@ -376,30 +752,7 @@ describe('Testing application service', () => { id: 'example Id', }, include: { - userAccounts: true, - applicant: { - include: { - applicantAddress: true, - applicantWorkAddress: true, - }, - }, - applicationsMailingAddress: true, - applicationsAlternateAddress: true, - alternateContact: { - include: { - address: true, - }, - }, - accessibility: true, - demographics: true, - householdMember: { - include: { - householdMemberAddress: true, - householdMemberWorkAddress: true, - }, - }, - listings: true, - preferredUnitTypes: true, + ...detailView, }, }); }); @@ -427,30 +780,7 @@ describe('Testing application service', () => { id: 'example Id', }, include: { - userAccounts: true, - applicant: { - include: { - applicantAddress: true, - applicantWorkAddress: true, - }, - }, - applicationsMailingAddress: true, - applicationsAlternateAddress: true, - alternateContact: { - include: { - address: true, - }, - }, - accessibility: true, - demographics: true, - householdMember: { - include: { - householdMemberAddress: true, - householdMemberWorkAddress: true, - }, - }, - listings: true, - preferredUnitTypes: true, + ...detailView, }, }); }); @@ -649,29 +979,7 @@ describe('Testing application service', () => { id: 'example Id', }, include: { - applicant: { - include: { - applicantAddress: true, - applicantWorkAddress: true, - }, - }, - applicationsMailingAddress: true, - applicationsAlternateAddress: true, - alternateContact: { - include: { - address: true, - }, - }, - accessibility: true, - demographics: true, - householdMember: { - include: { - householdMemberAddress: true, - householdMemberWorkAddress: true, - }, - }, - listings: true, - preferredUnitTypes: true, + ...baseView, }, }); }); @@ -822,32 +1130,7 @@ describe('Testing application service', () => { }); expect(prisma.applications.create).toHaveBeenCalledWith({ - include: { - accessibility: true, - applicationsAlternateAddress: true, - applicationsMailingAddress: true, - demographics: true, - listings: true, - preferredUnitTypes: true, - userAccounts: true, - alternateContact: { - include: { - address: true, - }, - }, - applicant: { - include: { - applicantAddress: true, - applicantWorkAddress: true, - }, - }, - householdMember: { - include: { - householdMemberAddress: true, - householdMemberWorkAddress: true, - }, - }, - }, + include: { ...detailView }, data: { contactPreferences: ['example contact preference'], status: ApplicationStatusEnum.submitted, @@ -1108,30 +1391,7 @@ describe('Testing application service', () => { expect(prisma.applications.create).toHaveBeenCalledWith({ include: { - accessibility: true, - applicationsAlternateAddress: true, - applicationsMailingAddress: true, - demographics: true, - listings: true, - preferredUnitTypes: true, - userAccounts: true, - alternateContact: { - include: { - address: true, - }, - }, - applicant: { - include: { - applicantAddress: true, - applicantWorkAddress: true, - }, - }, - householdMember: { - include: { - householdMemberAddress: true, - householdMemberWorkAddress: true, - }, - }, + ...detailView, }, data: { contactPreferences: ['example contact preference'], @@ -1359,30 +1619,7 @@ describe('Testing application service', () => { expect(prisma.applications.update).toHaveBeenCalledWith({ include: { - accessibility: true, - applicationsAlternateAddress: true, - applicationsMailingAddress: true, - demographics: true, - listings: true, - preferredUnitTypes: true, - userAccounts: true, - alternateContact: { - include: { - address: true, - }, - }, - applicant: { - include: { - applicantAddress: true, - applicantWorkAddress: true, - }, - }, - householdMember: { - include: { - householdMemberAddress: true, - householdMemberWorkAddress: true, - }, - }, + ...detailView, }, data: { contactPreferences: ['example contact preference'], @@ -1652,30 +1889,195 @@ describe('Testing application service', () => { id: mockedValue.id, }, include: { - userAccounts: true, applicant: { - include: { - applicantAddress: true, - applicantWorkAddress: true, + select: { + id: true, + firstName: true, + middleName: true, + lastName: true, + birthMonth: true, + birthDay: true, + birthYear: true, + emailAddress: true, + noEmail: true, + phoneNumber: true, + phoneNumberType: true, + noPhone: true, + workInRegion: true, + applicantAddress: { + select: { + id: true, + placeName: true, + city: true, + county: true, + state: true, + street: true, + street2: true, + zipCode: true, + latitude: true, + longitude: true, + }, + }, + applicantWorkAddress: { + select: { + id: true, + placeName: true, + city: true, + county: true, + state: true, + street: true, + street2: true, + zipCode: true, + latitude: true, + longitude: true, + }, + }, + }, + }, + accessibility: { + select: { + id: true, + mobility: true, + vision: true, + hearing: true, + }, + }, + applicationsMailingAddress: { + select: { + id: true, + placeName: true, + city: true, + county: true, + state: true, + street: true, + street2: true, + zipCode: true, + latitude: true, + longitude: true, + }, + }, + applicationsAlternateAddress: { + select: { + id: true, + placeName: true, + city: true, + county: true, + state: true, + street: true, + street2: true, + zipCode: true, + latitude: true, + longitude: true, }, }, - applicationsMailingAddress: true, - applicationsAlternateAddress: true, alternateContact: { - include: { - address: true, + select: { + id: true, + type: true, + otherType: true, + firstName: true, + lastName: true, + agency: true, + phoneNumber: true, + emailAddress: true, + address: { + select: { + id: true, + placeName: true, + city: true, + county: true, + state: true, + street: true, + street2: true, + zipCode: true, + latitude: true, + longitude: true, + }, + }, + }, + }, + demographics: { + select: { + id: true, + createdAt: true, + updatedAt: true, + ethnicity: true, + gender: true, + sexualOrientation: true, + howDidYouHear: true, + race: true, + }, + }, + preferredUnitTypes: { + select: { + id: true, + name: true, + numBedrooms: true, + }, + }, + listings: { + select: { + id: true, + name: true, + jurisdictions: { + select: { + id: true, + name: true, + }, + }, }, }, - accessibility: true, - demographics: true, householdMember: { - include: { - householdMemberAddress: true, - householdMemberWorkAddress: true, + select: { + id: true, + orderId: true, + firstName: true, + middleName: true, + lastName: true, + birthMonth: true, + birthDay: true, + birthYear: true, + sameAddress: true, + relationship: true, + workInRegion: true, + householdMemberAddress: { + select: { + id: true, + placeName: true, + city: true, + county: true, + state: true, + street: true, + street2: true, + zipCode: true, + latitude: true, + longitude: true, + }, + }, + householdMemberWorkAddress: { + select: { + id: true, + placeName: true, + city: true, + county: true, + state: true, + street: true, + street2: true, + zipCode: true, + latitude: true, + longitude: true, + }, + }, + }, + }, + userAccounts: { + select: { + id: true, + firstName: true, + lastName: true, + email: true, }, }, - listings: true, - preferredUnitTypes: true, }, }); });