From 79babe4977ff45b666efea80bb73089e29418d70 Mon Sep 17 00:00:00 2001 From: Raul Coelho Date: Wed, 10 Apr 2024 15:57:09 -0300 Subject: [PATCH 01/12] installing and importing schedule package nestjs schedule package for Cron jobs --- package-lock.json | 83 +++++++++++++++++++++++++++++++++++++++++++++++ package.json | 1 + src/app.module.ts | 2 ++ 3 files changed, 86 insertions(+) diff --git a/package-lock.json b/package-lock.json index d9e28fd..cb7c2fc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,7 @@ "@nestjs/core": "^9.0.0", "@nestjs/mapped-types": "*", "@nestjs/platform-express": "^9.0.0", + "@nestjs/schedule": "^4.0.1", "@nestjs/swagger": "^6.1.2", "@nestjs/typeorm": "^9.0.1", "class-transformer": "^0.5.1", @@ -1893,6 +1894,31 @@ "node": ">= 0.8" } }, + "node_modules/@nestjs/schedule": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@nestjs/schedule/-/schedule-4.0.1.tgz", + "integrity": "sha512-cz2FNjsuoma+aGsG0cMmG6Dqg/BezbBWet1UTHtAuu6d2mXNTVcmoEQM2DIVG5Lfwb2hfSE2yZt8Moww+7y+mA==", + "dependencies": { + "cron": "3.1.6", + "uuid": "9.0.1" + }, + "peerDependencies": { + "@nestjs/common": "^8.0.0 || ^9.0.0 || ^10.0.0", + "@nestjs/core": "^8.0.0 || ^9.0.0 || ^10.0.0" + } + }, + "node_modules/@nestjs/schedule/node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/@nestjs/schematics": { "version": "9.1.0", "resolved": "https://registry.npmjs.org/@nestjs/schematics/-/schematics-9.1.0.tgz", @@ -2360,6 +2386,11 @@ "integrity": "sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==", "dev": true }, + "node_modules/@types/luxon": { + "version": "3.3.8", + "resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.3.8.tgz", + "integrity": "sha512-jYvz8UMLDgy3a5SkGJne8H7VA7zPV2Lwohjx0V8V31+SqAjNmurWMkk9cQhfvlcnXWudBpK9xPM1n4rljOcHYQ==" + }, "node_modules/@types/mime": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz", @@ -4088,6 +4119,15 @@ "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", "devOptional": true }, + "node_modules/cron": { + "version": "3.1.6", + "resolved": "https://registry.npmjs.org/cron/-/cron-3.1.6.tgz", + "integrity": "sha512-cvFiQCeVzsA+QPM6fhjBtlKGij7tLLISnTSvFxVdnFGLdz+ZdXN37kNe0i2gefmdD17XuZA6n2uPVwzl4FxW/w==", + "dependencies": { + "@types/luxon": "~3.3.0", + "luxon": "~3.4.0" + } + }, "node_modules/cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -7699,6 +7739,14 @@ "node": ">=10" } }, + "node_modules/luxon": { + "version": "3.4.4", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.4.4.tgz", + "integrity": "sha512-zobTr7akeGHnv7eBOXcRgMeCP6+uyYsczwmeRCauvpvaAltgNyTbLH/+VaEAPUeWBT+1GuNmz4wC/6jtQzbbVA==", + "engines": { + "node": ">=12" + } + }, "node_modules/macos-release": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/macos-release/-/macos-release-2.5.0.tgz", @@ -12149,6 +12197,22 @@ } } }, + "@nestjs/schedule": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@nestjs/schedule/-/schedule-4.0.1.tgz", + "integrity": "sha512-cz2FNjsuoma+aGsG0cMmG6Dqg/BezbBWet1UTHtAuu6d2mXNTVcmoEQM2DIVG5Lfwb2hfSE2yZt8Moww+7y+mA==", + "requires": { + "cron": "3.1.6", + "uuid": "9.0.1" + }, + "dependencies": { + "uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==" + } + } + }, "@nestjs/schematics": { "version": "9.1.0", "resolved": "https://registry.npmjs.org/@nestjs/schematics/-/schematics-9.1.0.tgz", @@ -12526,6 +12590,11 @@ "integrity": "sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==", "dev": true }, + "@types/luxon": { + "version": "3.3.8", + "resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.3.8.tgz", + "integrity": "sha512-jYvz8UMLDgy3a5SkGJne8H7VA7zPV2Lwohjx0V8V31+SqAjNmurWMkk9cQhfvlcnXWudBpK9xPM1n4rljOcHYQ==" + }, "@types/mime": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz", @@ -13862,6 +13931,15 @@ "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", "devOptional": true }, + "cron": { + "version": "3.1.6", + "resolved": "https://registry.npmjs.org/cron/-/cron-3.1.6.tgz", + "integrity": "sha512-cvFiQCeVzsA+QPM6fhjBtlKGij7tLLISnTSvFxVdnFGLdz+ZdXN37kNe0i2gefmdD17XuZA6n2uPVwzl4FxW/w==", + "requires": { + "@types/luxon": "~3.3.0", + "luxon": "~3.4.0" + } + }, "cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -16684,6 +16762,11 @@ "yallist": "^4.0.0" } }, + "luxon": { + "version": "3.4.4", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.4.4.tgz", + "integrity": "sha512-zobTr7akeGHnv7eBOXcRgMeCP6+uyYsczwmeRCauvpvaAltgNyTbLH/+VaEAPUeWBT+1GuNmz4wC/6jtQzbbVA==" + }, "macos-release": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/macos-release/-/macos-release-2.5.0.tgz", diff --git a/package.json b/package.json index de8dfcb..92ef0e1 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "@nestjs/core": "^9.0.0", "@nestjs/mapped-types": "*", "@nestjs/platform-express": "^9.0.0", + "@nestjs/schedule": "^4.0.1", "@nestjs/swagger": "^6.1.2", "@nestjs/typeorm": "^9.0.1", "class-transformer": "^0.5.1", diff --git a/src/app.module.ts b/src/app.module.ts index d6710be..b8d5464 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -18,11 +18,13 @@ import { import { APP_GUARD } from '@nestjs/core'; import { AppDataSource } from './app.datasource'; import { ScholarshipModule } from './scholarship/scholarship.module'; +import { ScheduleModule } from '@nestjs/schedule'; @Module({ imports: [ ConfigModule.forRoot({ isGlobal: true }), TypeOrmModule.forRoot(AppDataSource.options), + ScheduleModule.forRoot(), ImportXmlModule, ProfessorModule, QualisModule, From 3a21c9c0208ca6a7731d6e161d4a5c7fc20d101f Mon Sep 17 00:00:00 2001 From: Raul Coelho Date: Thu, 11 Apr 2024 10:46:06 -0300 Subject: [PATCH 02/12] creating crons for journals and conferences refresh --- src/qualis/conference/conference.service.ts | 49 +++++++++++++++++++++ src/qualis/dto/refresh-conference.dto.ts | 17 +++++++ src/qualis/qualis.service.ts | 9 ++++ 3 files changed, 75 insertions(+) create mode 100644 src/qualis/dto/refresh-conference.dto.ts diff --git a/src/qualis/conference/conference.service.ts b/src/qualis/conference/conference.service.ts index 5abf3c0..fadd958 100644 --- a/src/qualis/conference/conference.service.ts +++ b/src/qualis/conference/conference.service.ts @@ -7,6 +7,8 @@ import { Log } from 'src/utils/exception-filters/log.entity'; import { EntityType } from 'src/utils/exception-filters/entity-type-enum'; import { AppDataSource } from 'src/app.datasource'; import createLog from 'src/utils/exception-filters/log-utils'; +import { RefreshConferenceDto } from '../dto/refresh-conference.dto'; +import { Cron, CronExpression } from '@nestjs/schedule'; @Injectable() export class ConferenceService { @@ -88,4 +90,51 @@ export class ConferenceService { ); } } + + @Cron(CronExpression.EVERY_1ST_DAY_OF_MONTH_AT_MIDNIGHT, { + name: 'refresh_conferences', + timeZone: 'America/Recife', + }) + async refresh() { + const refreshConferenceDtos: RefreshConferenceDto[] = [ + { + acronym: 'SBES', + name: 'Brazilian Symposium on Software Engineering', + qualis: 'A1', + }, + { + acronym: 'SBC', + name: 'Brazilian Computer Society', + qualis: 'A2', + }, + { + acronym: 'CBSoft', + name: 'Brazilian Conference on Software', + qualis: 'B1', + }, + ]; + const email = 'cron_job@cin.ufpe.br'; + for (const refreshConferenceDto of refreshConferenceDtos) { + const conference = await AppDataSource.manager.findOne(Conference, { + where: { name: refreshConferenceDto.name }, + }); + if (!conference) { + console.log('Creating conference'); + await this.create( + { ...refreshConferenceDto, isTop: false, official: true }, + email, + ); + } else if ( + conference.acronym !== refreshConferenceDto.acronym || + conference.qualis !== refreshConferenceDto.qualis + ) { + console.log('Updating conference'); + await this.update( + conference.id, + { ...refreshConferenceDto, id: conference.id }, + email, + ); + } + } + } } diff --git a/src/qualis/dto/refresh-conference.dto.ts b/src/qualis/dto/refresh-conference.dto.ts new file mode 100644 index 0000000..45982fe --- /dev/null +++ b/src/qualis/dto/refresh-conference.dto.ts @@ -0,0 +1,17 @@ +import { ApiProperty, ApiTags } from '@nestjs/swagger'; +import { IsString } from 'class-validator'; + +@ApiTags('RefreshConferenceDto') +export class RefreshConferenceDto { + @ApiProperty({ name: 'acronym', type: String }) + @IsString({ message: 'acronym must be a string' }) + acronym!: string; + + @ApiProperty({ name: 'name', type: String }) + @IsString({ message: 'name must be a string' }) + name!: string; + + @ApiProperty({ name: 'qualis', type: String }) + @IsString({ message: 'qualis must be a string' }) + qualis!: string; +} diff --git a/src/qualis/qualis.service.ts b/src/qualis/qualis.service.ts index 0383a31..bd2404f 100644 --- a/src/qualis/qualis.service.ts +++ b/src/qualis/qualis.service.ts @@ -7,6 +7,7 @@ import { Log } from 'src/utils/exception-filters/log.entity'; import { EntityType } from 'src/utils/exception-filters/entity-type-enum'; import { AppDataSource } from 'src/app.datasource'; import createLog from 'src/utils/exception-filters/log-utils'; +import { Cron, CronExpression } from '@nestjs/schedule'; @Injectable() export class JournalService { @@ -88,4 +89,12 @@ export class JournalService { remove(id: number) { return `This action removes a #${id} journal`; } + + @Cron(CronExpression.EVERY_1ST_DAY_OF_MONTH_AT_MIDNIGHT, { + name: 'refresh_journals', + timeZone: 'America/Recife', + }) + async refresh() { + console.log('Refreshing journals'); + } } From 5b6b9f382102cd6bfac4e24e09724825a3326a4f Mon Sep 17 00:00:00 2001 From: Raul Coelho Date: Thu, 11 Apr 2024 15:40:28 -0300 Subject: [PATCH 03/12] fetching csv and refreshing conferences --- .env.example | 5 +- package-lock.json | 30 +++++++ package.json | 2 + src/qualis/conference/conference.service.ts | 92 ++++++++++++--------- 4 files changed, 90 insertions(+), 39 deletions(-) diff --git a/.env.example b/.env.example index 8757d0a..dced2a4 100644 --- a/.env.example +++ b/.env.example @@ -19,4 +19,7 @@ KEYCLOAK_JSON={"realm":"your_realm","auth-server-url":"https://localhost:8080/", AUTH_URL=https://localhost:8080/realms/your_realm/protocol/openid-connect/auth TOKEN_URL=https://localhost:8080/realms/your_realm/protocol/openid-connect/token REDIRECT_URL=https://localhost:3000/ -CLIENT_ID=research-back \ No newline at end of file +CLIENT_ID=research-back + +CONFERENCES_SHEET_ID=1yvuCa__L7r0EJy6v6Jb17fvu-VdV80PbfAReR9Gy52I +JOURNALS_SHEET_ID=10sObNyyL7veHGFbOyizxM8oVsppQoWV-0ALrDr8FxQ0 \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index cb7c2fc..2fbc4db 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,6 +21,7 @@ "class-validator": "^0.14.0", "extract-zip": "^2.0.1", "nest-keycloak-connect": "^1.9.1", + "papaparse": "^5.4.1", "pg": "^8.7.1", "rimraf": "^3.0.2", "string-similarity": "^4.0.4", @@ -35,6 +36,7 @@ "@types/jest": "^27.4.1", "@types/multer": "^1.4.7", "@types/node": "^16.18.30", + "@types/papaparse": "^5.3.14", "@types/string-similarity": "^4.0.0", "@types/supertest": "^2.0.11", "@types/uuid": "^9.0.6", @@ -2412,6 +2414,15 @@ "integrity": "sha512-Kmp/wBZk19Dn7uRiol8kF8agnf8m0+TU9qIwyfPmXglVxMlmiIz0VQSMw5oFgwhmD2aKTlfBIO5FtsVj3y7hKQ==", "devOptional": true }, + "node_modules/@types/papaparse": { + "version": "5.3.14", + "resolved": "https://registry.npmjs.org/@types/papaparse/-/papaparse-5.3.14.tgz", + "integrity": "sha512-LxJ4iEFcpqc6METwp9f6BV6VVc43m6MfH0VqFosHvrUgfXiFe6ww7R3itkOQ+TCK6Y+Iv/+RnnvtRZnkc5Kc9g==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/parse-json": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.0.tgz", @@ -8267,6 +8278,11 @@ "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz", "integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==" }, + "node_modules/papaparse": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/papaparse/-/papaparse-5.4.1.tgz", + "integrity": "sha512-HipMsgJkZu8br23pW15uvo6sib6wne/4woLZPlFf3rpDyMe9ywEXUsuD7+6K9PRkJlVT51j/sCOYDKGGS3ZJrw==" + }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -12616,6 +12632,15 @@ "integrity": "sha512-Kmp/wBZk19Dn7uRiol8kF8agnf8m0+TU9qIwyfPmXglVxMlmiIz0VQSMw5oFgwhmD2aKTlfBIO5FtsVj3y7hKQ==", "devOptional": true }, + "@types/papaparse": { + "version": "5.3.14", + "resolved": "https://registry.npmjs.org/@types/papaparse/-/papaparse-5.3.14.tgz", + "integrity": "sha512-LxJ4iEFcpqc6METwp9f6BV6VVc43m6MfH0VqFosHvrUgfXiFe6ww7R3itkOQ+TCK6Y+Iv/+RnnvtRZnkc5Kc9g==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/parse-json": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.0.tgz", @@ -17154,6 +17179,11 @@ "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz", "integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==" }, + "papaparse": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/papaparse/-/papaparse-5.4.1.tgz", + "integrity": "sha512-HipMsgJkZu8br23pW15uvo6sib6wne/4woLZPlFf3rpDyMe9ywEXUsuD7+6K9PRkJlVT51j/sCOYDKGGS3ZJrw==" + }, "parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", diff --git a/package.json b/package.json index 92ef0e1..60d1252 100644 --- a/package.json +++ b/package.json @@ -43,6 +43,7 @@ "class-validator": "^0.14.0", "extract-zip": "^2.0.1", "nest-keycloak-connect": "^1.9.1", + "papaparse": "^5.4.1", "pg": "^8.7.1", "rimraf": "^3.0.2", "string-similarity": "^4.0.4", @@ -57,6 +58,7 @@ "@types/jest": "^27.4.1", "@types/multer": "^1.4.7", "@types/node": "^16.18.30", + "@types/papaparse": "^5.3.14", "@types/string-similarity": "^4.0.0", "@types/supertest": "^2.0.11", "@types/uuid": "^9.0.6", diff --git a/src/qualis/conference/conference.service.ts b/src/qualis/conference/conference.service.ts index fadd958..ea5da78 100644 --- a/src/qualis/conference/conference.service.ts +++ b/src/qualis/conference/conference.service.ts @@ -9,6 +9,8 @@ import { AppDataSource } from 'src/app.datasource'; import createLog from 'src/utils/exception-filters/log-utils'; import { RefreshConferenceDto } from '../dto/refresh-conference.dto'; import { Cron, CronExpression } from '@nestjs/schedule'; +import Papa from 'papaparse'; +import axios from 'axios'; @Injectable() export class ConferenceService { @@ -91,50 +93,64 @@ export class ConferenceService { } } - @Cron(CronExpression.EVERY_1ST_DAY_OF_MONTH_AT_MIDNIGHT, { + // CronExpression.EVERY_1ST_DAY_OF_MONTH_AT_MIDNIGHT + @Cron(new Date(Date.now() + 2 * 1000), { name: 'refresh_conferences', timeZone: 'America/Recife', }) async refresh() { - const refreshConferenceDtos: RefreshConferenceDto[] = [ - { - acronym: 'SBES', - name: 'Brazilian Symposium on Software Engineering', - qualis: 'A1', - }, - { - acronym: 'SBC', - name: 'Brazilian Computer Society', - qualis: 'A2', - }, - { - acronym: 'CBSoft', - name: 'Brazilian Conference on Software', - qualis: 'B1', - }, - ]; const email = 'cron_job@cin.ufpe.br'; - for (const refreshConferenceDto of refreshConferenceDtos) { - const conference = await AppDataSource.manager.findOne(Conference, { - where: { name: refreshConferenceDto.name }, - }); - if (!conference) { - console.log('Creating conference'); - await this.create( - { ...refreshConferenceDto, isTop: false, official: true }, - email, - ); - } else if ( - conference.acronym !== refreshConferenceDto.acronym || - conference.qualis !== refreshConferenceDto.qualis - ) { - console.log('Updating conference'); - await this.update( - conference.id, - { ...refreshConferenceDto, id: conference.id }, - email, - ); + const csvUrl = + 'https://docs.google.com/spreadsheets/d/' + + process.env.CONFERENCES_SHEET_ID + + '/gviz/tq?tqx=out:csv&sheet=Qualis'; + + const headers = new Map([ + ['sigla', 'acronym'], + ['Qualis_Final', 'qualis'], + ['evento', 'name'], + ]); + try { + const response = await axios.get(csvUrl); + const refreshConferenceDtos: RefreshConferenceDto[] = Papa.parse( + response.data, + { + header: true, + transformHeader: (header) => headers.get(header) || header, + }, + ).data as RefreshConferenceDto[]; + console.log(refreshConferenceDtos); + + for (const refreshConferenceDto of refreshConferenceDtos) { + const conference = await AppDataSource.manager.findOne(Conference, { + where: { + acronym: refreshConferenceDto.acronym, + }, + }); + if (!conference) { + console.log('Creating conference'); + // await this.create( + // { + // ...refreshConferenceDto, + // isTop: false, + // official: true, + // }, + // email, + // ); + } else if ( + refreshConferenceDto.name !== conference.name || + refreshConferenceDto.qualis !== conference.qualis + ) { + console.log('Updating conference'); + // await this.update( + // conference.id, + // { ...refreshConferenceDto, id: conference.id }, + // email, + // ); + } } + } catch (error) { + throw new Error(`Error refreshing conferences: ${error}`); } } } From 89827994eaee259783a697a3b776c049123ebe8c Mon Sep 17 00:00:00 2001 From: Raul Coelho Date: Wed, 17 Apr 2024 11:17:53 -0300 Subject: [PATCH 04/12] updating and creating conferences --- src/qualis/conference/conference.service.ts | 34 ++++++++++++--------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/src/qualis/conference/conference.service.ts b/src/qualis/conference/conference.service.ts index ea5da78..69e8380 100644 --- a/src/qualis/conference/conference.service.ts +++ b/src/qualis/conference/conference.service.ts @@ -128,25 +128,29 @@ export class ConferenceService { }, }); if (!conference) { - console.log('Creating conference'); - // await this.create( - // { - // ...refreshConferenceDto, - // isTop: false, - // official: true, - // }, - // email, - // ); + console.log( + `Creating conference ${refreshConferenceDto.acronym}-${refreshConferenceDto.name}`, + ); + await this.create( + { + ...refreshConferenceDto, + isTop: false, + official: true, + }, + email, + ); } else if ( refreshConferenceDto.name !== conference.name || refreshConferenceDto.qualis !== conference.qualis ) { - console.log('Updating conference'); - // await this.update( - // conference.id, - // { ...refreshConferenceDto, id: conference.id }, - // email, - // ); + console.log( + `Updating conference ${conference.acronym}-${conference.name}-${conference.qualis} to ${refreshConferenceDto.acronym}-${refreshConferenceDto.name}-${refreshConferenceDto.qualis}`, + ); + await this.update( + conference.id, + { ...refreshConferenceDto, id: conference.id }, + email, + ); } } } catch (error) { From 69de6664d9b32816d936b2e08a3a31372f6cc8f4 Mon Sep 17 00:00:00 2001 From: Raul Coelho Date: Tue, 23 Apr 2024 16:50:00 -0300 Subject: [PATCH 05/12] refreshing confererences with transaction using typeorm transaction to make sure there's never a case that only half of the operations are performed --- src/professor/professor.service.ts | 1 + src/qualis/conference/conference.service.ts | 59 ++++++++++++++++----- src/qualis/qualis.controller.ts | 35 ++++++++++-- src/qualis/qualis.service.ts | 1 + src/utils/exception-filters/log-utils.ts | 4 +- 5 files changed, 83 insertions(+), 17 deletions(-) diff --git a/src/professor/professor.service.ts b/src/professor/professor.service.ts index 4480688..3d8d03c 100644 --- a/src/professor/professor.service.ts +++ b/src/professor/professor.service.ts @@ -332,6 +332,7 @@ export class ProfessorService { await queryRunner.commitTransaction(); createLog( + queryRunner, EntityType.PROFESSOR, `Type: Delete Email: ${email} diff --git a/src/qualis/conference/conference.service.ts b/src/qualis/conference/conference.service.ts index 69e8380..4d9460e 100644 --- a/src/qualis/conference/conference.service.ts +++ b/src/qualis/conference/conference.service.ts @@ -1,5 +1,5 @@ import { Injectable } from '@nestjs/common'; -import { QueryRunner } from 'typeorm'; +import { EntityManager, QueryRunner } from 'typeorm'; import { CreateConferenceDto } from '../dto/create-conference.dto'; import { UpdateConferenceDto } from '../dto/update-conference.dto'; import { Conference } from '../entities/conference.entity'; @@ -24,7 +24,18 @@ export class ConferenceService { return conference; } - async create(createConferenceDto: CreateConferenceDto, email: string) { + async create( + queryRunner: QueryRunner | undefined, + createConferenceDto: CreateConferenceDto, + email: string, + ) { + let manager: EntityManager; + if (!queryRunner) { + manager = AppDataSource.manager; + } else { + manager = queryRunner.manager; + } + const conference = new Conference(); conference.acronym = createConferenceDto.acronym; conference.name = createConferenceDto.name; @@ -35,7 +46,9 @@ export class ConferenceService { const conferenceLog = new Log(); if (createConferenceDto.derivedFromId) { - const derivedFrom = await this.findOne(createConferenceDto.derivedFromId); + const derivedFrom = await manager.findOne(Conference, { + where: { id: createConferenceDto.derivedFromId }, + }); if (derivedFrom) conference.derivedFrom = derivedFrom; } @@ -49,9 +62,9 @@ export class ConferenceService { DerivedFrom: ${conference.derivedFrom?.id} `; - await AppDataSource.manager.save(conferenceLog); + await manager.save(conferenceLog); - return await AppDataSource.createQueryBuilder() + return await AppDataSource.createQueryBuilder(queryRunner) .insert() .into(Conference) .values(conference) @@ -70,18 +83,26 @@ export class ConferenceService { } async update( + queryRunner: QueryRunner | undefined, id: number, updateConferenceDto: UpdateConferenceDto, email: string, ) { - const conference = await AppDataSource.manager.findOne(Conference, { + let manager: EntityManager; + if (!queryRunner) { + manager = AppDataSource.manager; + } else { + manager = queryRunner.manager; + } + const conference = await manager.findOne(Conference, { where: { id: id }, }); if (conference) { Object.assign(conference, updateConferenceDto); - await AppDataSource.manager.save(conference); + await manager.save(conference); createLog( + queryRunner, EntityType.CONFERENCE, ` Type: Update @@ -93,13 +114,15 @@ export class ConferenceService { } } - // CronExpression.EVERY_1ST_DAY_OF_MONTH_AT_MIDNIGHT - @Cron(new Date(Date.now() + 2 * 1000), { + @Cron(new Date(Date.now() + 1000 * 3), { + // @Cron(CronExpression.EVERY_1ST_DAY_OF_MONTH_AT_MIDNIGHT, { name: 'refresh_conferences', timeZone: 'America/Recife', }) - async refresh() { - const email = 'cron_job@cin.ufpe.br'; + async refresh(email: string) { + if (!email) { + email = 'cron_job@cin.ufpe.br'; + } const csvUrl = 'https://docs.google.com/spreadsheets/d/' + process.env.CONFERENCES_SHEET_ID + @@ -110,6 +133,9 @@ export class ConferenceService { ['Qualis_Final', 'qualis'], ['evento', 'name'], ]); + const queryRunner = AppDataSource.createQueryRunner(); + await queryRunner.connect(); + await queryRunner.startTransaction(); try { const response = await axios.get(csvUrl); const refreshConferenceDtos: RefreshConferenceDto[] = Papa.parse( @@ -122,7 +148,7 @@ export class ConferenceService { console.log(refreshConferenceDtos); for (const refreshConferenceDto of refreshConferenceDtos) { - const conference = await AppDataSource.manager.findOne(Conference, { + const conference = await queryRunner.manager.findOne(Conference, { where: { acronym: refreshConferenceDto.acronym, }, @@ -132,6 +158,7 @@ export class ConferenceService { `Creating conference ${refreshConferenceDto.acronym}-${refreshConferenceDto.name}`, ); await this.create( + queryRunner, { ...refreshConferenceDto, isTop: false, @@ -147,14 +174,20 @@ export class ConferenceService { `Updating conference ${conference.acronym}-${conference.name}-${conference.qualis} to ${refreshConferenceDto.acronym}-${refreshConferenceDto.name}-${refreshConferenceDto.qualis}`, ); await this.update( + queryRunner, conference.id, { ...refreshConferenceDto, id: conference.id }, email, ); } } + await queryRunner.commitTransaction(); + return { msg: 'Conferences refreshed successfully' }; } catch (error) { - throw new Error(`Error refreshing conferences: ${error}`); + await queryRunner.rollbackTransaction(); + throw error; + } finally { + await queryRunner.release(); } } } diff --git a/src/qualis/qualis.controller.ts b/src/qualis/qualis.controller.ts index 8a39cb4..846a642 100644 --- a/src/qualis/qualis.controller.ts +++ b/src/qualis/qualis.controller.ts @@ -1,4 +1,4 @@ -import { Controller, Get, Body, Param, Patch, Post, Req } from '@nestjs/common'; +import { Controller, Get, Body, Param, Patch, Post, Res } from '@nestjs/common'; import { JournalService } from './qualis.service'; import { UpdateJournalDto } from './dto/update-journal.dto'; import { UpdateConferenceDto } from './dto/update-conference.dto'; @@ -10,6 +10,7 @@ import { CreateConferenceDto } from './dto/create-conference.dto'; import { CreateJournalDto } from './dto/create-journal.dto'; import { AuthenticatedUser, Roles } from 'nest-keycloak-connect'; import { SystemRoles } from 'src/types/enums'; +import { Response } from 'express'; @ApiTags('Qualis') @ApiOAuth2([]) @@ -31,7 +32,11 @@ export class QualisController { @AuthenticatedUser() user: any, @Body() createConferenceDto: CreateConferenceDto, ) { - return this.conferencesService.create(createConferenceDto, user.email); + return this.conferencesService.create( + undefined, + createConferenceDto, + user.email, + ); } @ApiResponse({ @@ -84,7 +89,12 @@ export class QualisController { @Param('id') id: string, @Body() updateConferenceDto: UpdateConferenceDto, ) { - return this.conferencesService.update(+id, updateConferenceDto, user.email); + return this.conferencesService.update( + undefined, + +id, + updateConferenceDto, + user.email, + ); } @ApiResponse({ @@ -101,4 +111,23 @@ export class QualisController { ) { return this.journalsService.update(+id, updateJournals, user.email); } + + @ApiResponse({ + status: 200, + description: 'Refresh Conferences', + }) + @Roles({ roles: [SystemRoles.ADMIN] }) + @Post('conferences/refresh') + async refreshConferences( + @Res() res: Response, + @AuthenticatedUser() user: any, + ) { + try { + const result = await this.conferencesService.refresh(user.email); + return res.status(201).send(result); + } catch (error: any) { + console.log({ error: error.message }); + return res.status(500).send({ message: 'Error refreshing conferences' }); + } + } } diff --git a/src/qualis/qualis.service.ts b/src/qualis/qualis.service.ts index bd2404f..8aa1bfb 100644 --- a/src/qualis/qualis.service.ts +++ b/src/qualis/qualis.service.ts @@ -77,6 +77,7 @@ export class JournalService { Object.assign(journal, updateJournalDto); await AppDataSource.manager.save(journal); createLog( + undefined, EntityType.JOURNAL_PUBLICATION, `Type: Update Email: ${email} diff --git a/src/utils/exception-filters/log-utils.ts b/src/utils/exception-filters/log-utils.ts index 53b64c2..820eb26 100644 --- a/src/utils/exception-filters/log-utils.ts +++ b/src/utils/exception-filters/log-utils.ts @@ -1,13 +1,15 @@ +import { QueryRunner } from 'typeorm'; import { Log } from './log.entity'; import { AppDataSource } from 'src/app.datasource'; const createLog = async ( + queryRunner: QueryRunner | undefined, entityType: string, message: string, entityId?: string, executionContextHost?: string, ) => { - await AppDataSource.createQueryBuilder() + await AppDataSource.createQueryBuilder(queryRunner) .insert() .into(Log) .values({ From 89cbd915ab818827188b606de44f9de5b138f941 Mon Sep 17 00:00:00 2001 From: Raul Coelho Date: Tue, 23 Apr 2024 16:54:48 -0300 Subject: [PATCH 06/12] updating Cron Expression --- src/qualis/conference/conference.service.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/qualis/conference/conference.service.ts b/src/qualis/conference/conference.service.ts index 4d9460e..582e299 100644 --- a/src/qualis/conference/conference.service.ts +++ b/src/qualis/conference/conference.service.ts @@ -114,8 +114,8 @@ export class ConferenceService { } } - @Cron(new Date(Date.now() + 1000 * 3), { - // @Cron(CronExpression.EVERY_1ST_DAY_OF_MONTH_AT_MIDNIGHT, { + // @Cron(new Date(Date.now() + 1000 * 3), { + @Cron(CronExpression.EVERY_1ST_DAY_OF_MONTH_AT_MIDNIGHT, { name: 'refresh_conferences', timeZone: 'America/Recife', }) From 440f409d850b282f1e9d63bf39caa2c35b932e9b Mon Sep 17 00:00:00 2001 From: Raul Coelho Date: Wed, 24 Apr 2024 14:54:18 -0300 Subject: [PATCH 07/12] refresh conferences refactored and debug logs removed --- src/qualis/conference/conference.service.ts | 51 +++++++++------------ src/qualis/qualis.controller.ts | 3 +- 2 files changed, 23 insertions(+), 31 deletions(-) diff --git a/src/qualis/conference/conference.service.ts b/src/qualis/conference/conference.service.ts index 582e299..722acd1 100644 --- a/src/qualis/conference/conference.service.ts +++ b/src/qualis/conference/conference.service.ts @@ -114,7 +114,6 @@ export class ConferenceService { } } - // @Cron(new Date(Date.now() + 1000 * 3), { @Cron(CronExpression.EVERY_1ST_DAY_OF_MONTH_AT_MIDNIGHT, { name: 'refresh_conferences', timeZone: 'America/Recife', @@ -123,30 +122,11 @@ export class ConferenceService { if (!email) { email = 'cron_job@cin.ufpe.br'; } - const csvUrl = - 'https://docs.google.com/spreadsheets/d/' + - process.env.CONFERENCES_SHEET_ID + - '/gviz/tq?tqx=out:csv&sheet=Qualis'; - - const headers = new Map([ - ['sigla', 'acronym'], - ['Qualis_Final', 'qualis'], - ['evento', 'name'], - ]); const queryRunner = AppDataSource.createQueryRunner(); await queryRunner.connect(); await queryRunner.startTransaction(); try { - const response = await axios.get(csvUrl); - const refreshConferenceDtos: RefreshConferenceDto[] = Papa.parse( - response.data, - { - header: true, - transformHeader: (header) => headers.get(header) || header, - }, - ).data as RefreshConferenceDto[]; - console.log(refreshConferenceDtos); - + const refreshConferenceDtos = await this.getSheetData(); for (const refreshConferenceDto of refreshConferenceDtos) { const conference = await queryRunner.manager.findOne(Conference, { where: { @@ -154,9 +134,6 @@ export class ConferenceService { }, }); if (!conference) { - console.log( - `Creating conference ${refreshConferenceDto.acronym}-${refreshConferenceDto.name}`, - ); await this.create( queryRunner, { @@ -170,9 +147,6 @@ export class ConferenceService { refreshConferenceDto.name !== conference.name || refreshConferenceDto.qualis !== conference.qualis ) { - console.log( - `Updating conference ${conference.acronym}-${conference.name}-${conference.qualis} to ${refreshConferenceDto.acronym}-${refreshConferenceDto.name}-${refreshConferenceDto.qualis}`, - ); await this.update( queryRunner, conference.id, @@ -182,12 +156,31 @@ export class ConferenceService { } } await queryRunner.commitTransaction(); - return { msg: 'Conferences refreshed successfully' }; - } catch (error) { + return { message: 'Conferences refreshed successfully' }; + } catch (error: any) { await queryRunner.rollbackTransaction(); + console.log(error.message); throw error; } finally { await queryRunner.release(); } } + + async getSheetData(): Promise { + const csvUrl = + 'https://docs.google.com/spreadsheets/d/' + + process.env.CONFERENCES_SHEET_ID + + '/gviz/tq?tqx=out:csv&sheet=Qualis'; + + const headers = new Map([ + ['sigla', 'acronym'], + ['Qualis_Final', 'qualis'], + ['evento', 'name'], + ]); + const response = await axios.get(csvUrl); + return Papa.parse(response.data, { + header: true, + transformHeader: (header) => headers.get(header) || header, + }).data as RefreshConferenceDto[]; + } } diff --git a/src/qualis/qualis.controller.ts b/src/qualis/qualis.controller.ts index 846a642..98e8119 100644 --- a/src/qualis/qualis.controller.ts +++ b/src/qualis/qualis.controller.ts @@ -125,8 +125,7 @@ export class QualisController { try { const result = await this.conferencesService.refresh(user.email); return res.status(201).send(result); - } catch (error: any) { - console.log({ error: error.message }); + } catch (error) { return res.status(500).send({ message: 'Error refreshing conferences' }); } } From 93a902f97b352294f114fc40936cc697d352d96a Mon Sep 17 00:00:00 2001 From: Raul Coelho Date: Thu, 25 Apr 2024 15:39:36 -0300 Subject: [PATCH 08/12] refactoring create, update and refresh conferences --- src/qualis/conference/conference.service.ts | 32 ++++++++++----------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/src/qualis/conference/conference.service.ts b/src/qualis/conference/conference.service.ts index 722acd1..aeac8b9 100644 --- a/src/qualis/conference/conference.service.ts +++ b/src/qualis/conference/conference.service.ts @@ -29,12 +29,9 @@ export class ConferenceService { createConferenceDto: CreateConferenceDto, email: string, ) { - let manager: EntityManager; - if (!queryRunner) { - manager = AppDataSource.manager; - } else { - manager = queryRunner.manager; - } + const manager: EntityManager = !queryRunner + ? AppDataSource.manager + : queryRunner.manager; const conference = new Conference(); conference.acronym = createConferenceDto.acronym; @@ -88,12 +85,9 @@ export class ConferenceService { updateConferenceDto: UpdateConferenceDto, email: string, ) { - let manager: EntityManager; - if (!queryRunner) { - manager = AppDataSource.manager; - } else { - manager = queryRunner.manager; - } + const manager: EntityManager = !queryRunner + ? AppDataSource.manager + : queryRunner.manager; const conference = await manager.findOne(Conference, { where: { id: id }, }); @@ -113,11 +107,15 @@ export class ConferenceService { ); } } - - @Cron(CronExpression.EVERY_1ST_DAY_OF_MONTH_AT_MIDNIGHT, { - name: 'refresh_conferences', - timeZone: 'America/Recife', - }) + // 1st day of the month if there is no env variable: + @Cron( + process.env.CRON_PATTERN || + CronExpression.EVERY_1ST_DAY_OF_MONTH_AT_MIDNIGHT, + { + name: 'refresh_conferences', + timeZone: 'America/Recife', + }, + ) async refresh(email: string) { if (!email) { email = 'cron_job@cin.ufpe.br'; From ad22aef2fc36fb26589b92e56750042f2d6ba2d9 Mon Sep 17 00:00:00 2001 From: Raul Coelho Date: Thu, 25 Apr 2024 15:40:32 -0300 Subject: [PATCH 09/12] refresh journals implemented with transaction --- src/qualis/dto/refresh-journal.dto.ts | 20 +++++ src/qualis/qualis.controller.ts | 26 +++++- src/qualis/qualis.service.ts | 112 ++++++++++++++++++++++---- 3 files changed, 141 insertions(+), 17 deletions(-) create mode 100644 src/qualis/dto/refresh-journal.dto.ts diff --git a/src/qualis/dto/refresh-journal.dto.ts b/src/qualis/dto/refresh-journal.dto.ts new file mode 100644 index 0000000..a1aabb0 --- /dev/null +++ b/src/qualis/dto/refresh-journal.dto.ts @@ -0,0 +1,20 @@ +import { ApiProperty, ApiTags } from '@nestjs/swagger'; +import { IsNotEmpty, IsString } from 'class-validator'; + +@ApiTags('RefreshJournalDto') +export class RefreshJournalDto { + @ApiProperty({ name: 'name', type: String }) + @IsString() + @IsNotEmpty() + name!: string; + + @ApiProperty({ name: 'issn', type: String }) + @IsString() + @IsNotEmpty() + issn!: string; + + @ApiProperty({ name: 'qualis', type: String }) + @IsString() + @IsNotEmpty() + qualis!: string; +} diff --git a/src/qualis/qualis.controller.ts b/src/qualis/qualis.controller.ts index 98e8119..ffbd810 100644 --- a/src/qualis/qualis.controller.ts +++ b/src/qualis/qualis.controller.ts @@ -50,7 +50,7 @@ export class QualisController { @Body() createJournalDto: CreateJournalDto, @AuthenticatedUser() user: any, ) { - return this.journalsService.create(createJournalDto, user.email); + return this.journalsService.create(undefined, createJournalDto, user.email); } @ApiResponse({ @@ -109,11 +109,16 @@ export class QualisController { @Body() updateJournals: UpdateJournalDto, @AuthenticatedUser() user: any, ) { - return this.journalsService.update(+id, updateJournals, user.email); + return this.journalsService.update( + undefined, + +id, + updateJournals, + user.email, + ); } @ApiResponse({ - status: 200, + status: 201, description: 'Refresh Conferences', }) @Roles({ roles: [SystemRoles.ADMIN] }) @@ -129,4 +134,19 @@ export class QualisController { return res.status(500).send({ message: 'Error refreshing conferences' }); } } + + @ApiResponse({ + status: 201, + description: 'Refresh Journals', + }) + @Roles({ roles: [SystemRoles.ADMIN] }) + @Post('journals/refresh') + async refreshJournals(@Res() res: Response, @AuthenticatedUser() user: any) { + try { + const result = await this.journalsService.refresh(user.email); + return res.status(201).send(result); + } catch (error) { + return res.status(500).send({ message: 'Error refreshing journals' }); + } + } } diff --git a/src/qualis/qualis.service.ts b/src/qualis/qualis.service.ts index 8aa1bfb..d1b7c21 100644 --- a/src/qualis/qualis.service.ts +++ b/src/qualis/qualis.service.ts @@ -8,10 +8,18 @@ import { EntityType } from 'src/utils/exception-filters/entity-type-enum'; import { AppDataSource } from 'src/app.datasource'; import createLog from 'src/utils/exception-filters/log-utils'; import { Cron, CronExpression } from '@nestjs/schedule'; +import Papa from 'papaparse'; +import axios from 'axios'; +import { RefreshJournalDto } from './dto/refresh-journal.dto'; @Injectable() export class JournalService { - async create(createJournalDto: CreateJournalDto, email: string) { + async create( + queryRunner: QueryRunner | undefined, + createJournalDto: CreateJournalDto, + email: string, + ) { + const manager = !queryRunner ? AppDataSource.manager : queryRunner.manager; const journal = new Journal(); journal.name = createJournalDto.name; journal.issn = createJournalDto.issn; @@ -31,16 +39,18 @@ export class JournalService { `; if (createJournalDto.derivedFromId) { - const derivedFrom = await this.findOne(createJournalDto.derivedFromId); + const derivedFrom = await manager.findOne(Journal, { + where: { id: createJournalDto.derivedFromId }, + }); if (derivedFrom) { journal.derivedFrom = derivedFrom; journalLog.message += `derivedFromId: ${derivedFrom.id}`; } } - await AppDataSource.manager.save(journalLog); + await manager.save(journalLog); - return await AppDataSource.createQueryBuilder() + return await AppDataSource.createQueryBuilder(queryRunner) .insert() .into(Journal) .values(journal) @@ -68,16 +78,22 @@ export class JournalService { return journal; } - async update(id: number, updateJournalDto: UpdateJournalDto, email: string) { - const journal = await AppDataSource.manager.findOne(Journal, { + async update( + queryRunner: QueryRunner | undefined, + id: number, + updateJournalDto: UpdateJournalDto, + email: string, + ) { + const manager = !queryRunner ? AppDataSource.manager : queryRunner.manager; + const journal = await manager.findOne(Journal, { where: { id: id }, }); if (journal) { Object.assign(journal, updateJournalDto); - await AppDataSource.manager.save(journal); + await manager.save(journal); createLog( - undefined, + queryRunner, EntityType.JOURNAL_PUBLICATION, `Type: Update Email: ${email} @@ -91,11 +107,79 @@ export class JournalService { return `This action removes a #${id} journal`; } - @Cron(CronExpression.EVERY_1ST_DAY_OF_MONTH_AT_MIDNIGHT, { - name: 'refresh_journals', - timeZone: 'America/Recife', - }) - async refresh() { - console.log('Refreshing journals'); + // 1st day of the month if there is no env variable: + @Cron( + process.env.CRON_PATTERN || + CronExpression.EVERY_1ST_DAY_OF_MONTH_AT_MIDNIGHT, + { + name: 'refresh_journals', + timeZone: 'America/Recife', + }, + ) + async refresh(email: string) { + if (!email) { + email = 'cron_job@cin.ufpe.br'; + } + const queryRunner = AppDataSource.createQueryRunner(); + await queryRunner.connect(); + await queryRunner.startTransaction(); + try { + const refreshJournalDtos = await this.getSheetData(); + console.log(refreshJournalDtos); + + for (const refreshJournalDto of refreshJournalDtos) { + const journal = await queryRunner.manager.findOne(Journal, { + where: { issn: refreshJournalDto.issn }, + }); + + if (!journal) { + const createJournalDto: CreateJournalDto = { + ...refreshJournalDto, + isTop: false, + official: true, + }; + const derivedFrom = await queryRunner.manager.findOne(Journal, { + where: { name: refreshJournalDto.name }, + }); + if (derivedFrom) { + createJournalDto.derivedFromId = derivedFrom.id; + } + await this.create(queryRunner, createJournalDto, email); + } else if ( + refreshJournalDto.name !== journal.name || + refreshJournalDto.qualis !== journal.qualis + ) { + const updateJournalDto: UpdateJournalDto = { + ...refreshJournalDto, + id: journal.id, + }; + await this.update(queryRunner, journal.id, updateJournalDto, email); + } + } + await queryRunner.commitTransaction(); + } catch (error: any) { + await queryRunner.rollbackTransaction(); + console.log(error.message); + throw error; + } finally { + await queryRunner.release(); + } + } + + async getSheetData(): Promise { + const csvUrl = + 'https://docs.google.com/spreadsheets/d/' + + process.env.JOURNALS_SHEET_ID + + '/gviz/tq?tqx=out:csv&sheet=Qualis'; + + const response = await axios.get(csvUrl); + const data = Papa.parse(response.data, { + header: true, + }).data; + return data.map((journal: any) => ({ + issn: journal['issn'].replace('-', ''), + name: journal['periodico'], + qualis: journal['Qualis_Final'], + })); } } From d03519572485b32bb49e9907db77bec63dee2161 Mon Sep 17 00:00:00 2001 From: Raul Coelho Date: Tue, 30 Apr 2024 16:15:10 -0300 Subject: [PATCH 10/12] refreshJournals considering multiple journals with the same issn there is a possibility that two journals have the same issn. For example, the print version and the online version. Now the code considers that possibility. --- src/qualis/qualis.service.ts | 42 ++++++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/src/qualis/qualis.service.ts b/src/qualis/qualis.service.ts index d1b7c21..c786332 100644 --- a/src/qualis/qualis.service.ts +++ b/src/qualis/qualis.service.ts @@ -128,11 +128,11 @@ export class JournalService { console.log(refreshJournalDtos); for (const refreshJournalDto of refreshJournalDtos) { - const journal = await queryRunner.manager.findOne(Journal, { + const journals = await queryRunner.manager.find(Journal, { where: { issn: refreshJournalDto.issn }, }); - if (!journal) { + if (journals.length === 0) { const createJournalDto: CreateJournalDto = { ...refreshJournalDto, isTop: false, @@ -146,17 +146,35 @@ export class JournalService { } await this.create(queryRunner, createJournalDto, email); } else if ( - refreshJournalDto.name !== journal.name || - refreshJournalDto.qualis !== journal.qualis + journals.length === 1 && + (refreshJournalDto.name !== journals[0].name || + refreshJournalDto.qualis !== journals[0].qualis) ) { const updateJournalDto: UpdateJournalDto = { ...refreshJournalDto, - id: journal.id, + id: journals[0].id, }; - await this.update(queryRunner, journal.id, updateJournalDto, email); + await this.update( + queryRunner, + journals[0].id, + updateJournalDto, + email, + ); + } else if ( + journals.length > 1 && + refreshJournalDto.qualis !== journals[0].qualis + ) { + for (const journal of journals) { + const updateJournalDto: UpdateJournalDto = { + ...journal, + qualis: refreshJournalDto.qualis, + }; + await this.update(queryRunner, journal.id, updateJournalDto, email); + } } } await queryRunner.commitTransaction(); + return { message: 'Journals refreshed successfully' }; } catch (error: any) { await queryRunner.rollbackTransaction(); console.log(error.message); @@ -176,10 +194,12 @@ export class JournalService { const data = Papa.parse(response.data, { header: true, }).data; - return data.map((journal: any) => ({ - issn: journal['issn'].replace('-', ''), - name: journal['periodico'], - qualis: journal['Qualis_Final'], - })); + return data + .filter((journal: any) => journal.issn !== 'nulo') + .map((journal: any) => ({ + issn: journal['issn'].replace('-', ''), + name: journal['periodico'], + qualis: journal['Qualis_Final'], + })); } } From 2e6bd0849975af3bbcfcfb841a2cc0f4e7eea66b Mon Sep 17 00:00:00 2001 From: Raul Coelho Date: Mon, 6 May 2024 16:51:35 -0300 Subject: [PATCH 11/12] removing refreshJournalDtos logs --- src/qualis/qualis.service.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/qualis/qualis.service.ts b/src/qualis/qualis.service.ts index c786332..2811bcc 100644 --- a/src/qualis/qualis.service.ts +++ b/src/qualis/qualis.service.ts @@ -125,8 +125,6 @@ export class JournalService { await queryRunner.startTransaction(); try { const refreshJournalDtos = await this.getSheetData(); - console.log(refreshJournalDtos); - for (const refreshJournalDto of refreshJournalDtos) { const journals = await queryRunner.manager.find(Journal, { where: { issn: refreshJournalDto.issn }, From f0810542363c661e20b2f6af2a510c048de066b8 Mon Sep 17 00:00:00 2001 From: Raul Coelho Date: Fri, 10 May 2024 12:32:33 -0300 Subject: [PATCH 12/12] adding CRON_PATTERN to env.example --- .env.example | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.env.example b/.env.example index dced2a4..99ce0c7 100644 --- a/.env.example +++ b/.env.example @@ -21,5 +21,7 @@ TOKEN_URL=https://localhost:8080/realms/your_realm/protocol/openid-connect/token REDIRECT_URL=https://localhost:3000/ CLIENT_ID=research-back +SERVER_URL=https://localhost:3001 CONFERENCES_SHEET_ID=1yvuCa__L7r0EJy6v6Jb17fvu-VdV80PbfAReR9Gy52I -JOURNALS_SHEET_ID=10sObNyyL7veHGFbOyizxM8oVsppQoWV-0ALrDr8FxQ0 \ No newline at end of file +JOURNALS_SHEET_ID=10sObNyyL7veHGFbOyizxM8oVsppQoWV-0ALrDr8FxQ0 +CRON_PATTERN=0 0 1 * * \ No newline at end of file