diff --git a/server/package.json b/server/package.json index d89345c9c..072079eb2 100644 --- a/server/package.json +++ b/server/package.json @@ -38,7 +38,7 @@ "remove:apprentis-nonRecus-depuis": "node -r dotenv/config src/jobs/remove-data/statuts-apprentis-nonRecus-depuis/index.js", "remove:dossiersApprenants-nonRecus-depuis": "node -r dotenv/config src/jobs/remove-data/dossiers-apprenants-nonRecus-depuis/index.js", "purge:events": "node -r dotenv/config src/jobs/remove-data/purgeEvents/index.js", - "formation:retrieve-from-cfd": "node -r dotenv/config src/jobs/formation-retrieve-from-cfd/index.js", + "seed:formations": "node -r dotenv/config src/jobs/seed/seedFormations.js", "create:effectifsApprenants-collection": "node -r dotenv/config src/jobs/create-effectifs-apprenants-collection/index.js", "dossiersApprenants:retrieve-networks": "node -r dotenv/config src/jobs/dossiersApprenants-retrieve-networks/index.js", "support:identify-network-duplicates": "node -r dotenv/config src/jobs/support/identify-network-duplicates/index.js", diff --git a/server/src/common/components/dossiersApprenants.js b/server/src/common/components/dossiersApprenants.js index 138f18491..bbf3149c8 100644 --- a/server/src/common/components/dossiersApprenants.js +++ b/server/src/common/components/dossiersApprenants.js @@ -2,10 +2,8 @@ const { DossierApprenantModel, CfaModel } = require("../model"); const omit = require("lodash.omit"); const { DUPLICATE_TYPE_CODES } = require("../constants/dossierApprenantConstants"); const { asyncForEach } = require("../../common/utils/asyncUtils"); -const { validateCfd } = require("../domain/cfd"); const { escapeRegExp } = require("../utils/regexUtils"); const { isEqual } = require("date-fns"); -const { existsFormation, createFormation, getFormationWithCfd } = require("./formations")(); const { DossierApprenant } = require("../factory/dossierApprenant"); const { faker } = require("@faker-js/faker/locale/fr"); @@ -138,13 +136,6 @@ const createDossierApprenant = async (itemToCreate) => { // if dossier apprenant établissement has a VALID uai try to retrieve information in Referentiel CFAs const etablissementInReferentielCfaFromUai = await CfaModel.findOne({ uai: itemToCreate.uai_etablissement }); - // if dossier apprenant has a valid cfd, check if it exists in db and create it otherwise - if (validateCfd(itemToCreate.formation_cfd) && !(await existsFormation(itemToCreate.formation_cfd))) { - await createFormation(itemToCreate.formation_cfd); - } - - const formationInfo = await getFormationWithCfd(itemToCreate.formation_cfd); - const dossierApprenantEntity = DossierApprenant.create({ ine_apprenant: itemToCreate.ine_apprenant, nom_apprenant: itemToCreate.nom_apprenant.toUpperCase(), @@ -152,8 +143,6 @@ const createDossierApprenant = async (itemToCreate) => { email_contact: itemToCreate.email_contact, formation_cfd: itemToCreate.formation_cfd, libelle_long_formation: itemToCreate.libelle_long_formation, - niveau_formation: formationInfo?.niveau, - niveau_formation_libelle: formationInfo?.niveau_libelle, uai_etablissement: itemToCreate.uai_etablissement, siret_etablissement: itemToCreate.siret_etablissement, nom_etablissement: itemToCreate.nom_etablissement, diff --git a/server/src/common/constants/jobsConstants.js b/server/src/common/constants/jobsConstants.js index daf4d5244..e6ed313c7 100644 --- a/server/src/common/constants/jobsConstants.js +++ b/server/src/common/constants/jobsConstants.js @@ -9,7 +9,6 @@ const JOB_NAMES = { seedRandomizedSample: "seed-randomized-sample", identifyUaisInCatalog: "identify-uais-types-catalog", identifyNetworkDuplicates: "identify-network-duplicates", - formationRetrieveFromCfd: "formation-retrieve-from-cfd", dossiersApprenantsRetrieveNetworks: "dossiersApprenants-retrieve-networks", dossiersApprenantsRetrieveNiveaux: "dossiersApprenants-retrieve-niveaux", dossiersApprenantsRetrieveFormateurGestionnairesCatalog: "dossiersApprenants-retrieve-formateur-gestionnaire-catalog", diff --git a/server/src/jobs/dossiersApprenants-retrieve-niveaux/index.js b/server/src/jobs/dossiersApprenants-retrieve-niveaux/index.js deleted file mode 100644 index 0a0d1aeea..000000000 --- a/server/src/jobs/dossiersApprenants-retrieve-niveaux/index.js +++ /dev/null @@ -1,44 +0,0 @@ -const { runScript } = require("../scriptWrapper"); -const logger = require("../../common/logger"); -const cliProgress = require("cli-progress"); -const { DossierApprenantModel, FormationModel } = require("../../common/model"); -const { asyncForEach } = require("../../common/utils/asyncUtils"); -const { JOB_NAMES } = require("../../common/constants/jobsConstants"); - -const loadingBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic); - -/** - * Ce script permet de mettre à jour les niveaux des DossierApprenant en se basant sur le référentiel Formations - * Pour chaque formation du référentiel ayant un niveau, on met à jour les statuts matchant sur le cfd - */ -runScript(async () => { - logger.info("Run DossierApprenant - Niveau Retrieving Job"); - - // get all valid formations with niveau & cfd not empty - const formationsWithNiveau = await FormationModel.find({ niveau: { $ne: "" }, cfd: { $ne: "" } }); - logger.info(`${formationsWithNiveau.length} Formations with niveau & cfd`); - - loadingBar.start(formationsWithNiveau.length, 0); - - await asyncForEach(formationsWithNiveau, async (currentFormation) => { - // get all DossierApprenant for this cfd - const statutsForFormation = await DossierApprenantModel.find({ formation_cfd: currentFormation.cfd }); - - // Update niveau for all DossierApprenant - await asyncForEach(statutsForFormation, async (currentStatutToUpdate) => { - await DossierApprenantModel.findByIdAndUpdate( - currentStatutToUpdate._id, - { - niveau_formation: currentFormation.niveau, - niveau_formation_libelle: currentFormation.niveau_libelle, - }, - { new: true } - ); - }); - - loadingBar.increment(); - }); - - loadingBar.stop(); - logger.info("End DossierApprenant - Niveau Retrieving Job"); -}, JOB_NAMES.dossiersApprenantsRetrieveNiveaux); diff --git a/server/src/jobs/formation-retrieve-from-cfd/index.js b/server/src/jobs/formation-retrieve-from-cfd/index.js deleted file mode 100644 index 055519e56..000000000 --- a/server/src/jobs/formation-retrieve-from-cfd/index.js +++ /dev/null @@ -1,39 +0,0 @@ -const { runScript } = require("../scriptWrapper"); -const logger = require("../../common/logger"); -const { asyncForEach } = require("../../common/utils/asyncUtils"); -const { JOB_NAMES } = require("../../common/constants/jobsConstants"); - -/* - Ce script récupère tous les CFDs valides présents dans la collection DossierApprenant et crée une formation en base pour chacun si elle n'existe pas -*/ -runScript(async ({ db, formations }) => { - logger.info("Run Retrieve Formations from CFD"); - - let createdFormationsCount = 0; - let formationsNotCreated = []; - - const collection = db.collection("dossiersApprenants"); - - // get all valid CFDs - const allCfds = await collection.distinct("formation_cfd"); - logger.info(`${allCfds.length} distinct CFDs found in collection DossierApprenant`); - - // for each CFD, if no formation exists in DB, create and store it - await asyncForEach(allCfds, async (cfd) => { - const formationExistsInDb = await formations.existsFormation(cfd); - if (!formationExistsInDb) { - try { - const created = await formations.createFormation(cfd); - created ? createdFormationsCount++ : formationsNotCreated.push(cfd); - } catch (err) { - formationsNotCreated.push(cfd); - logger.error("error while creating formation for CFD", cfd); - logger.error(err); - } - } - }); - - logger.info(`${createdFormationsCount} formations created in DB`); - logger.warn(`${formationsNotCreated.length} formations could not be created. CFD list: ${formationsNotCreated}`); - logger.info("End Retrieve Formations from CFD"); -}, JOB_NAMES.formationRetrieveFromCfd); diff --git a/server/src/jobs/seed/seedFormations.js b/server/src/jobs/seed/seedFormations.js new file mode 100644 index 000000000..8a8ee26f1 --- /dev/null +++ b/server/src/jobs/seed/seedFormations.js @@ -0,0 +1,56 @@ +const { runScript } = require("../scriptWrapper"); +const logger = require("../../common/logger"); +const { asyncForEach } = require("../../common/utils/asyncUtils"); +const { sleep } = require("../../common/utils/miscUtils"); + +const SLEEP_TIME_BETWEEN_CREATION = 100; // 100ms to avoid flooding TCO and LBA APIs + +/* + Ce script récupère tous les CFDs valides présents dans la collection DossierApprenant, crée une formation en base + pour chacun si elle n'existe pas et ajoute les infos de la formation aux dossiers apprenants correspondant +*/ +runScript(async ({ db, formations }) => { + let createdFormationsCount = 0; + let notCreatedFormationsCount = 0; + let dossiersApprenantUpdatedCount = 0; + + // get all CFDs from dossiers apprenants collection + const allCfds = await db.collection("dossiersApprenants").distinct("formation_cfd"); + logger.info(allCfds.length, "distinct CFD found in collection DossierApprenant"); + + // filter out CFD for which we already have a formation in db + const formationsCfdToCreate = []; + await asyncForEach(allCfds, async (cfd) => { + const formationExistsInDb = await formations.existsFormation(cfd); + if (!formationExistsInDb) formationsCfdToCreate.push(cfd); + }); + + logger.info(formationsCfdToCreate.length, "formations should be created"); + + // create a formation for every "new" CFD and update related dossiers apprenants + await asyncForEach(formationsCfdToCreate, async (cfd) => { + try { + const createdFormation = await formations.createFormation(cfd); + createdFormationsCount++; + + const { result: dossierApprenantsUpdateResult } = await db.collection("dossiersApprenants").updateMany( + { formation_cfd: cfd }, + { + $set: { + niveau_formation: createdFormation.niveau, + niveau_formation_libelle: createdFormation.niveau_libelle, + }, + } + ); + dossiersApprenantUpdatedCount += dossierApprenantsUpdateResult.nModified; + } catch (err) { + logger.error("error while creating formation for CFD", cfd, err); + notCreatedFormationsCount++; + } + await sleep(SLEEP_TIME_BETWEEN_CREATION); + }); + + logger.info(`${createdFormationsCount} formations created in DB`); + logger.warn(`${notCreatedFormationsCount} formations could not be created`); + logger.info(`${dossiersApprenantUpdatedCount} dossiers apprenants updated with formation info`); +}, "seed:formations"); diff --git a/server/tests/integration/common/components/dossiersApprenants.test.js b/server/tests/integration/common/components/dossiersApprenants.test.js index 2d5c71d03..9a7fcca7d 100644 --- a/server/tests/integration/common/components/dossiersApprenants.test.js +++ b/server/tests/integration/common/components/dossiersApprenants.test.js @@ -3,14 +3,13 @@ const assert = require("assert").strict; const MockDate = require("mockdate"); const { addDays, isEqual } = require("date-fns"); const dossiersApprenants = require("../../../../src/common/components/dossiersApprenants"); -const { DossierApprenantModel, CfaModel, FormationModel } = require("../../../../src/common/model"); +const { DossierApprenantModel, CfaModel } = require("../../../../src/common/model"); const { createRandomDossierApprenant, getRandomUaiEtablissement } = require("../../../data/randomizedSample"); const { CODES_STATUT_APPRENANT, DUPLICATE_TYPE_CODES, } = require("../../../../src/common/constants/dossierApprenantConstants"); const { RESEAUX_CFAS } = require("../../../../src/common/constants/networksConstants"); -const { Formation } = require("../../../../src/common/factory/formation"); describe(__filename, () => { let fakeNowDate; @@ -1022,50 +1021,6 @@ describe(__filename, () => { assert.equal(etablissement_reseaux[0], RESEAUX_CFAS.ANASUP.nomReseau); assert.equal(etablissement_reseaux[1], RESEAUX_CFAS.BTP_CFA.nomReseau); }); - - it("Vérifie qu'à la création d'un statut avec un CFD valide on crée la formation correspondante si elle n'existe pas", async () => { - const { createDossierApprenant } = await dossiersApprenants(); - - // Create statut - const cfd = "01022104"; - const statutWithValidCfd = { ...createRandomDossierApprenant(), formation_cfd: cfd }; - const createdStatut = await createDossierApprenant(statutWithValidCfd); - - assert.ok(createdStatut); - // Check that formation was created - const foundFormations = await FormationModel.find(); - assert.equal(foundFormations.length, 1); - assert.equal(foundFormations[0].cfd, cfd); - }); - - it("Vérifie qu'à la création d'un statut avec un CFD valide on ne crée pas de formation si elle existe", async () => { - const { createDossierApprenant } = await dossiersApprenants(); - - // Create Formation - const formationSeed = { - cfd: "01022103", - rncps: ["RNCP31811"], - libelle: "EMPLOYE TRAITEUR (CAP)", - cfd_start_date: new Date("2021-08-31").toISOString(), - cfd_end_date: new Date("2022-08-31").toISOString(), - }; - const formation = Formation.create(formationSeed); - await new FormationModel(formation).save(); - - // Create statut - const statutWithValidCfd = { - ...createRandomDossierApprenant(), - formation_cfd: formationSeed.cfd, - }; - const createdStatut = await createDossierApprenant(statutWithValidCfd); - assert.ok(createdStatut); - - // Find formations - const foundFormations = await FormationModel.find(); - - assert.equal(foundFormations.length, 1); - assert.equal(foundFormations[0].cfd, createdStatut.formation_cfd); - }); }); describe("getDuplicatesList", () => {