diff --git a/src/tasks/correction/correctionDoublonsPermanences.js b/src/tasks/correction/correctionDoublonsPermanences.js new file mode 100644 index 0000000..b5aea9e --- /dev/null +++ b/src/tasks/correction/correctionDoublonsPermanences.js @@ -0,0 +1,43 @@ +#!/usr/bin/env node +'use strict'; +const csv = require('csv-parser'); +const fs = require('fs'); +const { encrypt } = require('../../utils/encrypt'); +const { execute } = require('../utils'); + +const updateCraWithIdPermanence = dbDatalake => async (id, ids) => await dbDatalake.collection('cras').updateMany( + { 'permanenceId': { '$in': ids } }, + { 'permanenceId': id } +); + +execute(__filename, async ({ logger, dbDatalake }) => { + + const permanences = []; + const promises = []; + let nbMaj = 0; + + logger.info('Modification des ids suite à la fusion des doublons de permanences'); + + //eslint-disable-next-line max-len + //idPermanence|estStructure|nomEnseigne|numeroTelephone|email|siteWeb|siret|adresse|location|horaires|typeAcces|conseillers|lieuPrincipalPour|conseillersItinerants|structure|updatedAt|updatedBy|doublons + fs.createReadStream('data/imports/permanences-doublons.csv') + .pipe(csv({ separator: ';' })) + .on('data', data => permanences.push(data)) + .on('end', () => { + permanences.forEach(permanence => { + promises.push(new Promise(async resolve => { + const id = encrypt(permanence.idPermanence.toString()); + const ids = []; + permanence.doublons.split(',').forEach(id => { + ids.push(encrypt(id.toString())); + }); + await updateCraWithIdPermanence(dbDatalake)(id, ids); + nbMaj++; + resolve(); + })); + }); + Promise.all(promises); + }); + + logger.info(`${nbMaj} CRAs mis à jour`); +}); diff --git a/src/tasks/scripts/extract-cras-csv.js b/src/tasks/scripts/extract-cras-csv.js new file mode 100644 index 0000000..eef5521 --- /dev/null +++ b/src/tasks/scripts/extract-cras-csv.js @@ -0,0 +1,27 @@ +#!/usr/bin/env node +'use strict'; +const path = require('path'); +const fs = require('fs'); +const { program } = require('commander'); +const { execute } = require('../utils'); + +require('dotenv').config(); + +execute(__filename, async ({ logger, dbDatalake }) => { + program.helpOption('-e', 'HELP command'); + program.parse(process.argv); + + const cursor = await dbDatalake.collection('cras').find(); + + logger.info(`Generating CSV file...`); + let csvFile = path.join(__dirname, '../../../data/exports', `cras.csv`); + let file = fs.createWriteStream(csvFile, { flags: 'w' }); + + file.write(`ID;ConseillerId;Cra;CreatedAt;UpdatedAt;PermanenceId;StructureId\n`); + while (await cursor.hasNext()) { + const cra = await cursor.next(); + file.write(`${cra._id};${cra.conseillerId};${JSON.stringify(cra.cra)};${cra.createdAt};${cra.updatedAt};${cra.permanenceId};${cra.structureId};\n`); + } + + file.close(); +}); diff --git a/src/tasks/scripts/replace-cras-datalake.js b/src/tasks/scripts/replace-cras-datalake.js new file mode 100644 index 0000000..db1408f --- /dev/null +++ b/src/tasks/scripts/replace-cras-datalake.js @@ -0,0 +1,83 @@ +#!/usr/bin/env node +'use strict'; + +const { execute } = require('../utils'); +const { encrypt } = require('../../utils/encrypt'); + +// node src/tasks/scripts/replace-cras-datalake.js + +execute(__filename, async ({ logger, db, dbDatalake }) => { + let startDate = new Date(); + startDate.setDate(startDate.getDate() - 2); //Max 2 jours avant car on laissera le CRON pour les CRAS effectué dans les 2 derniers jours... + startDate.setUTCHours(0, 0, 0, 0); + const sleep = ms => new Promise(r => setTimeout(r, ms)); + const countTotalCras = await db.collection('cras').countDocuments({ $or: [ + { createdAt: { $lte: startDate } }, + { updatedAt: { $lte: startDate } } + ] }); + + const calculLots = () => { + const result = []; + const limit = 100000; + const lot = Math.ceil(countTotalCras / limit); + for (let i = 0; i < lot; i++) { + if (countTotalCras <= (i * limit)) { + break; + } + result.push({ id: i, limit, skip: i * limit }); + } + return result; + }; + + const lotsArray = await calculLots(); + + logger.info(`Début des ${lotsArray.length} lots...`); + + for (const lot of lotsArray) { + const cras = await db.collection('cras').find({ $or: [ + { createdAt: { $lte: startDate } }, + { updatedAt: { $lte: startDate } } + ] }).limit(lot.limit).skip(lot.skip).toArray(); + const promises = []; + let count = 0; + await cras.forEach(cra => { + promises.push(new Promise(async resolve => { + const whitelist = [ + '_id', + 'cra', + 'conseiller', + 'structure', + 'permanence', + 'createdAt', + 'updatedAt' + ]; + + for (const property in cra) { + if (!whitelist.includes(property)) { + delete cra[property]; + } + } + + cra._id = encrypt(cra._id.toString()); + cra.conseillerId = encrypt(cra.conseiller.oid.toString()); + if (cra.permanence) { + cra.permanenceId = encrypt(cra.permanence.oid.toString()); + + } + if (cra.structure) { + cra.structureId = encrypt(cra.structure.oid.toString()); + + } + delete cra.conseiller; + delete cra.permanence; + delete cra.structure; + count++; + resolve(); + })); + }); + await Promise.all(promises); + await dbDatalake.collection('cras-insert').insertMany(cras); + logger.info(`Le lot ${lot.id + 1} / ${lotsArray.length} effectué (+ ${count} CRAS)`); + await sleep(1000); + } +});