fix(ingester): set pmap instead of promise.all

SocialGouv · Dec 8, 2023 · 248870c · 248870c
1 parent 96aee3e
commit 248870c
Show file tree

Hide file tree

Showing 3 changed files with 32 additions and 30 deletions.
diff --git a/targets/ingester/src/articles/update-kali-articles.ts b/targets/ingester/src/articles/update-kali-articles.ts
@@ -11,26 +11,24 @@ export const updateKaliArticles = async (): Promise<void> => {
   const { agreements } = await loadAgreements();
 
   let i = 0;
-  await Promise.all(
-    await pMap(
-      agreements,
-      async ({ idcc, kali_id }) => {
-        console.log(
-          `updateKaliData: ${++i}/${
-            agreements.length
-          } Updating agreement for IDCC ${idcc}`
-        );
-        const articles = await loadAgreementArticles(
-          kali_id,
-          idcc,
-          loadAgreement,
-          loadArticles
-        );
-        await updateAgreementArticles(idcc, articles);
-      },
-      {
-        concurrency: 10,
-      }
-    )
+  await pMap(
+    agreements,
+    async ({ idcc, kali_id }) => {
+      console.log(
+        `updateKaliData: ${++i}/${
+          agreements.length
+        } Updating agreement for IDCC ${idcc}`
+      );
+      const articles = await loadAgreementArticles(
+        kali_id,
+        idcc,
+        loadAgreement,
+        loadArticles
+      );
+      await updateAgreementArticles(idcc, articles);
+    },
+    {
+      concurrency: 10,
+    }
   );
 };
diff --git a/targets/ingester/src/cli.ts b/targets/ingester/src/cli.ts
@@ -162,12 +162,15 @@ async function main() {
   }
 
   if (packagesToUpdate.get("@socialgouv/legi-data")) {
+    console.log("update legi articles");
     await updateLegiArticles();
   }
   if (packagesToUpdate.get("@socialgouv/kali-data")) {
+    console.log("update kali articles");
     await updateKaliArticles();
   }
   if (packagesToUpdate.get("@socialgouv/contributions-data")) {
+    console.log("set contributions availability to true");
     await setContributionsAvailabilityToTrue();
   }
   return ids;

diff --git a/targets/ingester/src/transform/contributions/setContributionsAvailabilityToTrue.ts b/targets/ingester/src/transform/contributions/setContributionsAvailabilityToTrue.ts
@@ -1,5 +1,6 @@
 import { fetchDocumentContributions } from "./fetchContributions";
 import { updateDocumentAvailabilityToTrue } from "./updateDocument";
+import pMap from "p-map";
 
 export async function setContributionsAvailabilityToTrue() {
   // Nous allons récupérer les contributions de la table document
@@ -13,14 +14,14 @@ export async function setContributionsAvailabilityToTrue() {
     (v) => v.slug
   );
 
-  const promises = [];
-
   // Pour chacune des nouvelles contributions nous allons les passer en is_available à true, car l'ingester les passe à false.
-  for (let i = 0; i < allNewContributionsBySlug.length; i++) {
-    promises.push(
-      updateDocumentAvailabilityToTrue(allNewContributionsBySlug[i])
-    );
-  }
-
-  await Promise.all(promises);
+  await pMap(
+    allNewContributionsBySlug,
+    (contrib) => {
+      return updateDocumentAvailabilityToTrue(contrib);
+    },
+    {
+      concurrency: 10,
+    }
+  );
 }