feat: 1406 contrib affichage des status sur ladmin

shared/types/src/elastic/common.ts

@@ -1,4 +1,5 @@
 import { SourceRoute } from "@socialgouv/cdtn-sources";
+import { ContributionsAnswers } from "../hasura";
 
 export type DocumentElasticWithSource<
   T,
@@ -11,6 +12,11 @@ export type Breadcrumb = {
   slug: string;
 };
 
+export type Export = {
+  id: string;
+  createdAt: string;
+};
+
 export type DocumentElastic<T extends SourceRoute = SourceRoute> = {
   id: string;
   cdtnId: string;
@@ -23,6 +29,8 @@ export type DocumentElastic<T extends SourceRoute = SourceRoute> = {
   excludeFromSearch: boolean;
   metaDescription: string;
   refs: DocumentRef[];
+  contribution?: ContributionsAnswers;
+  export?: Export;
 };
 
 export type RelatedDocument<T extends SourceRoute = SourceRoute> = {

shared/types/src/hasura/contributions.ts

@@ -1,6 +1,11 @@
 import { HasuraDocument } from "./common";
 import { FicheServicePublicDoc } from "./fiche-sp";
 
+export type ContributionStatus = {
+  createdAt: string;
+  status: string;
+};
+
 export type ContributionsAnswers = {
   id: string;
   content: string | null;
@@ -14,6 +19,8 @@ export type ContributionsAnswers = {
   content_fiche_sp: ContributionContentFicheSp | null;
   message_block_generic_no_CDT: string | null;
   agreement: ContributionAgreement;
+  updatedAt: string;
+  statuses?: ContributionStatus[];
 };
 
 export type ContributionQuestion = {

targets/export-elasticsearch/src/ingester/cdtnDocuments.ts

@@ -20,7 +20,10 @@ import {
 import { splitArticle } from "./fichesTravailSplitter";
 import { createGlossaryTransform } from "./glossary";
 import { getVersions } from "./versions";
-import { generateContributions } from "./contributions";
+import {
+  fetchContributionDocumentToPublish,
+  generateContributions,
+} from "./contributions";
 import { generateAgreements } from "./agreements";
 import { getGlossary } from "./common/fetchGlossary";
 import { fetchThemes } from "./themes/fetchThemes";
@@ -29,6 +32,7 @@ import { generatePrequalified } from "./prequalified";
 import { generateEditorialContents } from "./informations/generate";
 import { populateRelatedDocuments } from "./common/populateRelatedDocuments";
 import { mergeRelatedDocumentsToEditorialContents } from "./informations/mergeRelatedDocumentsToEditorialContents";
+import { updateExportStatuses } from "./documents/updateExportStatuses";
 
 /**
  * Find duplicate slugs
@@ -55,7 +59,8 @@ export async function getDuplicateSlugs(allDocuments: any) {
 }
 
 export async function cdtnDocumentsGen(
-  updateDocs: (source: string, documents: unknown[]) => Promise<void>
+  updateDocs: (source: string, documents: unknown[]) => Promise<void>,
+  isProd: boolean
 ) {
   let documentsCount: Partial<ExportEsStatus["documentsCount"]> = {};
 
@@ -66,6 +71,8 @@ export async function cdtnDocumentsGen(
   const glossaryTerms = await getGlossary();
   const addGlossary = createGlossaryTransform(glossaryTerms);
 
+  const contributionsToPublish = await fetchContributionDocumentToPublish();
+
   logger.info("=== Courriers ===");
   const modelesDeCourriers = await getDocumentBySource(
     SOURCES.LETTERS,
@@ -189,7 +196,9 @@ export async function cdtnDocumentsGen(
       };
     }),
   }));
-  logger.info(`Mapped ${fichesMTWithGlossary.length} fiches travail with glossary`);
+  logger.info(
+    `Mapped ${fichesMTWithGlossary.length} fiches travail with glossary`
+  );
   documentsCount = {
     ...documentsCount,
     [SOURCES.SHEET_MT_PAGE]: fichesMTWithGlossary.length,
@@ -331,6 +340,10 @@ export async function cdtnDocumentsGen(
     },
   ]);
 
+  if (isProd && contributionsToPublish) {
+    await updateExportStatuses(contributionsToPublish);
+  }
+
   logger.info("=== Save the documents length ===");
   documentsCount = {
     ...documentsCount,

targets/export-elasticsearch/src/ingester/contributions/__tests__/filterContributionToPublish.test.ts

@@ -0,0 +1,50 @@
+import { filterContributionDocumentsToPublish } from "../fetchContributionDocumentsToPublish";
+
+describe("filterContributionDocumentsToPublish", () => {
+  it("should send the list of contributions which have been updated after last export", () => {
+    const result = filterContributionDocumentsToPublish([
+      {
+        cdtn_id: "388f8d7860",
+        //@ts-ignore
+        export: {
+          createdAt: "2024-06-03T08:10:03.697249+00:00",
+        },
+        //@ts-ignore
+        contribution: {
+          id: "d61a3e21-9bf1-4199-b9d7-944c2382750c",
+          updatedAt: "2024-03-25T17:25:33.002376+00:00",
+          statuses: [
+            {
+              status: "TO_PUBLISH",
+              createdAt: "2024-03-27T14:26:28.387898+00:00",
+            },
+          ],
+        },
+      },
+    ]);
+    expect(result?.length).toBe(0);
+  });
+  it("should send the list of contributions which have been updated after last export2", () => {
+    const result = filterContributionDocumentsToPublish([
+      {
+        cdtn_id: "388f8d7860",
+        //@ts-ignore
+        export: {
+          createdAt: "2024-06-03T08:10:03.697249+00:00",
+        },
+        //@ts-ignore
+        contribution: {
+          id: "d61a3e21-9bf1-4199-b9d7-944c2382750c",
+          updatedAt: "2024-03-25T17:25:33.002376+00:00",
+          statuses: [
+            {
+              status: "TO_PUBLISH",
+              createdAt: "2024-06-27T14:26:28.387898+00:00",
+            },
+          ],
+        },
+      },
+    ]);
+    expect(result?.length).toBe(1);
+  });
+});

targets/export-elasticsearch/src/ingester/contributions/fetchContributionDocumentsToPublish.ts

@@ -0,0 +1,64 @@
+import {
+  ContributionDocumentJson,
+  DocumentElasticWithSource,
+} from "@socialgouv/cdtn-types";
+import { context } from "../context";
+import { gqlClient } from "@shared/utils";
+
+export const fetchContributionDocumentsQuery = `
+query fetchContributions {
+    documents(where: {source: {_eq: "contributions"}}) {
+      cdtnId: cdtn_id
+      export {
+        createdAt: created_at
+      }
+      contribution {
+        id
+        updatedAt: updated_at
+        statuses(where:{status: {_eq: "TO_PUBLISH"}}, order_by: {created_at: desc}, limit: 1) {
+          status
+          createdAt: created_at
+        }
+      }
+    }
+  }
+  `;
+
+interface HasuraReturn {
+  documents: [DocumentElasticWithSource<ContributionDocumentJson>] | undefined;
+}
+
+export function filterContributionDocumentsToPublish(
+  contributionDocs:
+    | DocumentElasticWithSource<ContributionDocumentJson>[]
+    | undefined
+): DocumentElasticWithSource<ContributionDocumentJson>[] | undefined {
+  return contributionDocs?.filter((doc) => {
+    const exportDate = doc.export?.createdAt
+      ? new Date(doc.export.createdAt).getTime()
+      : 0;
+    const statusDate = doc.contribution?.statuses?.length
+      ? new Date(doc.contribution.statuses[0].createdAt).getTime()
+      : 0;
+    return statusDate > exportDate;
+  });
+}
+
+export async function fetchContributionDocumentToPublish(): Promise<
+  DocumentElasticWithSource<ContributionDocumentJson>[] | undefined
+> {
+  const HASURA_GRAPHQL_ENDPOINT =
+    context.get("cdtnAdminEndpoint") || "http://localhost:8080/v1/graphql";
+  const HASURA_GRAPHQL_ENDPOINT_SECRET =
+    context.get("cdtnAdminEndpointSecret") || "admin1";
+  const res = await gqlClient({
+    graphqlEndpoint: HASURA_GRAPHQL_ENDPOINT,
+    adminSecret: HASURA_GRAPHQL_ENDPOINT_SECRET,
+  })
+    .query<HasuraReturn>(fetchContributionDocumentsQuery, {})
+    .toPromise();
+  if (res.error) {
+    throw res.error;
+  }
+  return filterContributionDocumentsToPublish(res.data?.documents);
+}

targets/export-elasticsearch/src/ingester/contributions/index.ts

@@ -1,2 +1,3 @@
 export * from "./helpers";
 export * from "./generate";
+export * from "./fetchContributionDocumentsToPublish";

targets/export-elasticsearch/src/ingester/documents/fetchLastExportStatus.ts

@@ -0,0 +1,33 @@
+import { Export } from "@socialgouv/cdtn-types";
+import { context } from "../context";
+import { gqlClient } from "@shared/utils";
+
+export const fetchLastExportStatusQuery = `
+    query lastExportStatus {
+        exportEsStatus: export_es_status(order_by: {created_at: desc}, limit: 1) {
+            id
+        }
+    }
+  `;
+
+interface HasuraReturn {
+    exportEsStatus: [Export] | undefined
+}
+
+
+export async function fetchLastExportStatus(): Promise<Export | undefined> {
+    const HASURA_GRAPHQL_ENDPOINT =
+        context.get("cdtnAdminEndpoint") || "http://localhost:8080/v1/graphql";
+    const HASURA_GRAPHQL_ENDPOINT_SECRET =
+        context.get("cdtnAdminEndpointSecret") || "admin1";
+    const res = await gqlClient({
+        graphqlEndpoint: HASURA_GRAPHQL_ENDPOINT,
+        adminSecret: HASURA_GRAPHQL_ENDPOINT_SECRET,
+    })
+        .query<HasuraReturn>(fetchLastExportStatusQuery, {})
+        .toPromise();
+    if (res.error) {
+        throw res.error;
+    }
+    return res.data?.exportEsStatus?.[0];
+}

targets/export-elasticsearch/src/ingester/documents/updateExportStatuses.ts

@@ -0,0 +1,50 @@
+import { fetchLastExportStatus } from "./fetchLastExportStatus";
+import { gqlClient } from "@shared/utils";
+import { context } from "../context";
+import {
+  ContributionDocumentJson,
+  DocumentElasticWithSource,
+} from "@socialgouv/cdtn-types";
+
+export const updateToLastExportStatusMutation = `mutation updateToLastExportStatus($cdtnIds: [String!], $exportId: uuid) {
+  updateDocuments: update_documents(
+    where: {cdtn_id: {_in: $cdtnIds}},
+    _set: {
+      exportId: $exportId
+    }
+  ) {
+    returning {
+      cdtn_id
+    }
+  }
+}`;
+
+export async function updateExportStatuses(
+  contributionsToPublish: DocumentElasticWithSource<ContributionDocumentJson>[]
+) {
+  const HASURA_GRAPHQL_ENDPOINT =
+    context.get("cdtnAdminEndpoint") || "http://localhost:8080/v1/graphql";
+  const HASURA_GRAPHQL_ENDPOINT_SECRET =
+    context.get("cdtnAdminEndpointSecret") || "admin1";
+  const exportStatus = await fetchLastExportStatus();
+
+  if (!exportStatus?.id) {
+    return;
+  }
+  const cdtnIds = contributionsToPublish.map(
+    (contribution) => contribution.cdtnId
+  );
+  const { id: exportId } = exportStatus;
+
+  if (cdtnIds.length) {
+    const res = await gqlClient({
+      graphqlEndpoint: HASURA_GRAPHQL_ENDPOINT,
+      adminSecret: HASURA_GRAPHQL_ENDPOINT_SECRET,
+    })
+      .mutation(updateToLastExportStatusMutation, { cdtnIds, exportId })
+      .toPromise();
+    if (res.error) {
+      throw res.error;
+    }
+  }
+}

targets/export-elasticsearch/src/ingester/ingest.ts

@@ -16,6 +16,7 @@ import pMap from "p-map";
 import { cdtnDocumentsGen } from "./cdtnDocuments";
 import { context } from "./context";
 import { populateSuggestions } from "./suggestion";
+import { Environment } from "@socialgouv/cdtn-types";
 
 async function addVector(data: any) {
   const NLP_URL = context.get("nlpUrl");
@@ -61,7 +62,8 @@ export async function ingest(
   suggestIndexName: string | undefined,
   bufferSize: number | undefined,
   suggestFile: string | undefined,
-  disableGlossary: boolean | undefined
+  disableGlossary: boolean | undefined,
+  isProd = false
 ) {
   context.provide();
   process.env.NLP_URL = nlpUrl; //pour setter la variable d'environment du package elasticsearch...
@@ -75,7 +77,8 @@ export async function ingest(
     suggestIndexName,
     bufferSize,
     suggestFile,
-    disableGlossary
+    disableGlossary,
+    isProd
   );
 }
 
@@ -89,7 +92,8 @@ async function runIngester(
   suggestIndexName: string | undefined,
   bufferSize: number | undefined,
   suggestFile: string | undefined,
-  disableGlossary: boolean | undefined
+  disableGlossary: boolean | undefined,
+  isProd: boolean
 ) {
   const ES_INDEX_PREFIX = esIndexPrefix ?? "cdtn";
 
@@ -159,7 +163,7 @@ async function runIngester(
       size: 800,
     });
   };
-  await cdtnDocumentsGen(updateDocs);
+  await cdtnDocumentsGen(updateDocs, isProd);
 
   logger.info(`done in ${(Date.now() - t0) / 1000} s`);
 

targets/export-elasticsearch/src/workers/ingester-prod.ts

@@ -16,7 +16,8 @@ const ingester = async (): Promise<string> => {
         undefined,
         undefined,
         undefined,
-        false
+        false,
+        true
       );
       resolve("Export elasticsearch completed successfully");
     } catch (error: unknown) {