Merge pull request #1305 from AletheiaFact/verification-request-rag-r…

…ecommendation

WIP: Verification Request Recommendations

migrations/20240729172331-add-embedding-field-in-verification-request-schema.ts

@@ -0,0 +1,30 @@
+import { Db } from "mongodb";
+import { OpenAIEmbeddings } from "@langchain/openai";
+
+export async function up(db: Db) {
+    const embeddings = new OpenAIEmbeddings();
+    const verificationRequestCursor = await db
+        .collection("verificationrequests")
+        .find();
+
+    while (await verificationRequestCursor.hasNext()) {
+        const verificationRequest = await verificationRequestCursor.next();
+
+        const embedding = await embeddings.embedQuery(
+            verificationRequest.content
+        );
+
+        await db
+            .collection("verificationrequests")
+            .updateOne(
+                { _id: verificationRequest._id },
+                { $set: { embedding } }
+            );
+    }
+}
+
+export async function down(db: Db) {
+    await db
+        .collection("verificationrequests")
+        .updateMany({}, { $unset: { embedding: "" } });
+}

newrelic_agent.log

@@ -38,3 +38,7 @@
 {"v":0,"level":30,"name":"newrelic","hostname":"Pepe-Zenbook-UX3404VA-Q420VA","pid":1152344,"time":"2024-05-30T22:19:11.828Z","msg":"Using New Relic for Node.js. Agent version: 11.5.0; Node version: v18.14.0."}
 {"v":0,"level":30,"name":"newrelic","hostname":"Pepe-Zenbook-UX3404VA-Q420VA","pid":1152344,"time":"2024-05-30T22:19:11.890Z","msg":"Using AsyncLocalContextManager"}
 {"v":0,"level":50,"name":"newrelic","hostname":"Pepe-Zenbook-UX3404VA-Q420VA","pid":1152344,"time":"2024-05-30T22:19:11.891Z","msg":"New Relic for Node.js was unable to bootstrap itself due to an error:","stack":"Error: New Relic requires that you name this application!\nSet app_name in your newrelic.js or newrelic.cjs file or set environment variable\nNEW_RELIC_APP_NAME. Not starting!\n    at createAgent (/home/pepe/aletheia/node_modules/newrelic/index.js:160:11)\n    at initialize (/home/pepe/aletheia/node_modules/newrelic/index.js:105:15)\n    at Object.<anonymous> (/home/pepe/aletheia/node_modules/newrelic/index.js:39:3)\n    at Module._compile (node:internal/modules/cjs/loader:1226:14)\n    at Module._extensions..js (node:internal/modules/cjs/loader:1280:10)\n    at Module.load (node:internal/modules/cjs/loader:1089:32)\n    at Module._load (node:internal/modules/cjs/loader:930:12)\n    at Module.require (node:internal/modules/cjs/loader:1113:19)\n    at require (node:internal/modules/cjs/helpers:103:18)\n    at Object.<anonymous> (/home/pepe/aletheia/dist/server/main.js:16:5)\n    at Module._compile (node:internal/modules/cjs/loader:1226:14)\n    at Module._extensions..js (node:internal/modules/cjs/loader:1280:10)\n    at Module.load (node:internal/modules/cjs/loader:1089:32)\n    at Module._load (node:internal/modules/cjs/loader:930:12)\n    at Function.executeUserEntryPoint [as runMain] (node:internal/modules/run_main:81:12)\n    at node:internal/main/run_main_module:23:47","message":"New Relic requires that you name this application!\nSet app_name in your newrelic.js or newrelic.cjs file or set environment variable\nNEW_RELIC_APP_NAME. Not starting!"}
+{"v":0,"level":30,"name":"newrelic","hostname":"Pepe-Zenbook-UX3404VA-Q420VA","pid":212209,"time":"2024-07-31T09:50:42.327Z","msg":"Unable to find configuration file. If a configuration file is desired (common for non-containerized environments), a base configuration file can be copied from /home/pepe/aletheia/node_modules/newrelic/newrelic.js and renamed to \"newrelic.js\" in the directory from which you will start your application. Attempting to start agent using environment variables."}
+{"v":0,"level":30,"name":"newrelic","hostname":"Pepe-Zenbook-UX3404VA-Q420VA","pid":212209,"time":"2024-07-31T09:50:42.330Z","msg":"Using New Relic for Node.js. Agent version: 11.5.0; Node version: v18.14.0."}
+{"v":0,"level":30,"name":"newrelic","hostname":"Pepe-Zenbook-UX3404VA-Q420VA","pid":212209,"time":"2024-07-31T09:50:42.380Z","msg":"Using AsyncLocalContextManager"}
+{"v":0,"level":50,"name":"newrelic","hostname":"Pepe-Zenbook-UX3404VA-Q420VA","pid":212209,"time":"2024-07-31T09:50:42.381Z","msg":"New Relic for Node.js was unable to bootstrap itself due to an error:","stack":"Error: New Relic requires that you name this application!\nSet app_name in your newrelic.js or newrelic.cjs file or set environment variable\nNEW_RELIC_APP_NAME. Not starting!\n    at createAgent (/home/pepe/aletheia/node_modules/newrelic/index.js:160:11)\n    at initialize (/home/pepe/aletheia/node_modules/newrelic/index.js:105:15)\n    at Object.<anonymous> (/home/pepe/aletheia/node_modules/newrelic/index.js:39:3)\n    at Module._compile (node:internal/modules/cjs/loader:1226:14)\n    at Module._extensions..js (node:internal/modules/cjs/loader:1280:10)\n    at Module.load (node:internal/modules/cjs/loader:1089:32)\n    at Module._load (node:internal/modules/cjs/loader:930:12)\n    at Module.require (node:internal/modules/cjs/loader:1113:19)\n    at require (node:internal/modules/cjs/helpers:103:18)\n    at Object.<anonymous> (/home/pepe/aletheia/dist/server/main.js:16:5)\n    at Module._compile (node:internal/modules/cjs/loader:1226:14)\n    at Module._extensions..js (node:internal/modules/cjs/loader:1280:10)\n    at Module.load (node:internal/modules/cjs/loader:1089:32)\n    at Module._load (node:internal/modules/cjs/loader:930:12)\n    at Function.executeUserEntryPoint [as runMain] (node:internal/modules/run_main:81:12)\n    at node:internal/main/run_main_module:23:47","message":"New Relic requires that you name this application!\nSet app_name in your newrelic.js or newrelic.cjs file or set environment variable\nNEW_RELIC_APP_NAME. Not starting!"}

public/locales/en/verificationRequest.json

@@ -2,9 +2,17 @@
     "verificationRequestListHeader": "Verification Requests",
     "verificationRequestTitle": "Verification Request",
     "manageVerificationRequests": "Manage selected verification requests",
+    "noVerificationRequestsMessage": "No verification requests",
+    "manageVerificationRequestsGroup": "Manage verification request's group",
     "createClaimFromVerificationRequest": "No claim was created with this verification request",
     "openVerificationRequestClaimLabel": "A claim created was related to this verification request",
     "openVerificationRequestClaimButton": "Open claim",
     "agroupVerificationRequest": "Related verification requests",
-    "openVerificationRequest": "Open"
+    "openVerificationRequest": "Open",
+    "recommendationTitle": "Recommedations",
+    "addInGroupButton": "Add in group",
+    "alreadyInGroupMessage": "Already in group",
+    "searchResultsTitle": "Results",
+    "noResultsMessage": "No results",
+    "searchPlaceholder": "Search verification requests"
 }

public/locales/pt/verificationRequest.json

@@ -2,9 +2,17 @@
     "verificationRequestListHeader": "Denúncias",
     "verificationRequestTitle": "Denúncia",
     "manageVerificationRequests": "Gerenciar denúncias selecionadas",
+    "noVerificationRequestsMessage": "Nenhuma denúncia",
+    "manageVerificationRequestsGroup": "Gerenciar grupo de denúncias",
     "createClaimFromVerificationRequest": "Nenhuma afirmação foi criada com esta denúncia",
     "openVerificationRequestClaimLabel": "Uma afirmação criada foi relacionada com essa denúncia",
     "openVerificationRequestClaimButton": "Abrir afirmação",
     "agroupVerificationRequest": "Denúncias relacionadas",
-    "openVerificationRequest": "Abrir"
+    "openVerificationRequest": "Abrir",
+    "recommendationTitle": "Recomendações",
+    "addInGroupButton": "Adicionar no grupo",
+    "alreadyInGroupMessage": "Já adicionado ao grupo",
+    "searchResultsTitle": "Resultados",
+    "noResultsMessage": "Nenhum resultado",
+    "searchPlaceholder": "Busque denúncias"
 }

server/verification-request/dto/create-verification-request-dto.ts

@@ -20,4 +20,9 @@ export class CreateVerificationRequestDTO {
     @IsOptional()
     @ApiProperty()
     data_hash: string;
+
+    @IsOptional()
+    @IsArray()
+    @ApiProperty()
+    embedding?: number[];
 }

server/verification-request/schemas/verification-request.schema.ts

@@ -28,6 +28,9 @@ export class VerificationRequest {
 
     @Prop({ required: false, type: Boolean })
     isSensitive: boolean;
+
+    @Prop({ required: true, type: [Number] })
+    embedding: number[];
 }
 
 export const VerificationRequestSchema =

server/verification-request/verification-request.controller.ts

@@ -136,12 +136,24 @@ export class VerificationRequestController {
                 dataHash
             );
 
+        const recommendationFilter = verificationRequest.group?.content?.map(
+            (v: any) => v._id
+        ) || [verificationRequest?._id];
+
+        const recommendations =
+            await this.verificationRequestService.findSimilarRequests(
+                verificationRequest.content,
+                recommendationFilter,
+                5
+            );
+
         await this.viewService.getNextServer().render(
             req,
             res,
             "/verification-request-review-page",
             Object.assign(parsedUrl.query, {
                 reviewTask,
+                recommendations,
                 sitekey: this.configService.get<string>("recaptcha_sitekey"),
                 hideDescriptions: {},
                 websocketUrl: this.configService.get<string>("websocketUrl"),

server/verification-request/verification-request.service.ts

@@ -9,6 +9,7 @@ import { InjectModel } from "@nestjs/mongoose";
 import { GroupService } from "../group/group.service";
 import { CreateVerificationRequestDTO } from "./dto/create-verification-request-dto";
 import { UpdateVerificationRequestDTO } from "./dto/update-verification-request.dto";
+import { OpenAIEmbeddings } from "@langchain/openai";
 const md5 = require("md5");
 
 @Injectable()
@@ -20,8 +21,18 @@ export class VerificationRequestService {
         private groupService: GroupService
     ) {}
 
-    async listAll({ page, pageSize, order }): Promise<VerificationRequest[]> {
-        return this.VerificationRequestModel.find({})
+    async listAll({
+        content,
+        page,
+        pageSize,
+        order,
+    }): Promise<VerificationRequest[]> {
+        const query: any = {};
+        if (content) {
+            query.content = { $regex: content, $options: "i" };
+        }
+
+        return this.VerificationRequestModel.find(query, { embedding: 0 })
             .skip(page * parseInt(pageSize, 10))
             .limit(parseInt(pageSize, 10))
             .sort({ _id: order })
@@ -36,12 +47,15 @@ export class VerificationRequestService {
     async findAll(verifiedRequestQuery: {
         searchContent: string;
     }): Promise<VerificationRequest[]> {
-        return this.VerificationRequestModel.find({
-            content: {
-                $regex: verifiedRequestQuery.searchContent || "",
-                $options: "i",
+        return this.VerificationRequestModel.find(
+            {
+                content: {
+                    $regex: verifiedRequestQuery.searchContent || "",
+                    $options: "i",
+                },
             },
-        });
+            { embedding: 0 }
+        );
     }
 
     /**
@@ -50,26 +64,30 @@ export class VerificationRequestService {
      * @returns the verification request document
      */
     async getById(verificationRequestId: string): Promise<VerificationRequest> {
-        return this.VerificationRequestModel.findById(
-            verificationRequestId
-        ).populate("group");
+        return this.VerificationRequestModel.findById(verificationRequestId, {
+            embedding: 0,
+        }).populate("group");
     }
 
     /**
      * Creates a new verification request document
+     * Executes the createEmbed function to store the verification request content embedding
      * For each sources in verification request, creates a new source document
      * @param verificationRequest verificationRequestBody
      * @returns the verification request document
      */
-    create(
+    async create(
         verificationRequest: CreateVerificationRequestDTO
     ): Promise<VerificationRequestDocument> {
         try {
             verificationRequest.data_hash = md5(verificationRequest.content);
+            verificationRequest.embedding = await this.createEmbedContent(
+                verificationRequest.content
+            );
             const newVerificationRequest = new this.VerificationRequestModel(
                 verificationRequest
             );
-            if (verificationRequest.sources.length) {
+            if (verificationRequest?.sources?.length) {
                 for (const source of verificationRequest.sources) {
                     this.sourceService.create({
                         href: source,
@@ -93,9 +111,10 @@ export class VerificationRequestService {
     async findByDataHash(
         data_hash: string
     ): Promise<VerificationRequestDocument> {
-        return this.VerificationRequestModel.findOne({ data_hash }).populate(
-            "group"
-        );
+        return this.VerificationRequestModel.findOne(
+            { data_hash },
+            { embedding: 0 }
+        ).populate("group");
     }
 
     /**
@@ -282,4 +301,78 @@ export class VerificationRequestService {
     count(query) {
         return this.VerificationRequestModel.countDocuments().where(query);
     }
+
+    /**
+     * Creates an embedding based on a query parameter
+     * @param content verification request content
+     * @returns verification request content embedding
+     */
+    createEmbedContent(content: string): Promise<number[]> {
+        const embeddings = new OpenAIEmbeddings();
+
+        return embeddings.embedQuery(content);
+    }
+
+    /**
+     * Find similar verification requests related to the embedding content
+     * @param content verification request content
+     * @param filter verification requests IDs to filter, does not recommend verification requests those are part of the same group
+     * @param pageSize limit of documents
+     * @returns Verification requests with the similarity score greater than 0.80
+     */
+    async findSimilarRequests(
+        content,
+        filter,
+        pageSize
+    ): Promise<VerificationRequest[]> {
+        const queryEmbedding = await this.createEmbedContent(content);
+        const filterIds = filter.map((verificationRequestId) =>
+            Types.ObjectId(verificationRequestId)
+        );
+
+        return await this.VerificationRequestModel.aggregate([
+            { $match: { _id: { $nin: filterIds } } },
+            {
+                $addFields: {
+                    similarity: {
+                        $reduce: {
+                            input: {
+                                $zip: {
+                                    inputs: ["$embedding", queryEmbedding],
+                                },
+                            },
+                            initialValue: 0,
+                            in: {
+                                $add: [
+                                    "$$value",
+                                    {
+                                        $multiply: [
+                                            { $arrayElemAt: ["$$this", 0] },
+                                            { $arrayElemAt: ["$$this", 1] },
+                                        ],
+                                    },
+                                ],
+                            },
+                        },
+                    },
+                },
+            },
+            {
+                $match: {
+                    similarity: { $gte: 0.8, $type: "number" },
+                },
+            },
+            {
+                $project: {
+                    embedding: 0,
+                },
+            },
+            {
+                $sort: { similarity: -1 },
+            },
+            {
+                $limit: parseInt(pageSize),
+            },
+        ]);
+    }
 }

src/api/verificationRequestApi.ts

@@ -1,17 +1,24 @@
 import axios from "axios";
-import { NameSpaceEnum } from "../types/Namespace";
+import { ActionTypes } from "../store/types";
+
+interface SearchOptions {
+    searchText?: string;
+    page?: number;
+    pageSize?: number;
+    order?: string;
+}
 
 const request = axios.create({
     withCredentials: true,
     baseURL: `/api/verification-request`,
 });
 
-const get = (options) => {
+const get = (options: SearchOptions, dispatch = null) => {
     const params = {
+        content: options.searchText,
         page: options.page ? options.page - 1 : 0,
         order: options.order || "asc",
         pageSize: options.pageSize ? options.pageSize : 10,
-        nameSpace: options?.nameSpace || NameSpaceEnum.Main,
     };
 
     return request
@@ -23,11 +30,22 @@ const get = (options) => {
                 totalVerificationRequests,
             } = response.data;
 
-            return {
-                data: verificationRequests,
-                total: totalVerificationRequests,
+            if (!dispatch) {
+                return {
+                    data: verificationRequests,
+                    total: totalVerificationRequests,
+                    totalPages,
+                };
+            }
+
+            dispatch({
+                type: ActionTypes.SEARCH_RESULTS,
+                results: verificationRequests,
+            });
+            dispatch({
+                type: ActionTypes.SET_TOTAL_PAGES,
                 totalPages,
-            };
+            });
         })
         .catch((err) => {
             console.log(err);