✨ (parsers): Use Unpaywall.org as a preprint matching source

vict0rsch · Dec 4, 2023 · 2995a74 · 2995a74
1 parent 58ab638
commit 2995a74
Show file tree

Hide file tree

Showing 4 changed files with 49 additions and 5 deletions.
diff --git a/src/bibMatcher/bibMatcher.js b/src/bibMatcher/bibMatcher.js
@@ -196,6 +196,22 @@ const matchPaper = async (paper) => {
         }
         match?.venue && console.log("crossRefMatch: ", match);
     }
+    if (!bibtex) {
+        setHTML("matching-status-provider", "unpaywall.org ...");
+        match = await tryUnpaywall(paper);
+        venue = match.venue;
+        if (venue) {
+            paper.journal = venue;
+            for (const [key, value] of paper.entries()) {
+                if ((value + "").toLowerCase().includes("arxiv")) {
+                    delete paper[key];
+                }
+            }
+            bibtex = bibtexToString(paper);
+            source = "Unpaywall";
+        }
+        match?.venue && console.log("unpaywallMatch: ", match);
+    }
     return { bibtex, match, source, venue };
 };
 

diff --git a/src/options/options.js b/src/options/options.js
@@ -543,6 +543,14 @@ const startMatching = async (papersToMatch) => {
             note = !paper.note && match?.note;
         }
 
+        if (!venue) {
+            setHTML("matching-status-provider", "unpaywall.org ...");
+            match = await tryUnpaywall(paper);
+            console.log("unpaywallMatch: ", match);
+            venue = match?.venue;
+            note = !paper.note && match?.note;
+        }
+
         if (!venue) {
             setHTML("matching-status-provider", "scholar.google.com ...");
             match = await tryCrossRef(paper);

diff --git a/src/shared/js/utils/paper.js b/src/shared/js/utils/paper.js
@@ -675,7 +675,7 @@ const addOrUpdatePaper = async ({
         // anyway: try and update note with actual publication
         if (!paper.note || !paper.venue || paper.author.toLowerCase() === "anonymous") {
             const preprintMatch = await tryPreprintMatch(paper);
-            for (const key of ["note", "venue", "bibtex"]) {
+            for (const key of ["note", "venue", "bibtex", "doi"]) {
                 if (!paper[key] || key === "bibtex") {
                     const value = preprintMatch[key] ?? pwc[key];
                     if (value) {

diff --git a/src/shared/js/utils/parsers.js b/src/shared/js/utils/parsers.js
@@ -1676,16 +1676,35 @@ const tryGoogleScholar = async (paper) => {
     return resp;
 };
 
+const tryUnpaywall = async (paper) => {
+    const url = `https://api.unpaywall.org/v2/search?query=${encodeURI(
+        paper.title
+    )}&is_oa=true&email=papermemory+${parseInt(Math.random() * 1000)}@gmail.com`;
+    const { data, status } = await fetchJSON(url);
+    if (data && status === 200) {
+        const match = data.results?.find(
+            (m) => miniHash(m.response.title) === miniHash(paper.title)
+        );
+        if (match) {
+            const venue = match.journal_name;
+            const note = `Accepted @ ${venue} (${match.year}) -- [unpaywall.org]`;
+            const doi = match.doi;
+            return { venue, note, doi };
+        }
+    }
+};
+
 const tryPreprintMatch = async (paper, tryPwc = false) => {
-    let note, venue, bibtex, code;
+    let note, venue, bibtex, code, doi;
     let matches = {};
 
-    let names = ["DBLP", "SemanticScholar", "CrossRef", "GoogleScholar"];
+    let names = ["DBLP", "SemanticScholar", "CrossRef", "GoogleScholar", "Unpaywall"];
     let matchPromises = [
         silentPromiseTimeout(tryGoogleScholar(paper)),
         silentPromiseTimeout(trySemanticScholar(paper)),
         silentPromiseTimeout(tryCrossRef(paper)),
         silentPromiseTimeout(tryDBLP(paper)),
+        silentPromiseTimeout(tryUnpaywall(paper)),
     ];
 
     if (tryPwc) {
@@ -1695,7 +1714,7 @@ const tryPreprintMatch = async (paper, tryPwc = false) => {
 
     for (const [n, name] of Object.entries(names)) {
         matches[name] = await matchPromises[n];
-        ({ note, venue, bibtex } = matches[name] ?? {});
+        ({ note, venue, bibtex, doi } = matches[name] ?? {});
         if (note) {
             break;
         } else {
@@ -1713,7 +1732,7 @@ const tryPreprintMatch = async (paper, tryPwc = false) => {
         }
     }
 
-    return { note, venue, bibtex, code };
+    return { note, venue, bibtex, code, doi };
 };
 
 // -----------------------------
@@ -1998,5 +2017,6 @@ if (typeof module !== "undefined" && module.exports != null) {
         tryPreprintMatch,
         tryPWCMatch,
         trySemanticScholar,
+        tryUnpaywall,
     };
 }