Fix/Meta extraction - rules and error update

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openreview/client",
-  "version": "0.0.31",
+  "version": "0.0.32",
   "description": "Node.js client library for OpenReview's academic publishing API",
   "main": "src/index.js",
   "type": "module",
@@ -42,4 +42,4 @@
     "mocha": "^10.2.0"
   },
   "gitHead": "e83fe20886ca81f61c67c1884941b8df937c24c3"
-}
+}

packages/client/src/tools.js

@@ -894,7 +894,7 @@ export default class Tools {
     const contentType = result.headers.get('content-type');
     throw new OpenReviewError({
       name: 'ExtractAbstractError',
-      message: (contentType && contentType.indexOf('application/json') !== -1) ? await result.json() : await result.text(),
+      message: (contentType && contentType.indexOf('application/json') !== -1) ? JSON.stringify(await result.json()) : await result.text(),
       status: result.status || 500
     });
 

packages/meta-extraction/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openreview/meta-extraction",
-  "version": "0.0.6",
+  "version": "0.0.7",
   "description": "Extract abstracts for DBLP papers",
   "main": "src/index.js",
   "type": "module",

packages/meta-extraction/src/abstractExtractionRules.js

@@ -586,19 +586,23 @@ const neuripsCCRule = {
 
     const sections = await page.$$('h4');
 
-    let abstract = null;
+    let nextElementText = null;
+    let nextNextElementText = null;
+
     for (let index = 0; index < sections.length; index++) {
       const textContent = await page.evaluate((p) => p.textContent, sections[index]);
       if (textContent==='Abstract'){
-        const abstractContentElement = await sections[index].evaluateHandle(el => el.nextElementSibling.nextElementSibling);
-        abstract = await page.evaluate((p) => p.textContent, abstractContentElement);
-
+        const nextElement = await sections[index].evaluateHandle(el => el.nextElementSibling);
+        const nextNextElement = await sections[index].evaluateHandle(el => el.nextElementSibling.nextElementSibling);
+        nextElementText = await page.evaluate((p) => p?.textContent, nextElement);
+        nextNextElementText = await page.evaluate((p) => p?.textContent, nextNextElement);
       }
     }
 
     const allEvidence = [
       ...highwirePressTags,
-      { type: 'abstract', value: abstract }
+      { type: 'abstract', value: nextElementText },
+      { type: 'abstract', value: nextNextElementText }
     ];
 
     const abstractEvidences = allEvidence.filter(
@@ -976,6 +980,7 @@ const runAllRules = async (html, page, url) => {
     const { abstract, pdf, error } = await rule.executeRule(html, page);
 
     if (error) {
+      if (error==='openreview rule') return {};
       return {error};
     }
 

packages/meta-extraction/src/helpers.js

@@ -4,7 +4,7 @@ const shouldEnableMultiRedirect = (url) => [/doi.org/, /linkinghub.elsevier.com/
 
 const getTimeout = (url) => {
   const defaultTimeout = 15_000;
-  if ([/doi.org/, /spiedigitallibrary.org/, /iospress.com/].some((regex) => regex.test(url))) return defaultTimeout*2;
+  if ([/doi.org/, /spiedigitallibrary.org/, /iospress.com/].some((regex) => regex.test(url))) return defaultTimeout*3;
   return defaultTimeout;
 };