Merge pull request #17 from AllanChain/feat/references

feat(cite): enable reference autocomplete

develop/dev.env

@@ -18,3 +18,4 @@ WEBPACK_HOST=webpack
 WEB_API_PASSWORD=overleaf
 WEB_API_USER=overleaf
 WEB_HOST=web
+REFERENCES_HOST=references

develop/docker-compose.dev.yml

@@ -79,6 +79,17 @@ services:
       - ../services/history-v1/knexfile.js:/overleaf/services/history-v1/knexfile.js
       - ../services/history-v1/migrations:/overleaf/services/history-v1/migrations
 
+  references:
+    command: ["node", "--watch", "app.js"]
+    environment:
+      - NODE_OPTIONS=--inspect=0.0.0.0:9229
+    ports:
+      - "127.0.0.1:9236:9229"
+    volumes:
+      - ../services/references/app:/overleaf/services/references/app
+      - ../services/references/config:/overleaf/services/references/config
+      - ../services/references/app.js:/overleaf/services/references/app.js
+
   notifications:
     command: ["node", "--watch", "app.js"]
     environment:

develop/docker-compose.yml

@@ -96,6 +96,13 @@ services:
     volumes:
       - mongo-data:/data/db
 
+  references:
+    build:
+      context: ..
+      dockerfile: services/references/Dockerfile
+    env_file:
+      - dev.env
+
   notifications:
     build:
       context: ..
@@ -161,6 +168,7 @@ services:
       - filestore
       - history-v1
       - notifications
+      - references
       - project-history
       - real-time
       - spelling

server-ce/runit/references-overleaf/run

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+NODE_PARAMS=""
+if [ "$DEBUG_NODE" == "true" ]; then
+    echo "running debug - references"
+    NODE_PARAMS="--inspect=0.0.0.0:30060"
+fi
+
+NODE_CONFIG_DIR=/overleaf/services/references/config exec /sbin/setuser www-data /usr/bin/node $NODE_PARAMS /overleaf/services/references/app.js >> /var/log/overleaf/references.log 2>&1

services/references/Dockerfile

@@ -0,0 +1,24 @@
+FROM node:18.20.2 AS base
+
+WORKDIR /overleaf/services/references
+
+# Google Cloud Storage needs a writable $HOME/.config for resumable uploads
+# (see https://googleapis.dev/nodejs/storage/latest/File.html#createWriteStream)
+RUN mkdir /home/node/.config && chown node:node /home/node/.config
+
+FROM base AS app
+
+COPY package.json package-lock.json /overleaf/
+COPY services/references/package.json /overleaf/services/references/
+COPY libraries/ /overleaf/libraries/
+COPY patches/ /overleaf/patches/
+
+RUN cd /overleaf && npm ci --quiet
+
+COPY services/references/ /overleaf/services/references/
+
+FROM app
+USER node
+
+CMD ["node", "--expose-gc", "app.js"]
+

services/references/app.js

@@ -0,0 +1,40 @@
+import '@overleaf/metrics/initialize.js'
+
+import express from 'express'
+import Settings from '@overleaf/settings'
+import logger from '@overleaf/logger'
+import metrics from '@overleaf/metrics'
+import ReferencesAPIController from './app/js/ReferencesAPIController.js'
+import bodyParser from 'body-parser'
+
+const app = express()
+metrics.injectMetricsRoute(app)
+
+app.use(bodyParser.json({ limit: '2mb' }))
+app.use(metrics.http.monitor(logger))
+
+app.post('/project/:project_id/index', ReferencesAPIController.index)
+app.get('/status', (req, res) => res.send({ status: 'references api is up' }))
+
+const settings =
+  Settings.internal && Settings.internal.references
+    ? Settings.internal.references
+    : undefined
+const host = settings && settings.host ? settings.host : 'localhost'
+const port = settings && settings.port ? settings.port : 3006
+
+logger.debug('Listening at', { host, port })
+
+const server = app.listen(port, host, function (error) {
+  if (error) {
+    throw error
+  }
+  logger.info({ host, port }, 'references HTTP server starting up')
+})
+
+process.on('SIGTERM', () => {
+  server.close(() => {
+    logger.info({ host, port }, 'references HTTP server closed')
+    metrics.close()
+  })
+})

services/references/app/js/BibParser.js

@@ -0,0 +1,209 @@
+"use strict";
+
+// Grammar implemented here:
+//  bibtex -> (string | entry)*;
+//  string -> '@STRING' kv_left key_equals_value kv_right;
+//  entry -> '@' key kv_left key ',' key_value_list kv_right;
+//  key_value_list -> key_equals_value (',' key_equals_value)* ','?;
+//  key_equals_value -> key '=' value;
+//  value -> value_quotes | value_braces | key;
+//  value_quotes -> '"' .*? '"'; // not quite
+//  value_braces -> '{' .*? '"'; // not quite
+//  kv_left -> '(' | '{'
+//  kv_right -> ')' | '}'
+function BibtexParser() {
+  this._entries = {};
+  this._comments = [];
+  this._strings = {};
+  this.input = '';
+  this.config = {
+    upperKeys: false
+  };
+  this._pos = 0;
+  var pairs = {
+    '{': '}',
+    '(': ')',
+    '"': '"'
+  };
+  var regs = {
+    atKey: /@([a-zA-Z0-9_:\\./-]+)\s*/,
+    enLeft: /^([\{\(])\s*/,
+    enRight: function enRight(left) {
+      return new RegExp("^(\\".concat(pairs[left], ")\\s*"));
+    },
+    entryId: /^\s*([^@={}",\s]+)\s*,\s*/,
+    key: /^([a-zA-Z0-9_:\\./-]+)\s*=\s*/,
+    vLeft: /^([\{"])\s*/,
+    vRight: function vRight(left) {
+      return new RegExp("^(\\".concat(pairs[left], ")\\s*"));
+    },
+    inVLeft: /^(\{)\s*/,
+    inVRight: function inVRight(left) {
+      return new RegExp("^(\\".concat(pairs[left], ")\\s*"));
+    },
+    value: /^[\{"]((?:[^\{\}]|\n)*?(?:(?:[^\{\}]|\n)*?\{(?:[^\{\}]|\n)*?\})*?(?:[^\{\}]|\n)*?)[\}"]\s*,?\s*/,
+    word: /^([^\{\}"\s]+)\s*/,
+    comma: /^(,)\s*/,
+    quota: /^(")\s*/
+  };
+
+  this.setInput = function (t) {
+    this.input = t;
+  };
+
+  this.matchFirst = function (reg) {
+    var notMove = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : false;
+    var result = this.input.slice(this._pos).match(reg);
+
+    if (result) {
+      if (!notMove) {
+        // console.log("!@#!@#", result[1]);
+        this._pos += result.index + result[0].length;
+      }
+
+      return {
+        success: true,
+        text: result[1],
+        index: result.index,
+        step: result[0].length
+      };
+    } else {
+      return {
+        success: false
+      };
+    }
+  };
+
+  this.assert = function (obj) {
+    for (var key in obj) {
+      if (obj[key] === undefined) {
+        throw "[BibParser:ERROR] ".concat(key, " not found at ").concat(this._pos);
+      }
+    }
+  };
+
+  this.getValue = function () {
+    var stack = [];
+    var values = [];
+
+    var _this$matchFirst = this.matchFirst(regs.vLeft),
+        vLeft = _this$matchFirst.text;
+
+    this.assert({
+      vLeft: vLeft
+    });
+    stack.push(vLeft);
+
+    while (stack.length > 0) {
+      if (this.matchFirst(regs.inVLeft, true).success) {
+        var _this$matchFirst2 = this.matchFirst(regs.inVLeft),
+            inVLeft = _this$matchFirst2.text;
+
+        stack.push(inVLeft);
+        values.push(inVLeft);
+      } else if (this.matchFirst(regs.inVRight(stack[stack.length - 1]), true).success) {
+        values.push(this.matchFirst(regs.inVRight(stack[stack.length - 1])).text);
+        stack.pop();
+      } else if (this.matchFirst(regs.word, true).success) {
+        values.push(this.matchFirst(regs.word).text);
+      } else if (this.matchFirst(regs.quota, true).success) {
+        values.push(this.matchFirst(regs.quota).text);
+      } else {
+        throw "[BibParser:ERROR] stack overflow at ".concat(this._pos);
+      }
+    }
+
+    values.pop();
+    this.matchFirst(regs.comma);
+    return values;
+  };
+
+  this.string = function () {
+    var _this$matchFirst3 = this.matchFirst(regs.key),
+        key = _this$matchFirst3.text;
+
+    this.assert({
+      key: key
+    });
+
+    var _this$matchFirst4 = this.matchFirst(regs.value),
+        value = _this$matchFirst4.text;
+
+    this.assert({
+      value: value
+    });
+    this._strings[key] = value;
+  };
+
+  this.preamble = function () {};
+
+  this.comment = function () {};
+
+  this.entry = function (head) {
+    var _this$matchFirst5 = this.matchFirst(regs.entryId),
+        entryId = _this$matchFirst5.text;
+
+    this.assert({
+      entryId: entryId
+    });
+    var entry = {};
+
+    while (this.matchFirst(regs.key, true).success) {
+      var _this$matchFirst6 = this.matchFirst(regs.key),
+          key = _this$matchFirst6.text;
+
+      var value = this.getValue();
+      entry[key] = value.join(' '); // if(key === 'author'){
+      //   const {text:value} = this.matchFirst(regs.value);
+      //   this.assert({value});
+      //   entry[key] = value;
+      // } else {
+      //   const {text:value} = this.matchFirst(regs.value);
+      //   this.assert({value});
+      //   entry[key] = value;
+      // }
+    }
+
+    entry.$type = head;
+    this._entries[entryId] = entry;
+  };
+
+  this.parse = function () {
+    while (this.matchFirst(regs.atKey, true).success) {
+      var _this$matchFirst7 = this.matchFirst(regs.atKey),
+          head = _this$matchFirst7.text;
+
+      var _this$matchFirst8 = this.matchFirst(regs.enLeft),
+          enLeft = _this$matchFirst8.text;
+
+      this.assert({
+        enLeft: enLeft
+      });
+
+      if (head.toUpperCase() == 'STRING') {
+        this.string();
+      } else if (head.toUpperCase() == 'PREAMBLE') {
+        this.preamble();
+      } else if (head.toUpperCase() == 'COMMENT') {
+        this.comment();
+      } else {
+        this.entry(head);
+      }
+
+      var _this$matchFirst9 = this.matchFirst(regs.enRight(enLeft)),
+          enRight = _this$matchFirst9.text;
+
+      this.assert({
+        enRight: enRight
+      });
+    }
+  };
+} //Runs the parser
+
+
+export function bibParse(input) {
+  var b = new BibtexParser();
+  b.setInput(input);
+  b.parse();
+  return b._entries;
+}

services/references/app/js/ReferencesAPIController.js

@@ -0,0 +1,36 @@
+import logger from '@overleaf/logger'
+import { bibParse } from './BibParser.js'
+
+// req: { allUrls: string[], fullIndex: boolean }
+// res: { keys: string[]}
+export default {
+  index(req, res) {
+    const { docUrls, fullIndex } = req.body;
+    Promise.all(docUrls.map(async (docUrl) => {
+      try {
+        const response = await fetch(docUrl);
+        if (!response.ok) {
+          throw new Error(`HTTP error! status: ${response.status}`);
+        }
+        return response.text();
+      } catch (error) {
+        logger.error({ error }, "Failed to fetch document from URL: " + docUrl);
+        return null;
+      }
+    })).then((responses) => {
+      const keys = [];
+      for (const body of responses) {
+        if (!body) continue
+        try {
+          const result = bibParse(body);
+          const resultKeys = Object.keys(result);
+          keys.push(...resultKeys);
+        } catch(error) {
+          logger.error({error}, "skip the file.")
+        }
+      }
+      logger.info({ keys }, "all keys");
+      res.send({ keys })
+    })
+  }
+}

services/references/config/settings.defaults.cjs

@@ -0,0 +1,9 @@
+module.exports = {
+  internal: {
+    references: {
+      port: 3006,
+      host: process.env.REFERENCES_HOST || '127.0.0.1',
+    },
+  },
+}
+