Merge branch 'master' into retorqueregh-2300

.github/workflows/ci.yaml

@@ -117,7 +117,6 @@ jobs:
         exclude:
         - client: jurism-beta
         - client: jurism
-          worker: --worker
     steps:
     - uses: actions/checkout@v3
     - name: fetch build artifacts

.github/workflows/src/ci.yaml

@@ -152,7 +152,7 @@ jobs:
         exclude:
           - client: jurism-beta
           - client: jurism
-            worker: '--worker'
+          #  worker: '--worker'
     steps:
     - uses: actions/checkout@v3
     - *download_build_artifacts

content/Preferences.pug

@@ -10,23 +10,23 @@
 | <?xml-stylesheet href="chrome://zotero-better-bibtex/skin/error-report.css" type="text/css"?>
 | <!DOCTYPE window SYSTEM "chrome://zotero-better-bibtex/locale/zotero-better-bibtex.dtd">
 overlay#zotero-better-bibtex-preferences(xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul" xmlns:html="http://www.w3.org/1999/xhtml" xmlns:bbt="http://retorque.re/zotero-better-bibtex/")
-  popupset
-    tooltip#tooltip-bibtexURL
-      description &better-bibtex.Preferences.export.fields.doi-and-url.warning;
-    tooltip#tooltip-jabrefFormat
-      description &better-bibtex.Preferences.export.jabrefFormat.warn;
-    tooltip#tooltip-importSentenceCase
-      description &better-bibtex.Preferences.advanced.import.sentenceCase.warning;
-    tooltip#tooltip-importCaseProtection
-      description &better-bibtex.Preferences.advanced.import.caseProtection.warning;
-    tooltip#tooltip-exportTitleCase
-      description &better-bibtex.Preferences.advanced.export.titleCase.warning;
-    tooltip#tooltip-exportBraceProtection
-      description &better-bibtex.Preferences.advanced.export.braceProtection.warning;
-    tooltip#tooltip-retainCache
-      description &better-bibtex.Preferences.advanced.export.retainCache.warning;
   prefwindow#zotero-prefs
     prefpane#zotero-prefpane-better-bibtex(insertafter="zotero-prefpane-advanced" label="&better-bibtex.Preferences.prefpane.better-bibtex;" image="chrome://zotero-better-bibtex/skin/bibtex.svg" onpaneload="load()" helpTopic="BetterBibTeX")
+      popupset
+        tooltip#tooltip-bibtexURL
+          description &better-bibtex.Preferences.export.fields.doi-and-url.warning;
+        tooltip#tooltip-jabrefFormat
+          description &better-bibtex.Preferences.export.jabrefFormat.warn;
+        tooltip#tooltip-importSentenceCase
+          description &better-bibtex.Preferences.advanced.import.sentenceCase.warning;
+        tooltip#tooltip-importCaseProtection
+          description &better-bibtex.Preferences.advanced.import.caseProtection.warning;
+        tooltip#tooltip-exportTitleCase
+          description &better-bibtex.Preferences.advanced.export.titleCase.warning;
+        tooltip#tooltip-exportBraceProtection
+          description &better-bibtex.Preferences.advanced.export.braceProtection.warning;
+        tooltip#tooltip-retainCache
+          description &better-bibtex.Preferences.advanced.export.retainCache.warning;
 
       preferences#zotero-preferences-cite
         include Preferences/preferences.pug

content/Preferences.ts

@@ -16,9 +16,62 @@ import * as l10n from './l10n'
 import { Events } from './events'
 import { pick } from './file-picker'
 import { flash } from './flash'
+const dtdparser = require('./dtd-file.peggy')
 
 const namespace = 'http://retorque.re/zotero-better-bibtex/'
 
+function escapeHtml(unsafe: string): string {
+  return unsafe
+    .replace(/&/g, '&')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+    .replace(/"/g, '&quot;')
+    .replace(/'/g, '&#039;')
+}
+
+export function start(win: Window): any {
+  const prefwindow = win.document.querySelector('prefwindow#zotero-prefs')
+  if (!prefwindow) return log.error('prefs.start: prefwindow not found')
+  if (prefwindow) return 0
+
+  let xml = Zotero.File.getContentsFromURL('chrome://zotero-better-bibtex/content/Preferences.xul')
+  const url = xml.match(/<!DOCTYPE window SYSTEM "([^"]+)">/)[1]
+  const dtd: Record<string, string> = dtdparser.parse(Zotero.File.getContentsFromURL(url))
+  for (const [key, value] of Object.entries(dtd)) {
+    xml = xml.replace(new RegExp(`&${key};`, 'g'), escapeHtml(value))
+  }
+  const parser = new DOMParser
+  const xul = parser.parseFromString(xml, 'text/xml')
+
+  xul.querySelectorAll('*[onpaneload]').forEach(elt => {
+    elt.removeAttribute('onpaneload')
+  })
+  xul.querySelectorAll('script').forEach(elt => {
+    elt.remove()
+  })
+
+  const prefpane = xul.querySelector('prefpane')
+  let id: string = prefpane.getAttribute('id')
+  log.debug('prefs id=', id)
+  if (win.document.querySelector(`prefpane#${id}`)) {
+    log.debug('prefpane: already loaded')
+    return
+  }
+
+  let neighbour: Element
+  if ((id = prefpane.getAttribute('insertafter')) && (neighbour = win.document.querySelector(`prefpane#${id}`)) && neighbour.nextSibling) {
+    prefwindow.insertBefore(prefpane, neighbour.nextSibling)
+  }
+  else if ((id = prefpane.getAttribute('insertbefore')) && (neighbour = win.document.querySelector(`prefpane#${id}`))) {
+    prefwindow.insertBefore(prefpane, neighbour)
+  }
+  else {
+    prefwindow.appendChild(prefpane)
+  }
+  log.debug('>>>\n', win.document.documentElement.outerHTML, '\n<<<')
+  log.debug('prefpane: appended:', !!win.document.querySelector(`prefpane#${id}`))
+}
+
 class AutoExportPane {
   private label: { [key: string]: string }
   private globals: Record<string, any>

content/ajv.ts

@@ -81,11 +81,13 @@ for (const ajv of [coercing, noncoercing]) {
 
 import betterAjvErrors from 'better-ajv-errors'
 
+type AjvError = { error: string, suggestion: string }
 export function validator(schema, ajv): (data: any) => string { // eslint-disable-line @typescript-eslint/explicit-module-boundary-types
   const ok = ajv.compile(schema)
   return function(data: any): string { // eslint-disable-line prefer-arrow/prefer-arrow-functions
     if (ok(data)) return ''
-    return betterAjvErrors(schema, data, ok.errors, { format: 'js' }).map(err => err.error + (err.suggestion ? ', ' : '') + (err.suggestion || '')).join('\n')
+    return (betterAjvErrors(schema, data, ok.errors, { format: 'js' }) as AjvError[])
+      .map((err: AjvError) => err.error + (err.suggestion ? `, ${err.suggestion}` : '')).join('\n')
   }
 }
 

content/auto-export.ts

@@ -315,7 +315,7 @@ const queue = new class TaskQueue {
       }
 
       log.debug('ae.starting', jobs.length, 'jobs for', ae.$loki)
-      await Promise.all(jobs.map(job => Translators.exportItemsByWorker(job)))
+      await Promise.all(jobs.map(job => Translators.queueJob(job)))
       log.debug('ae.done', jobs.length, 'jobs for', ae.$loki)
 
       await repo.push(l10n.localize('Preferences.auto-export.git.message', { type: Translators.byId[ae.translatorID].label.replace('Better ', '') }))

content/better-bibtex.ts

@@ -25,6 +25,7 @@ import { flash } from './flash'
 import { Deferred } from './deferred'
 
 import { Preference } from './prefs' // needs to be here early, initializes the prefs observer
+import { start as prefpane_start } from './Preferences'
 import * as preferences from '../gen/preferences/meta'
 require('./pull-export') // just require, initializes the pull-export end points
 require('./json-rpc') // just require, initializes the json-rpc end point
@@ -1018,9 +1019,21 @@ export class BetterBibTeX {
       */
       setProgress(percent, message) // eslint-disable-line no-magic-numbers
     })
+    Events.on('window-loaded', (win: Window, href: string) => {
+      switch (href) {
+        case 'chrome://zotero/content/preferences/preferences.xul':
+          prefpane_start(win)
+          break
+        default:
+          log.debug('window-loaded', href)
+      }
+    })
   }
 
   public parseDate(date: string): ParsedDate { return DateParser.parse(date) }
+  public unload(): void {
+    Zotero.debug('Unloading BBT?')
+  }
 }
 
 Zotero.BetterBibTeX = Zotero.BetterBibTeX || new BetterBibTeX

content/db/main.ts

@@ -144,7 +144,7 @@ class Main extends Loki {
       Preference.scrubDatabase = false
     }
 
-    autoexport.on(['insert', 'update'], (ae: { path: string, $loki: number }) => {
+    autoexport.on(['pre-insert', 'pre-update'], (ae: { path: string, $loki: number }) => {
       autoexport.removeWhere({ $and: [ { path: ae.path }, { $loki: { $ne: ae.$loki } } ] })
     })
   }

content/dtd-file.peggy

@@ -0,0 +1,24 @@
+start = strings:string* {
+      return strings.reduce((acc, string) => {
+        if (string) acc[string.key] = string.value
+        return acc
+      }, {})
+    }
+
+string
+  = _ { return null }
+  / '<!--' comment* '-->' { return null }
+  / '<!' _? 'entity'i _ key:key _ value:value _? '>' { return { key, value } }
+
+comment
+  = !'-->' '-'
+  / [^-]
+
+key = key:$[-a-zA-Z0-9._]+ { return key }
+
+value
+  = "'" value:$[^']* "'" { return value }
+  / '"' value:$[^"]* '"' { return value }
+
+_ = [ \t\n\r]+
+

content/events.ts

@@ -4,32 +4,34 @@ import { EventEmitter as EventEmitter3 } from 'eventemitter3'
 import { log } from './logger'
 
 const events: string[] = [
+  'collections-changed',
+  'collections-removed',
   'error',
-  'preference-changed',
+  'export-progress',
   'item-tag',
   'items-changed',
   'items-removed',
   'libraries-changed',
-  'collections-changed',
-  'collections-removed',
   'libraries-removed',
-  'export-progress',
   'loaded',
+  'preference-changed',
+  'window-loaded',
 ]
 const event_prefix = events.map(name => name + '.')
 
-const log_events = Zotero.Prefs.get('translators.better-bibtex.log-events')
-
 export const Events = new class EventEmitter extends EventEmitter3 {
+  testing: boolean
+
   constructor() {
     super()
+    this.testing = Zotero.Prefs.get('translators.better-bibtex.log-events')
     this.on('error', err => {
       throw Zotero.debug(err)
     })
   }
 
   private verify(event: string | symbol) {
-    if (!log_events) return true
+    if (!this.testing) return true
     if (typeof event === 'symbol') return false
     if (events.includes(event) || event_prefix.find(prefix => event.startsWith(prefix))) return true
     throw new Error(`Unsupported event ${event}`)
@@ -48,7 +50,14 @@ export const Events = new class EventEmitter extends EventEmitter3 {
     const results: boolean[] = []
     for (const listening of this.eventNames()) {
       if (listening === event || (typeof listening === 'string' && listening.startsWith(prefix))) {
-        log.debug('event.emit(', listening, args, ')')
+        if (this.testing) {
+          try {
+            log.debug('event.emit(', listening, args, ')')
+          }
+          catch (err) {
+            log.debug('event.emit(', listening, ')')
+          }
+        }
         results.push(super.emit.apply(this, [listening, ...args]))
       }
     }
@@ -81,3 +90,16 @@ export const Events = new class EventEmitter extends EventEmitter3 {
     if (changed.libraries.size) this.emit('libraries-changed', [...changed.libraries])
   }
 }
+
+const windowListener = {
+  onOpenWindow: xulWindow => {
+    const win = xulWindow.QueryInterface(Components.interfaces.nsIInterfaceRequestor).getInterface(Components.interfaces.nsIDOMWindow)
+    win.addEventListener('load', function listener() { // eslint-disable-line prefer-arrow/prefer-arrow-functions
+      win.removeEventListener('load', listener, false)
+      Events.emit('window-loaded', win, win.location.href)
+    }, false)
+  },
+  // onCloseWindow: () => { },
+  // onWindowTitleChange: _xulWindow => { },
+}
+Services.wm.addListener(windowListener)

content/json-rpc.ts

@@ -145,7 +145,7 @@ class NSItem {
 
       // eslint-disable-next-line @typescript-eslint/no-unsafe-return
       return {
-        ...Zotero.Utilities.itemToCSLJSON(item),
+        ...Zotero.Utilities.Item.itemToCSLJSON(item),
         library: libraries[item.libraryID],
         citekey: Zotero.BetterBibTeX.KeyManager.keys.findOne($and({ libraryID: item.libraryID, itemID: item.id })).citekey,
       }

content/key-manager.ts

@@ -34,7 +34,7 @@ type CitekeySearchRecord = { itemID: number, libraryID: number, itemKey: string,
 export class KeyManager {
   public keys: any
   public query: {
-    field: { extra?: number }
+    field: { extra?: number, title?: number }
     type: {
       note?: number
       attachment?: number
@@ -386,14 +386,19 @@ export class KeyManager {
       const m = keyLine.exec(extra)
       return m ? m[2].trim() : ''
     }
-    const db: Map<number, { itemKey: string, citationKey: string }> = (await ZoteroDB.queryAsync(`
+
+    type DBState = Map<number, { itemKey: string, citationKey: string }>
+    const inzdb: DBState = (await ZoteroDB.queryAsync(`
       SELECT item.itemID, item.key, extra.value as extra
       FROM items item
-      LEFT JOIN itemData field ON field.itemID = item.itemID AND field.fieldID = ${this.query.field.extra}
-      LEFT JOIN itemDataValues extra ON extra.valueID = field.valueID
-      WHERE item.itemID NOT IN (select itemID from deletedItems)
-      AND item.itemTypeID NOT IN (${this.query.type.attachment}, ${this.query.type.note}, ${this.query.type.annotation || this.query.type.note})
-    `)).reduce((acc: Map<number, { itemKey: string, citationKey: string }>, item) => {
+
+      LEFT JOIN itemData extraField ON extraField.itemID = item.itemID AND extraField.fieldID = ${this.query.field.extra}
+      LEFT JOIN itemDataValues extra ON extra.valueID = extraField.valueID
+
+      WHERE item.itemID NOT IN (SELECT itemID FROM deletedItems)
+        AND item.itemTypeID NOT IN (${this.query.type.attachment}, ${this.query.type.note}, ${this.query.type.annotation || this.query.type.note})
+        AND item.itemID NOT IN (SELECT itemID from feedItems)
+    `)).reduce((acc: DBState, item) => {
       acc.set(item.itemID, {
         itemKey: item.key,
         citationKey: getKey(item.extra),
@@ -402,40 +407,34 @@ export class KeyManager {
     }, new Map)
 
     const deleted: number[] = []
-    for (const item of this.keys.data) {
-      const key = db.get(item.itemID)
+    for (const bbt of this.keys.data) {
+      const zotero = inzdb.get(bbt.itemID)
 
-      if (!key) {
-        deleted.push(item.itemID)
+      if (!zotero) {
+        deleted.push(bbt.itemID)
+        // log.debug('keymanager.rescan: deleted', bbt, 'from key database, no counterpart in Zotero DB')
       }
-      else if (key.citationKey && (!item.pinned || item.citekey !== key.citationKey)) {
-        this.keys.update({...item, pinned: true, citekey: key.citationKey, itemKey: key.itemKey })
+      else if (zotero.citationKey && (!bbt.pinned || bbt.citekey !== zotero.citationKey)) {
+        this.keys.update({...bbt, pinned: true, citekey: zotero.citationKey, itemKey: zotero.itemKey })
+        // log.debug('keymanager.rescan: updated', bbt, 'using', zotero)
       }
-      else if (!key.citationKey && item.citekey && item.pinned) {
-        this.keys.update({...item, pinned: false, itemKey: key.itemKey})
+      else if (!zotero.citationKey && bbt.citekey && bbt.pinned) {
+        this.keys.update({...bbt, pinned: false, itemKey: zotero.itemKey})
+        // log.debug('keymanager.rescan: updated', bbt, 'using', zotero)
       }
-      else if (!item.citekey) {
-        this.regenerate.push(item.itemID)
+      else if (!bbt.citekey) { // this should not be possible
+        this.regenerate.push(bbt.itemID)
+        // log.debug('keymanager.rescan: regenerating', bbt, 'missing citekey')
       }
 
-      db.delete(item.itemID)
+      inzdb.delete(bbt.itemID)
     }
+    // if (inzdb.size) log.debug('keymanager.rescan:', inzdb.size, 'new items', { itemIDs: [...inzdb.entries()] })
 
     this.keys.findAndRemove({ itemID: { $in: [...deleted, ...this.regenerate] } })
-    this.regenerate.push(...db.keys())
-
-    const regenerate = (
-      this.regenerate.length !== 0
-      /*
-      &&
-      (
-        Preference.testing
-        ||
-        Services.prompt.confirm(null, l10n.localize('KeyManager.regenerate'), l10n.localize('KeyManager.regenerate.confirm', { n: this.regenerate.length }))
-      )
-      */
-    )
-    if (regenerate) {
+    this.regenerate.push(...inzdb.keys()) // generate new keys for items that are in the Z db but not in the BBT db
+
+    if (this.regenerate.length) {
       const progressWin = new Zotero.ProgressWindow({ closeOnClick: false })
       progressWin.changeHeadline('Better BibTeX: Assigning citation keys')
       progressWin.addDescription(`Found ${this.regenerate.length} items without a citation key`)