Merge branch 'release/4.15'

.github/workflows/lighthouse.yml

@@ -19,7 +19,7 @@ jobs:
         ports:
           - 5432:5432
       solr:
-        image: solr:8.6
+        image: solr:9.2
         ports:
           - 8983:8983
     steps:

.github/workflows/sphinx_docs.yml

@@ -49,7 +49,7 @@ jobs:
       # for pull requests, exit with error if documentation coverage is incomplete
       - name: Report on documentation coverage
         if: ${{ github.event_name == 'pull_request' }}
-        run: if [[ $((`wc -l < docs/_build/coverage/python.txt`)) -eq 2 ]] ; then echo "Documentation coverage complete"; else cat docs/_build/coverage/python.txt && exit 1; fi
+        run: if [[ $((`wc -l < docs/_build/coverage/python.txt`)) -eq 3 ]] ; then echo "Documentation coverage complete"; else cat docs/_build/coverage/python.txt && exit 1; fi
 
       # when building on push to main, publish the built docs
       - name: Deploy built docs to github pages

.github/workflows/unit_tests.yml

@@ -30,7 +30,7 @@ jobs:
         ports:
           - 5432:5432
       solr:
-        image: solr:8.6
+        image: solr:9.2
         ports:
           - 8983:8983
     steps:
@@ -43,7 +43,7 @@ jobs:
           docker cp solr_conf ${{ job.services.solr.id }}:/opt/solr/server/solr/configsets/geniza
           docker exec --user root ${{ job.services.solr.id }} /bin/bash -c "chown -R solr:solr /opt/solr/server/solr/configsets/geniza"
 
-      - name: Copy solr configsets to solr home directory (Solr 8 specific)
+      - name: Copy solr configsets to solr home directory
         run: "docker exec -d ${{ job.services.solr.id }} cp -r /opt/solr/server/solr/configsets /var/solr/data"
 
       # Python version to use is stored in the .python-version file, which is the

.github/workflows/visual_tests.yml

@@ -53,7 +53,7 @@ jobs:
         ports:
           - 5432:5432
       solr:
-        image: solr:8.6
+        image: solr:9.2
         ports:
           - 8983:8983
     steps:

.pre-commit-config.yaml

@@ -1,22 +1,22 @@
 repos:
   - repo: https://github.com/psf/black
-    rev: 22.3.0 # Replace by any tag/version: https://github.com/psf/black/tags
+    rev: 23.3.0 # Replace by any tag/version: https://github.com/psf/black/tags
     hooks:
       - id: black
         # Assumes that your shell's `python` command is linked to python3.6+
         language_version: python
   - repo: https://github.com/pycqa/isort
-    rev: 5.9.3
+    rev: 5.12.0
     hooks:
       - id: isort
         args: ["--profile", "black", "--filter-files"]
   - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: v2.4.1
+    rev: v2.7.1
     hooks:
       - id: prettier
         exclude: \.html$
         # exclude django templates, which prettier does not support
   - repo: https://github.com/rtts/djhtml
-    rev: v1.4.9
+    rev: 3.0.6
     hooks:
       - id: djhtml

CHANGELOG.rst

@@ -1,6 +1,44 @@
 Change Log
 ==========
 
+4.15
+----
+
+- public site
+  - bugfix: On tag change, document indexing is one revision behind
+  - bugfix: Input date not always populating
+  - bugfix: Digital translation footnote in scholarship records behaving incorrectly, excluding other footnotes on source
+
+- image, transcription, translation viewer/editor
+  - As a front end desktop user, I would like to see a bigger version of the document image in order to read the document (especially when no transcription exists).
+  - As a public site viewer, I would like to see translation alongside the document image by default if both are present, so that I can read the document in my native language.
+  - As a content editor, I want the "pop out" button in the transcription editor up higher, so it's immediately accessible.
+  - As a content editor, I want the ability to add polygon annotation boxes using the transcription editor, so I can draw accurate bounding boxes around text.
+  - As a content editor, I want the location field for digital edition/translations to automatically populate from an existing edition/translation on the same source, so that I can save time manually re-entering it.
+  - bugfix: Editing/deleting parts of annotation box titles results in unexpected behavior (no change or deleting entire annotation box)
+  - bugfix: In Safari, ITT panel toggles leave trails
+  - bugfix: Annotations on the document detail page do not respect reordering
+  - bugfix: Transcription and translation may become misaligned when resizing window
+  - bugfix: Alignment between Arabic transcriptions and English translations is slightly off
+
+- admin
+  - As a content admin, I would like filters in the document admin to search by English and Hebrew language of translation, so that I can collect those documents for CSV export for use in teaching.
+  - As a content admin, I would like to include a rationale for the inferred date field from a list of options, so that I can enter data more efficiently and consistently.
+  - As a content admin, I want inferred date and accompanying notes in the csv exports of documents, so that I can keep track of this information in my own research.
+  - As a content editor, I want a "no language" option when entering source languages (with help text) for unpublished transcriptions because the language will automatically be determined by the document languages already present on the doc detail pages.
+  - As a content editor, I want clear help text when adding a source to explain how to select the source language, so that it is done consistently for translations and transcriptions.
+  - As a content admin, I want both dates on document and inferred dates to merge when I merge duplicate PGPIDS so no data is lost when cleaning up duplicates. If there are two different dates on documents for the same PGPID, I want there to be an error message drawing my attention to the issue so I can choose the correct date or otherwise record the discrepancy.
+  - As a content editor, I want a way to filter documents by date in the admin for enhanced csv exports
+  - bugfix: Mixed inlines/formsets breaks on lack of permissions
+  - bugfix: Merging two documents with digital content footnotes for the same source results in unique constraint violation
+
+- people and places
+  - As a content editor, I want a separate field to record people's names and roles in each document, so that I can build a structured dataset of all people across the PGP.
+  - As a content editor, I want a separate field in the document detail page so that I can record place information mentioned in the document.
+  - As a content editor, I want Person-Person relationship types visually sorted into their categories in the admin form, so that I can select them at a glance.
+  - As a content admin, when adding people-to-people relationships in person pages, I want an added "ambiguity" category to the drop down so I can clarify when people are similar/not the same.
+  - As a content admin, when viewing people-to-people relationships in person pages, I want reverse relationships to be visible, so that I don't inadvertently add a relationship twice.
+
 4.14.2
 ------
 

DEPLOYNOTES.md

@@ -1,5 +1,10 @@
 # Deploy Notes
 
+## 4.15
+
+-   The minimum required Solr version has been bumped to 9.2. Please upgrade to this version,
+    update Solr configset, and then reindex all content with `python manage.py index`.
+
 ## 4.14
 
 -   Seleucid calendar conversion is now implemented, so automatic conversion should be applied

README.rst

@@ -6,7 +6,7 @@ Princeton Geniza Project
 Python/Django web application for a version 4.x of the `Princeton Geniza Project
 <https://cdh.princeton.edu/projects/princeton-geniza-project/>`_.
 
-Python 3.9 / Django 3.2 / Node 16 / Postgresql / Solr 8.6
+Python 3.9 / Django 3.2 / Node 16 / Postgresql / Solr 9.2
 
 .. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.7347726.svg
    :target: https://doi.org/10.5281/zenodo.7347726

docs/conf.py

@@ -105,3 +105,6 @@
     "get_.*_display",  # django auto-generated method for choice fields
     "get_doc_relation_list",  # multiselectfield auto method
 ]
+
+# Disable Sphinx 7.2+ coverage statistics, as this breaks CI
+coverage_statistics_to_report = coverage_statistics_to_stdout = False

geniza/__init__.py

@@ -1,4 +1,4 @@
-__version_info__ = (4, 14, 2, None)
+__version_info__ = (4, 15, 0, None)
 
 
 # Dot-connect all but the last. Last is dash-connected if not None.

geniza/annotations/tests/test_annotations_views.py

@@ -1,23 +1,20 @@
-import ast
 import json
 import uuid
-from unittest.mock import patch
 
 import pytest
 from django.contrib.admin.models import ADDITION, CHANGE, DELETION, LogEntry
+from django.contrib.contenttypes.models import ContentType
 from django.urls import reverse
-from parasolr.django.indexing import ModelIndexable
 from pytest_django.asserts import assertContains, assertNotContains
 
 from geniza.annotations.models import Annotation
 from geniza.annotations.views import AnnotationResponse
 from geniza.corpus.models import Document
-from geniza.footnotes.models import Footnote, Source
+from geniza.footnotes.models import Footnote, Source, SourceType
 
 
 @pytest.mark.django_db
 class TestAnnotationList:
-
     anno_list_url = reverse("annotations:list")
 
     def test_get_annotation_list(self, client, annotation):
@@ -127,6 +124,9 @@ def test_create_annotation(self, admin_client, document, source, annotation_json
         )
         # will raise error if digital edition footnote does not exist
         footnote = document.digital_editions().get(source=source)
+        # since there was no corresponding Edition footnote with a Location, the
+        # resulting digital footnote should not get a location
+        assert not footnote.location
 
         # should log action
         assert LogEntry.objects.filter(
@@ -136,7 +136,6 @@ def test_create_annotation(self, admin_client, document, source, annotation_json
 
 @pytest.mark.django_db
 class TestAnnotationDetail:
-
     anno_list_url = reverse("annotations:list")
 
     def test_get_annotation_detail(self, client, annotation):
@@ -325,6 +324,44 @@ def test_delete_last_translation_anno(self, admin_client, translation_annotation
         assert footnote.annotation_set.count() == 0
         assert Footnote.DIGITAL_TRANSLATION not in footnote.doc_relation
 
+    def test_corresponding_footnote_location(self, admin_client, document):
+        document_contenttype = ContentType.objects.get_for_model(Document)
+        # create an Edition footnote on the document and source
+        book = SourceType.objects.create(type="Book")
+        source = Source.objects.create(source_type=book)
+        Footnote.objects.create(
+            doc_relation=[Footnote.EDITION],
+            object_id=document.pk,
+            content_type=document_contenttype,
+            source=source,
+            location="doc. 123",
+        )
+        # POST JSON to create a new annotation on the document and source
+        anno_dict = {
+            "body": [{"value": "new text"}],
+            "target": {
+                "source": {
+                    "partOf": {"id": document.manifest_uri},
+                }
+            },
+            "dc:source": source.uri,
+            "motivation": "transcribing",
+        }
+        admin_client.post(
+            self.anno_list_url,
+            json.dumps(anno_dict),
+            content_type="application/json",
+        )
+        # should not raise error because digital edition created by request
+        created_digital_edition = Footnote.objects.get(
+            doc_relation=[Footnote.DIGITAL_EDITION],
+            source__pk=source.pk,
+            content_type=document_contenttype,
+            object_id=document.pk,
+        )
+        # should have its location copied from the existing Edition footnote
+        assert created_digital_edition.location == "doc. 123"
+
 
 @pytest.mark.django_db
 class TestAnnotationSearch:

geniza/annotations/views.py

@@ -61,8 +61,10 @@ def parse_annotation_data(request):
     # determine if this is a transcription or translation
     if "motivation" in json_data and "translating" in json_data["motivation"]:
         doc_relation = Footnote.DIGITAL_TRANSLATION
+        corresponding_relation = Footnote.TRANSLATION
     else:
         doc_relation = Footnote.DIGITAL_EDITION
+        corresponding_relation = Footnote.EDITION
 
     # find or create DIGITAL_EDITION footnote for this source and document
     try:
@@ -74,11 +76,32 @@ def parse_annotation_data(request):
         )
     except Footnote.DoesNotExist:
         source = Source.objects.get(pk=source_id)
+
+        # try to find a corresponding non-digital footnote for location field
+        # (i.e. Translation for Digital Translation, Edition for Digital Edition)
+        # NOTE: assumes that if there is exactly one non-digital footnote for this source, then
+        # the digital content is coming from the same location
+        try:
+            # use .get to ensure there is exactly one corresponding;
+            # otherwise ambiguous which location to use
+            corresponding_footnote = Footnote.objects.exclude(location="").get(
+                doc_relation__contains=corresponding_relation,
+                source__pk=source_id,
+                content_type=document_contenttype,
+                object_id=document_id,
+            )
+            location = corresponding_footnote.location
+        except (Footnote.DoesNotExist, Footnote.MultipleObjectsReturned):
+            # if there are 0 or > 1 footnotes, location should be blank
+            location = ""
+
+        # create a new digital footnote
         footnote = Footnote.objects.create(
             source=source,
             doc_relation=[doc_relation],
             object_id=document_id,
             content_type=document_contenttype,
+            location=location,
         )
         LogEntry.objects.log_action(
             user_id=request.user.id,

geniza/common/models.py

@@ -1,6 +1,17 @@
+from functools import cache
+
 from django.contrib.auth.models import User
 from django.db import models
 from django.utils.safestring import mark_safe
+from modeltranslation.utils import fallbacks
+
+
+def cached_class_property(f):
+    """
+    Reusable decorator to cache a class property, as opposed to an instance property.
+    from https://stackoverflow.com/a/71887897
+    """
+    return classmethod(property(cache(f)))
 
 
 # Create your models here.
@@ -60,3 +71,42 @@ class UserProfile(models.Model):
     def __str__(self):
         # needed for display label in admin
         return "User profile for %s" % (self.user)
+
+
+class DisplayLabelMixin:
+    """
+    Mixin for models with translatable display labels that may differ from names, in
+    order to override fallback behavior when a label for the current language is not defined.
+    Used for search response handling and display on the public frontend.
+
+    Example: DocumentType with name 'Legal' has a display label in English, 'Legal document'.
+    In Hebrew, it only has a name 'מסמך משפטי' and no display label. In English, we want to show
+    DocumentType.display_label_en. In Hebrew, we want to show DocumentType.name_he because
+    display_label_he is not defined. We also need to ensure that the document type
+    מסמך משפטי can be looked up by display_label_en, as that is what gets indexed in solr.
+    """
+
+    def __str__(self):
+        # temporarily turn off model translate fallbacks;
+        # if display label for current language is not defined,
+        # we want name for the current language rather than the
+        # fallback value for display label
+        with fallbacks(False):
+            current_lang_label = self.display_label or self.name
+
+        return current_lang_label or self.display_label or self.name
+
+    def natural_key(self):
+        """Natural key, name"""
+        return (self.name,)
+
+    @classmethod
+    def objects_by_label(cls):
+        """A dict of object instances keyed on English display label, used for search form
+        and search results, which should be based on Solr facet and query responses (indexed in
+        English)."""
+        return {
+            # lookup on display_label_en/name_en since solr should always index in English
+            (obj.display_label_en or obj.name_en): obj
+            for obj in cls.objects.all()
+        }

geniza/context_processors.py

@@ -18,5 +18,9 @@ def template_globals(request):
         ),
         "site": site,
         "GTAGS_ANALYTICS_ID": getattr(settings, "GTAGS_ANALYTICS_ID", None),
+        "IS_ARCHIVE_CRAWLER": "archive.org_bot"
+        in request.META.get("HTTP_USER_AGENT", "")
+        if hasattr(request, "META")
+        else False,
     }
     return context_extras