Initial commit

.github/workflows/ci.yml

@@ -0,0 +1,39 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - develop
+      - main
+env:
+  DJANGO_ENV: test
+
+jobs:
+  pytest:
+    runs-on: ubuntu-22.04
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: Install Python dependencies
+        run: |
+          cd test_app
+          pip install --upgrade pip
+          pip install -r requirements.txt
+
+      - name: Run migrations
+        run: |
+          cd test_app
+          python manage.py makemigrations readux_ingest_ecds
+          python manage.py migrate
+
+      - name: Run Tests
+        run: |
+          cd test_app
+          pytest tests/

MANIFEST.in

@@ -0,0 +1,3 @@
+include LICENSE
+include README.md
+prune test*

README.md

@@ -1,2 +1,23 @@
 # readux-ingest-ecds
+
 Django app for Readux ingest specific to ECDS' infrastructure
+
+## Install
+
+bash~~~
+pip ...
+~~~
+
+### Make and run migrations
+
+bash~~~
+python manage.py makemigrations readux_ingest_ecds
+python manage.py migrate
+~~~
+
+## Settings
+
+- IIIF_APPS
+- INGEST_TMP_DIR
+- INGEST_PROCESSING_DIR
+- INGEST_OCR_DIR

pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ['setuptools>=40.8.0']
+build-backend = 'setuptools.build_meta'

readux_ingest_ecds/admin.py

@@ -0,0 +1,35 @@
+import os
+import logging
+from django.contrib import admin
+from django.urls import reverse
+from django.utils.html import format_html
+from django.shortcuts import redirect
+from django_celery_results.models import TaskResult
+from .models import Local
+from .tasks import local_ingest_task
+
+LOGGER = logging.getLogger(__name__)
+
+class LocalAdmin(admin.ModelAdmin):
+    """Django admin ingest.models.local resource."""
+    fields = ('bundle', 'image_server', 'collections')
+    show_save_and_add_another = False
+
+    def save_model(self, request, obj, form, change):
+        LOGGER.info(f'INGEST: Local ingest - {obj.id} - started by {request.user.username}')
+        obj.creator = request.user
+        super().save_model(request, obj, form, change)
+        if os.environ["DJANGO_ENV"] != 'test': # pragma: no cover
+            local_ingest_task.apply_async(args=[obj.id])
+        else:
+            local_ingest_task(obj.id)
+
+    def response_add(self, request, obj, post_url_continue=None):
+        obj.refresh_from_db()
+        manifest_id = obj.manifest.id
+        return redirect('/admin/manifests/manifest/{m}/change/'.format(m=manifest_id))
+
+    class Meta: # pylint: disable=too-few-public-methods, missing-class-docstring
+        model = Local
+
+admin.site.register(Local, LocalAdmin)

readux_ingest_ecds/apps.py

@@ -0,0 +1,9 @@
+"""Configuration for :class:`apps.ingest`"""
+from django.apps import AppConfig
+
+
+class ReaduxIngestEcdsConfig(AppConfig):
+    """Ingest config"""
+    name = 'readux_ingest_ecds'
+    verbose_name = 'Readux Ingest ECDS'
+    label = 'readux_ingest_ecds'

readux_ingest_ecds/celery.py

@@ -0,0 +1,16 @@
+"""
+Celery config for ingest tasks.
+"""
+import os
+from celery import Celery
+from django.conf import settings
+# import config.settings.local as settings
+
+# set the default Django settings module for the 'celery' program.
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings.local')
+app = Celery('readux_ingest_ecds', result_extended=True)
+
+# Using a string here means the worker will not have to
+# pickle the object when using Windows.
+app.config_from_object('django.conf:settings')
+app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)

readux_ingest_ecds/forms.py

@@ -0,0 +1,11 @@
+from django import forms
+from django.forms import ClearableFileInput
+from .models import Bulk
+
+class BulkVolumeUploadForm(forms.ModelForm):
+    class Meta:
+        model = Bulk
+        fields = ['image_server', 'volume_files', 'collections']
+        widgets = {
+            'volume_files': ClearableFileInput(attrs={'allow_multiple_selected': True}),
+        }

readux_ingest_ecds/helpers.py

@@ -0,0 +1,21 @@
+from django.conf import settings
+from django.apps import apps
+from django.core.exceptions import AppRegistryNotReady
+
+def get_iiif_models():
+   try:
+      return {
+         'Manifest': apps.get_model(settings.IIIF_MANIFEST_MODEL),
+         'ImageServer': apps.get_model(settings.IIIF_IMAGE_SERVER_MODEL),
+         'RelatedLink': apps.get_model(settings.IIIF_RELATED_LINK_MODEL),
+         'Canvas': apps.get_model(settings.IIIF_CANVAS_MODEL),
+         'Collection': apps.get_model(settings.IIIF_COLLECTION_MODEL),
+      }
+   except AppRegistryNotReady:
+      return {
+         'Manifest': settings.IIIF_MANIFEST_MODEL,
+         'ImageServer': settings.IIIF_IMAGE_SERVER_MODEL,
+         'RelatedLink': settings.IIIF_RELATED_LINK_MODEL,
+         'Canvas': settings.IIIF_CANVAS_MODEL,
+         'Collection': settings.IIIF_COLLECTION_MODEL,
+      }

readux_ingest_ecds/mail.py

@@ -0,0 +1,68 @@
+from traceback import format_tb
+from django.urls.base import reverse
+from django.template.loader import get_template
+from django.conf import settings
+from django.core.mail import send_mail
+
+def send_email_on_failure(task_watcher=None, exception=None, traceback=None):
+    """Function to send an email on task failure signal from Celery.
+
+    :param task_watcher: The task watcher object
+    :type task_watcher: app.ingest.models.TaskWatcher
+    :param exception: Exception instance raised
+    :type exception: Exception
+    :param traceback: Stack trace object
+    :type traceback: traceback
+    """
+    context = {}
+    if task_watcher is not None:
+        context['filename'] = task_watcher.filename
+    if exception is not None:
+        context['exception'] = exception.__repr__()
+    if traceback is not None:
+        context['traceback'] = '\n'.join(format_tb(traceback))
+    context['result_url'] = settings.HOSTNAME + reverse(
+        "admin:%s_%s_change"
+        % (
+            task_watcher.task_result._meta.app_label,
+            task_watcher.task_result._meta.model_name,
+        ),
+        args=[task_watcher.task_result.id],
+    )
+    html_email = get_template('ingest_failure_email.html').render(context)
+    text_email = get_template('ingest_failure_email.txt').render(context)
+    if task_watcher is not None and task_watcher.task_creator is not None:
+        send_mail(
+            '[Readux] Failed: Ingest ' + task_watcher.filename,
+            text_email,
+            settings.READUX_EMAIL_SENDER,
+            [task_watcher.task_creator.email],
+            fail_silently=False,
+            html_message=html_email
+        )
+
+def send_email_on_success(task_watcher=None):
+    context = {}
+    if task_watcher is not None:
+        context['filename'] = task_watcher.filename
+    if task_watcher is not None and task_watcher.associated_manifest is not None:
+        context['manifest_url'] = settings.HOSTNAME + reverse(
+            'admin:manifests_manifest_change', args=(task_watcher.associated_manifest.id,)
+        )
+        context['manifest_pid'] = task_watcher.associated_manifest.pid
+        context['volume_url'] = task_watcher.associated_manifest.get_volume_url()
+    else:
+        context['manifests_list_url'] = settings.HOSTNAME + reverse(
+            'admin:manifests_manifest_changelist'
+        )
+    html_email = get_template('ingest_success_email.html').render(context)
+    text_email = get_template('ingest_success_email.txt').render(context)
+    if task_watcher is not None and task_watcher.task_creator is not None:
+        send_mail(
+            '[Readux] Ingest complete: ' + task_watcher.filename,
+            text_email,
+            settings.READUX_EMAIL_SENDER,
+            [task_watcher.task_creator.email],
+            fail_silently=False,
+            html_message=html_email
+        )