Remove FITS

This removes: * the fits service and its package dependencies from the Compose environment * the FITS_v0.0 client script from the MCPClient and updates the MCPServer workflow to use the identifyFileFormat_v0.0 and characterizeFile_v0.0 scripts for manually normalized preservation files * the FPR model instances (FPTool, FPCommand and FPRule) related to FITS with a database data migration in the fpr application of the Dashboard
artefactual · Sep 11, 2024 · 0212db1 · 0212db1
1 parent f991796
commit 0212db1
Show file tree

Hide file tree

Showing 10 changed files with 63 additions and 189 deletions.
diff --git a/hack/Dockerfile b/hack/Dockerfile
@@ -168,7 +168,6 @@ RUN set -ex \
 		clamav \
 		coreutils \
 		ffmpeg \
-		fits \
 		g++ \
 		gcc \
 		gearman \
@@ -190,7 +189,6 @@ RUN set -ex \
 		md5deep \
 		mediaconch \
 		mediainfo \
-		nailgun \
 		nfs-common \
 		openjdk-8-jre-headless \
 		p7zip-full \

diff --git a/hack/README.md b/hack/README.md
@@ -64,7 +64,6 @@ am-archivematica-mcp-server-1        39.43MiB / 7.763GiB
 am-archivematica-storage-service-1   83.96MiB / 7.763GiB
 am-nginx-1                           2.715MiB / 7.763GiB
 am-elasticsearch-1                   900.2MiB / 7.763GiB
-am-fits-1                            71.09MiB / 7.763GiB
 am-gearmand-1                        3.395MiB / 7.763GiB
 am-mysql-1                           551.9MiB / 7.763GiB
 am-clamavd-1                         570MiB / 7.763GiB
@@ -312,7 +311,6 @@ echo workers | socat - tcp:127.0.0.1:62004,shut-none | grep "_v0.0" | awk '{prin
 | mysql                                   | `tcp/3306`     | `tcp/62001` |
 | elasticsearch                           | `tcp/9200`     | `tcp/62002` |
 | gearman                                 | `tcp/4730`     | `tcp/62004` |
-| fits                                    | `tcp/2113`     | `tcp/62005` |
 | clamavd                                 | `tcp/3310`     | `tcp/62006` |
 | nginx » archivematica-dashboard         | `tcp/80`       | `tcp/62080` |
 | nginx » archivematica-storage-service   | `tcp/8000`     | `tcp/62081` |

diff --git a/hack/docker-compose.yml b/hack/docker-compose.yml
@@ -76,14 +76,6 @@ services:
     ports:
       - "127.0.0.1:62004:4730"
 
-  fits:
-    image: "artefactual/fits-ngserver:0.8.4"
-    user: ${USER_ID:-1000}
-    ports:
-      - "127.0.0.1:62005:2113"
-    volumes:
-      - "archivematica_pipeline_data:/var/archivematica/sharedDirectory:rw"  # Read and write needed!
-
   clamavd:
     image: "artefactual/clamav:latest"
     environment:
@@ -146,8 +138,6 @@ services:
     environment:
       DJANGO_SECRET_KEY: "12345"
       DJANGO_SETTINGS_MODULE: "settings.common"
-      NAILGUN_SERVER: "fits"
-      NAILGUN_PORT: "2113"
       ARCHIVEMATICA_MCPCLIENT_CLIENT_USER: "archivematica"
       ARCHIVEMATICA_MCPCLIENT_CLIENT_PASSWORD: "demo"
       ARCHIVEMATICA_MCPCLIENT_CLIENT_HOST: "mysql"
@@ -169,7 +159,6 @@ services:
       - "../:/src"
       - "archivematica_pipeline_data:/var/archivematica/sharedDirectory:rw"
     links:
-      - "fits"
       - "clamavd"
       - "mysql"
       - "gearmand"

diff --git a/src/MCPClient/lib/archivematicaClientModules b/src/MCPClient/lib/archivematicaClientModules
@@ -26,7 +26,6 @@ removeunneededfiles_v0.0 = remove_unneeded_files
 archivematicaclamscan_v0.0 = archivematica_clamscan
 createevent_v0.0 = create_event
 examinecontents_v0.0 = examine_contents
-fits_v0.0 = fits
 identifydspacefiles_v0.0 = identify_dspace_files
 identifydspacemetsfiles_v0.0 = identify_dspace_mets_files
 identifyfileformat_v0.0 = identify_file_format

diff --git a/src/MCPClient/lib/clientScripts/characterize_file.py b/src/MCPClient/lib/clientScripts/characterize_file.py
@@ -5,7 +5,7 @@
 # b) Prints the tool's stdout, for tools which do not output XML
 #
 # If a tool has no defined characterization commands, then the default
-# will be run instead (currently FITS).
+# will be run instead.
 import argparse
 import dataclasses
 import multiprocessing

diff --git a/src/MCPClient/lib/clientScripts/fits.py b/src/MCPClient/lib/clientScripts/fits.py
diff --git a/src/MCPClient/lib/clientScripts/manual_normalization_create_metadata_and_restructure.py b/src/MCPClient/lib/clientScripts/manual_normalization_create_metadata_and_restructure.py
@@ -120,7 +120,12 @@ def main(job):
     # We found the original file somewhere above
     job.print_output(
         "Matched original file %s (%s) to  preservation file %s (%s)"
-        % (original_file.currentlocation, original_file.uuid, filePath, fileUUID)
+        % (
+            original_file.currentlocation.decode(),
+            original_file.uuid,
+            filePath,
+            fileUUID,
+        )
     )
     # Generate the new preservation path: path/to/original/filename-uuid.ext
     basename = os.path.basename(filePath)

diff --git a/src/MCPClient/lib/clientScripts/manual_normalization_move_access_files_to_dip.py b/src/MCPClient/lib/clientScripts/manual_normalization_move_access_files_to_dip.py
@@ -106,7 +106,7 @@ def main(job):
             }
             f = File.objects.get(**kwargs)
         else:
-            if isinstance(e, File.DoesNotExist, ValidationError):
+            if isinstance(e, (File.DoesNotExist, ValidationError)):
                 job.print_error(
                     "No matching file for: ",
                     opts.filePath.replace(opts.SIPDirectory, "%SIPDirectory%", 1),

diff --git a/src/MCPServer/lib/assets/workflow.json b/src/MCPServer/lib/assets/workflow.json
@@ -1927,15 +1927,15 @@
       "config": {
         "@manager": "linkTaskManagerFiles",
         "@model": "StandardTaskConfig",
-        "arguments": "\"%relativeLocation%\" \"%SIPLogsDirectory%fileMeta/%fileUUID%.xml\" \"%date%\" \"%taskUUID%\" \"%fileUUID%\" \"%fileGrpUse%\"",
-        "execute": "FITS_v0.0",
+        "arguments": "\"%fileUUID%\" \"%SIPUUID%\"",
+        "execute": "characterizeFile_v0.0",
         "filter_subdir": "objects/manualNormalization/preservation"
       },
       "description": {
-        "en": "Run FITS on manual normalized preservation files",
-        "no": "Kjør FITS på manuelt normaliserte bevaringsfiler",
-        "pt_BR": "Executar o FITS em arquivos de preservação normalizados manuaimente",
-        "sv": "Kör FITS på manuellt normaliserade bevarandefiler"
+        "en": "Characterize and extract metadata on manual normalized preservation files",
+        "no": "Karakteriser og hent ut metadata på manuelt normaliserte bevaringsfiler",
+        "pt_BR": "Caracterizar e extrair metadados em arquivos de preservação normalizados manuaimente",
+        "sv": "Karaktärisera och extrahera metadata på manuellt normaliserade bevarandefiler"
       },
       "exit_codes": {
         "0": {
@@ -1948,7 +1948,7 @@
       "group": {
         "en": "Process manually normalized files",
         "es": "Procesar manualmente ficheros normalizados",
-        "no": "Prosesser normaliserte filer manuelt",
+        "no": "Prosesser manuelt normaliserte filer",
         "pt_BR": "Processar arquivos normalizados manualmente",
         "sv": "Bearbeta manuellt normaliserade filer"
       }
@@ -2934,11 +2934,11 @@
       "exit_codes": {
         "0": {
           "job_status": "Completed successfully",
-          "link_id": "bd382151-afd0-41bf-bb7a-b39aef728a32"
+          "link_id": "1b1a4565-b501-407b-b40f-2f20889423f1"
         }
       },
       "fallback_job_status": "Failed",
-      "fallback_link_id": "bd382151-afd0-41bf-bb7a-b39aef728a32",
+      "fallback_link_id": "1b1a4565-b501-407b-b40f-2f20889423f1",
       "group": {
         "en": "Extract packages",
         "es": "Extraer paquetes",
@@ -7667,7 +7667,7 @@
         }
       },
       "fallback_job_status": "Failed",
-      "fallback_link_id": "bd382151-afd0-41bf-bb7a-b39aef728a32",
+      "fallback_link_id": "1b1a4565-b501-407b-b40f-2f20889423f1",
       "group": {
         "en": "Extract packages",
         "es": "Extraer paquetes",
@@ -9044,41 +9044,6 @@
         "sv": "Normalisera"
       }
     },
-    "bd382151-afd0-41bf-bb7a-b39aef728a32": {
-      "config": {
-        "@manager": "linkTaskManagerFiles",
-        "@model": "StandardTaskConfig",
-        "arguments": "\"%relativeLocation%\" \"%SIPLogsDirectory%fileMeta/%fileUUID%.xml\" \"%date%\" \"%taskUUID%\" \"%fileUUID%\" \"%fileGrpUse%\"",
-        "execute": "FITS_v0.0",
-        "filter_subdir": "objects/attachments"
-      },
-      "description": {
-        "en": "Characterize and extract metadata for attachments",
-        "es": "Caracterizar y extraer los metadatos de los adjuntos",
-        "fr": "Caractériser et extraire les métadonnées pour mettre en pièces jointes",
-        "ja": "添付ファイルのメタデータの特徴付けと抽出",
-        "no": "Karakteriser og hent ut metadata fra vedlegg",
-        "pt_BR": "Caracterizar e extrair metadados para anexos",
-        "sv": "Karaktärisera och extrahera metadata för bilagor"
-      },
-      "exit_codes": {
-        "0": {
-          "job_status": "Completed successfully",
-          "link_id": "1b1a4565-b501-407b-b40f-2f20889423f1"
-        }
-      },
-      "fallback_job_status": "Failed",
-      "fallback_link_id": "61c316a6-0a50-4f65-8767-1f44b1eeb6dd",
-      "group": {
-        "en": "Characterize and extract metadata",
-        "es": "Caracterizar y extraer metadatos",
-        "fr": "Caractériser et extraire les métadonnées",
-        "ja": "メタデータの特徴付けと抽出",
-        "no": "Karakteriser og hent ut metadata",
-        "pt_BR": "Caracterizar e extrair metadados",
-        "sv": "Karaktärisera och extrahera metadata"
-      }
-    },
     "bd792750-a55b-42e9-903a-8c898bb77df1": {
       "config": {
         "@manager": "linkTaskManagerDirectories",
@@ -9276,6 +9241,38 @@
         "sv": "Byt namn på SIP-mappen med SIP UUID"
       }
     },
+    "bf0ea0f6-211b-4b34-8f25-8a68145403c8": {
+      "config": {
+        "@manager": "linkTaskManagerFiles",
+        "@model": "StandardTaskConfig",
+        "arguments": "\"True\" \"%relativeLocation%\" \"%fileUUID%\" --disable-reidentify",
+        "execute": "identifyFileFormat_v0.0",
+        "filter_subdir": "objects/manualNormalization/preservation"
+      },
+      "description": {
+        "en": "Identify file format",
+        "es": "Identificar formato de fichero",
+        "fr": "Identifier le format de fichier",
+        "no": "Identifiser filformat",
+        "pt_BR": "Identifique o formato do arquivo",
+        "sv": "Identifiera filformat"
+      },
+      "exit_codes": {
+        "0": {
+          "job_status": "Completed successfully",
+          "link_id": "10c40e41-fb10-48b5-9d01-336cd958afe8"
+        }
+      },
+      "fallback_job_status": "Failed",
+      "fallback_link_id": "10c40e41-fb10-48b5-9d01-336cd958afe8",
+      "group": {
+        "en": "Process manually normalized files",
+        "es": "Procesar manualmente ficheros normalizados",
+        "no": "Prosesser manuelt normaliserte filer",
+        "pt_BR": "Processar arquivos normalizados manualmente",
+        "sv": "Bearbeta manuellt normaliserade filer"
+      }
+    },
     "c103b2fb-9a6b-4b68-8112-b70597a6cd14": {
       "config": {
         "@manager": "linkTaskManagerDirectories",
@@ -10760,7 +10757,7 @@
       "exit_codes": {
         "0": {
           "job_status": "Completed successfully",
-          "link_id": "10c40e41-fb10-48b5-9d01-336cd958afe8"
+          "link_id": "bf0ea0f6-211b-4b34-8f25-8a68145403c8"
         }
       },
       "fallback_job_status": "Failed",

diff --git a/src/dashboard/src/fpr/migrations/0044_remove_fits.py b/src/dashboard/src/fpr/migrations/0044_remove_fits.py
@@ -0,0 +1,12 @@
+from django.db import migrations
+
+
+def data_migration(apps, schema_editor):
+    FPTool = apps.get_model("fpr", "FPTool")
+    FPTool.objects.filter(description="FITS").delete()
+
+
+class Migration(migrations.Migration):
+    dependencies = [("fpr", "0043_update_default_thumbnail_command")]
+
+    operations = [migrations.RunPython(data_migration, migrations.RunPython.noop)]