Add request-scoped processing config

This commit removes support for processingMCP.xml. Instead, users are asked to submit the processing configuration via the API. The workflow had a couple of now unused link types removed and added a new LinkChoice type removing the need of workflow chains. Fixes #76.
artefactual-labs · Oct 27, 2020 · 98f05f1 · 98f05f1
1 parent 0566dcf
commit 98f05f1
Show file tree

Hide file tree

Showing 49 changed files with 1,034 additions and 1,881 deletions.
diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
diff --git a/Makefile b/Makefile
@@ -8,6 +8,7 @@ CURRENT_UID := $(shell id -u)
 shell:  ## Open a shell in a new container.
 	docker-compose run --rm --entrypoint bash a3m
 
+.PHONY: build
 build:  ## Build and recreate containers.
 	env COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker-compose build
 	env COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker-compose up -d --force-recreate

diff --git a/a3m/__init__.py b/a3m/__init__.py
@@ -1,3 +1,3 @@
-__version__ = "0.4.0"
+__version__ = "0.5.0"
 
 __all__ = ["__version__"]
diff --git a/a3m/archivematicaFunctions.py b/a3m/archivematicaFunctions.py
@@ -34,7 +34,7 @@
     "objects",
 )
 
-OPTIONAL_FILES = ("processingMCP.xml", "README.html")
+OPTIONAL_FILES = "README.html"
 
 MANUAL_NORMALIZATION_DIRECTORIES = [
     "objects/manualNormalization/access",
@@ -189,13 +189,6 @@ def format_subdir_path(dir_path, path_prefix_to_repl):
     )
 
 
-def str2bool(val):
-    """'True' is ``True``; aught else is ``False."""
-    if val == "True":
-        return True
-    return False
-
-
 def div_el_to_dir_paths(div_el, parent="", include=True):
     """Recursively extract the list of filesystem directory paths encoded in
     <mets:div> element ``div_el``.

diff --git a/a3m/assets/workflow-schema-v1.json b/a3m/assets/workflow-schema-v1.json
@@ -1,7 +1,7 @@
 {
-  "$id": "https://www.archivematica.org/labs/workflow/schema/v1.json",
+  "$id": "https://a3m.readthedocs.io/workflow/schema/v1.json",
   "$schema": "http://json-schema.org/draft-06/schema#",
-  "title": "Archivematica Workflow JSON Schema",
+  "title": "a3m Workflow JSON Schema",
   "description": "In-progress JSON Schema document for Archivematica JSON-encoded workflows.",
   "definitions": {
     "uuid": {
@@ -28,36 +28,14 @@
         "Failed"
       ]
     },
-    "chain": {
-      "type": "object",
-      "properties": {
-        "link_id": {
-          "$ref": "#/definitions/uuid"
-        },
-        "description": {
-          "$ref": "#/definitions/translations"
-        },
-        "start": {
-          "type": "boolean"
-        }
-      },
-      "additionalProperties": false,
-      "required": [
-        "link_id",
-        "description"
-      ]
-    },
     "link": {
       "type": "object",
       "properties": {
         "config": {
           "oneOf": [
             { "$ref": "#/definitions/link_model_StandardTaskFile" },
             { "$ref": "#/definitions/link_model_StandardTaskDir" },
-            { "$ref": "#/definitions/link_model_ReplacementDic" },
-            { "$ref": "#/definitions/link_model_ChainChoice" },
-            { "$ref": "#/definitions/link_model_GetServiceOutput" },
-            { "$ref": "#/definitions/link_model_GetServiceOutputPrompt" }
+            { "$ref": "#/definitions/link_model_LinkChoice" }
           ]
         },
         "exit_codes": {
@@ -99,6 +77,9 @@
         "group": {
           "$ref": "#/definitions/translations"
         },
+        "start": {
+          "type": "boolean"
+        },
         "end": {
           "type": "boolean"
         }
@@ -113,39 +94,10 @@
         "group"
       ]
     },
-    "watched_dir": {
-      "type": "object",
-      "properties": {
-        "chain_id": {
-          "$ref": "#/definitions/uuid"
-        },
-        "only_dirs": {
-          "type": "boolean"
-        },
-        "path": {
-          "type": "string"
-        },
-        "unit_type": {
-          "type": "string",
-          "enum": [
-            "Transfer",
-            "SIP"
-          ]
-        }
-      },
-      "additionalProperties": false,
-      "required": [
-        "chain_id",
-        "only_dirs",
-        "path",
-        "unit_type"
-      ]
-    },
     "link_model_StandardTaskFile": {
       "type": "object",
       "properties": {
         "@manager": {"type": "string", "pattern": "linkTaskManagerFiles"},
-        "@model": {"type": "string", "pattern": "StandardTaskConfig"},
         "execute": {
           "type": "string"
         },
@@ -171,7 +123,6 @@
       "additionalProperties": false,
       "required": [
         "@manager",
-        "@model",
         "execute",
         "arguments",
         "filter_file_start",
@@ -185,7 +136,6 @@
       "type": "object",
       "properties": {
         "@manager": {"type": "string", "pattern": "linkTaskManagerDirectories"},
-        "@model": {"type": "string", "pattern": "StandardTaskConfig"},
         "execute": {
           "type": "string"
         },
@@ -211,7 +161,6 @@
       "additionalProperties": false,
       "required": [
         "@manager",
-        "@model",
         "execute",
         "arguments",
         "filter_file_start",
@@ -221,153 +170,40 @@
         "stderr_file"
       ]
     },
-    "link_model_ReplacementDic": {
-      "type": "object",
-      "properties": {
-        "@manager": {"type": "string", "pattern": "linkTaskManagerReplacementDicFromChoice"},
-        "@model": {"type": "string", "pattern": "MicroServiceChoiceReplacementDic"},
-        "replacements": {
-          "type": "array",
-          "minItems": 0,
-          "items": [
-            {
-              "type": "object",
-              "properties": {
-                "id": { "$ref": "#/definitions/uuid" },
-                "description": { "$ref": "#/definitions/translations" },
-                "items": {
-                  "type": "object",
-                  "additionalProperties": {
-                    "type": "string"
-                  }
-                }
-              },
-              "additionalProperties": false,
-              "required": [
-                "id",
-                "description",
-                "items"
-              ]
-            }
-          ]
-        }
-      },
-      "additionalProperties": false,
-      "required": [
-        "@manager",
-        "@model",
-        "replacements"
-      ]
-    },
-    "link_model_ChainChoice": {
+    "link_model_LinkChoice": {
       "type": "object",
       "properties": {
         "@manager": {"type": "string", "pattern": "linkTaskManagerChoice"},
-        "@model": {"type": "string", "pattern": "MicroServiceChainChoice"},
-        "chain_choices": {
+        "config_attr": {"type": "string"},
+        "default": {"type": "boolean"},
+        "choices": {
           "type": "array",
           "minItems": 1,
-          "items": [
-            {"$ref": "#/definitions/uuid"}
-          ]
-        }
-      },
-      "additionalProperties": false,
-      "required": [
-        "@manager",
-        "@model",
-        "chain_choices"
-      ]
-    },
-    "link_model_GetServiceOutput": {
-      "type": "object",
-      "properties": {
-        "@manager": {"type": "string", "pattern": "linkTaskManagerGetMicroserviceGeneratedListInStdOut"},
-        "@model": {"type": "string", "pattern": "StandardTaskConfig"},
-        "execute": {
-          "type": "string"
-        },
-        "arguments": {
-          "type": ["string", "null"]
-        },
-        "filter_file_start": {
-          "type": ["string", "null"]
-        },
-        "filter_file_end": {
-          "type": ["string", "null"]
-        },
-        "filter_subdir": {
-          "type": ["string", "null"]
-        },
-        "stdout_file": {
-          "type": ["string", "null"]
-        },
-        "stderr_file": {
-          "type": ["string", "null"]
-        }
-      },
-      "additionalProperties": false,
-      "required": [
-        "@manager",
-        "@model",
-        "execute",
-        "arguments",
-        "filter_file_start",
-        "filter_file_end",
-        "filter_subdir",
-        "stdout_file",
-        "stderr_file"
-      ]
-    },
-    "link_model_GetServiceOutputPrompt": {
-      "type": "object",
-      "properties": {
-        "@manager": {"type": "string", "pattern": "linkTaskManagerGetUserChoiceFromMicroserviceGeneratedList"},
-        "@model": {"type": "string", "pattern": "StandardTaskConfig"},
-        "execute": {
-          "type": "string"
-        },
-        "arguments": {
-          "type": ["string", "null"]
-        },
-        "filter_file_start": {
-          "type": ["string", "null"]
-        },
-        "filter_file_end": {
-          "type": ["string", "null"]
-        },
-        "filter_subdir": {
-          "type": ["string", "null"]
-        },
-        "stdout_file": {
-          "type": ["string", "null"]
-        },
-        "stderr_file": {
-          "type": ["string", "null"]
+          "items": {
+            "type": "object",
+            "properties": {
+              "value": {"type": "boolean"},
+              "link_id": {"$ref": "#/definitions/uuid"}
+            },
+            "additionalProperties": true,
+            "required": [
+              "value",
+              "link_id"
+            ]
+          }
         }
       },
       "additionalProperties": false,
       "required": [
         "@manager",
-        "@model",
-        "execute",
-        "arguments",
-        "filter_file_start",
-        "filter_file_end",
-        "filter_subdir",
-        "stdout_file",
-        "stderr_file"
+        "config_attr",
+        "default",
+        "choices"
       ]
     }
   },
   "type": "object",
   "properties": {
-    "chains": {
-      "type": "object",
-      "additionalProperties": {
-        "$ref": "#/definitions/chain"
-      }
-    },
     "links": {
       "type": "object",
       "additionalProperties": {
@@ -376,7 +212,6 @@
     }
   },
   "required": [
-    "chains",
     "links"
   ],
   "additionalProperties": false