diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml deleted file mode 100644 index d892c7ee3..000000000 --- a/.github/workflows/codeql.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: "Code Scanning - Action" -on: - push: - pull_request: - schedule: - - cron: '30 1 * * 0' -jobs: - CodeQL-Build: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v2 - with: - fetch-depth: 2 - - run: git checkout HEAD^2 - if: ${{ github.event_name == 'pull_request' }} - - name: Initialize CodeQL - uses: github/codeql-action/init@v1 - with: - languages: python - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 diff --git a/Makefile b/Makefile index 9602eb940..2cf031fe1 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,7 @@ CURRENT_UID := $(shell id -u) shell: ## Open a shell in a new container. docker-compose run --rm --entrypoint bash a3m +.PHONY: build build: ## Build and recreate containers. env COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker-compose build env COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker-compose up -d --force-recreate diff --git a/a3m/__init__.py b/a3m/__init__.py index b786da230..696d1692b 100644 --- a/a3m/__init__.py +++ b/a3m/__init__.py @@ -1,3 +1,3 @@ -__version__ = "0.4.0" +__version__ = "0.5.0" __all__ = ["__version__"] diff --git a/a3m/archivematicaFunctions.py b/a3m/archivematicaFunctions.py index b2fdc90ce..30df4d9b8 100644 --- a/a3m/archivematicaFunctions.py +++ b/a3m/archivematicaFunctions.py @@ -34,7 +34,7 @@ "objects", ) -OPTIONAL_FILES = ("processingMCP.xml", "README.html") +OPTIONAL_FILES = "README.html" MANUAL_NORMALIZATION_DIRECTORIES = [ "objects/manualNormalization/access", @@ -189,13 +189,6 @@ def format_subdir_path(dir_path, path_prefix_to_repl): ) -def str2bool(val): - """'True' is ``True``; aught else is ``False.""" - if val == "True": - return True - return False - - def div_el_to_dir_paths(div_el, parent="", include=True): """Recursively extract the list of filesystem directory paths encoded in element ``div_el``. diff --git a/a3m/assets/workflow-schema-v1.json b/a3m/assets/workflow-schema-v1.json index 23359e0cd..5c89487e7 100644 --- a/a3m/assets/workflow-schema-v1.json +++ b/a3m/assets/workflow-schema-v1.json @@ -1,7 +1,7 @@ { - "$id": "https://www.archivematica.org/labs/workflow/schema/v1.json", + "$id": "https://a3m.readthedocs.io/workflow/schema/v1.json", "$schema": "http://json-schema.org/draft-06/schema#", - "title": "Archivematica Workflow JSON Schema", + "title": "a3m Workflow JSON Schema", "description": "In-progress JSON Schema document for Archivematica JSON-encoded workflows.", "definitions": { "uuid": { @@ -28,25 +28,6 @@ "Failed" ] }, - "chain": { - "type": "object", - "properties": { - "link_id": { - "$ref": "#/definitions/uuid" - }, - "description": { - "$ref": "#/definitions/translations" - }, - "start": { - "type": "boolean" - } - }, - "additionalProperties": false, - "required": [ - "link_id", - "description" - ] - }, "link": { "type": "object", "properties": { @@ -54,10 +35,7 @@ "oneOf": [ { "$ref": "#/definitions/link_model_StandardTaskFile" }, { "$ref": "#/definitions/link_model_StandardTaskDir" }, - { "$ref": "#/definitions/link_model_ReplacementDic" }, - { "$ref": "#/definitions/link_model_ChainChoice" }, - { "$ref": "#/definitions/link_model_GetServiceOutput" }, - { "$ref": "#/definitions/link_model_GetServiceOutputPrompt" } + { "$ref": "#/definitions/link_model_LinkChoice" } ] }, "exit_codes": { @@ -99,6 +77,9 @@ "group": { "$ref": "#/definitions/translations" }, + "start": { + "type": "boolean" + }, "end": { "type": "boolean" } @@ -113,39 +94,10 @@ "group" ] }, - "watched_dir": { - "type": "object", - "properties": { - "chain_id": { - "$ref": "#/definitions/uuid" - }, - "only_dirs": { - "type": "boolean" - }, - "path": { - "type": "string" - }, - "unit_type": { - "type": "string", - "enum": [ - "Transfer", - "SIP" - ] - } - }, - "additionalProperties": false, - "required": [ - "chain_id", - "only_dirs", - "path", - "unit_type" - ] - }, "link_model_StandardTaskFile": { "type": "object", "properties": { "@manager": {"type": "string", "pattern": "linkTaskManagerFiles"}, - "@model": {"type": "string", "pattern": "StandardTaskConfig"}, "execute": { "type": "string" }, @@ -171,7 +123,6 @@ "additionalProperties": false, "required": [ "@manager", - "@model", "execute", "arguments", "filter_file_start", @@ -185,7 +136,6 @@ "type": "object", "properties": { "@manager": {"type": "string", "pattern": "linkTaskManagerDirectories"}, - "@model": {"type": "string", "pattern": "StandardTaskConfig"}, "execute": { "type": "string" }, @@ -211,7 +161,6 @@ "additionalProperties": false, "required": [ "@manager", - "@model", "execute", "arguments", "filter_file_start", @@ -221,153 +170,40 @@ "stderr_file" ] }, - "link_model_ReplacementDic": { - "type": "object", - "properties": { - "@manager": {"type": "string", "pattern": "linkTaskManagerReplacementDicFromChoice"}, - "@model": {"type": "string", "pattern": "MicroServiceChoiceReplacementDic"}, - "replacements": { - "type": "array", - "minItems": 0, - "items": [ - { - "type": "object", - "properties": { - "id": { "$ref": "#/definitions/uuid" }, - "description": { "$ref": "#/definitions/translations" }, - "items": { - "type": "object", - "additionalProperties": { - "type": "string" - } - } - }, - "additionalProperties": false, - "required": [ - "id", - "description", - "items" - ] - } - ] - } - }, - "additionalProperties": false, - "required": [ - "@manager", - "@model", - "replacements" - ] - }, - "link_model_ChainChoice": { + "link_model_LinkChoice": { "type": "object", "properties": { "@manager": {"type": "string", "pattern": "linkTaskManagerChoice"}, - "@model": {"type": "string", "pattern": "MicroServiceChainChoice"}, - "chain_choices": { + "config_attr": {"type": "string"}, + "default": {"type": "boolean"}, + "choices": { "type": "array", "minItems": 1, - "items": [ - {"$ref": "#/definitions/uuid"} - ] - } - }, - "additionalProperties": false, - "required": [ - "@manager", - "@model", - "chain_choices" - ] - }, - "link_model_GetServiceOutput": { - "type": "object", - "properties": { - "@manager": {"type": "string", "pattern": "linkTaskManagerGetMicroserviceGeneratedListInStdOut"}, - "@model": {"type": "string", "pattern": "StandardTaskConfig"}, - "execute": { - "type": "string" - }, - "arguments": { - "type": ["string", "null"] - }, - "filter_file_start": { - "type": ["string", "null"] - }, - "filter_file_end": { - "type": ["string", "null"] - }, - "filter_subdir": { - "type": ["string", "null"] - }, - "stdout_file": { - "type": ["string", "null"] - }, - "stderr_file": { - "type": ["string", "null"] - } - }, - "additionalProperties": false, - "required": [ - "@manager", - "@model", - "execute", - "arguments", - "filter_file_start", - "filter_file_end", - "filter_subdir", - "stdout_file", - "stderr_file" - ] - }, - "link_model_GetServiceOutputPrompt": { - "type": "object", - "properties": { - "@manager": {"type": "string", "pattern": "linkTaskManagerGetUserChoiceFromMicroserviceGeneratedList"}, - "@model": {"type": "string", "pattern": "StandardTaskConfig"}, - "execute": { - "type": "string" - }, - "arguments": { - "type": ["string", "null"] - }, - "filter_file_start": { - "type": ["string", "null"] - }, - "filter_file_end": { - "type": ["string", "null"] - }, - "filter_subdir": { - "type": ["string", "null"] - }, - "stdout_file": { - "type": ["string", "null"] - }, - "stderr_file": { - "type": ["string", "null"] + "items": { + "type": "object", + "properties": { + "value": {"type": "boolean"}, + "link_id": {"$ref": "#/definitions/uuid"} + }, + "additionalProperties": true, + "required": [ + "value", + "link_id" + ] + } } }, "additionalProperties": false, "required": [ "@manager", - "@model", - "execute", - "arguments", - "filter_file_start", - "filter_file_end", - "filter_subdir", - "stdout_file", - "stderr_file" + "config_attr", + "default", + "choices" ] } }, "type": "object", "properties": { - "chains": { - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/chain" - } - }, "links": { "type": "object", "additionalProperties": { @@ -376,7 +212,6 @@ } }, "required": [ - "chains", "links" ], "additionalProperties": false diff --git a/a3m/assets/workflow.json b/a3m/assets/workflow.json index cfb8991a0..9d5aa8070 100644 --- a/a3m/assets/workflow.json +++ b/a3m/assets/workflow.json @@ -1,376 +1,35 @@ { - "chains": { - "01d80b27-4ad1-4bd1-8f8d-f819f18bf685": { - "description": { - "en": "Yes", - "es": "Sí", - "fr": "Oui", - "ja": "はい", - "pt_BR": "Sim", - "sv": "Ja" - }, - "link_id": "f19926dd-8fb5-4c79-8ade-c83f61f55b40" - }, - "06f03bb3-121d-4c85-bec7-abbc5320a409": { - "description": { - "en": "Examine contents", - "es": "Examinar contenidos", - "fr": "Examiner le contenu", - "ja": "内容を検査する", - "pt_BR": "Examinar conteúdos", - "sv": "Undersök innehåll" - }, - "link_id": "100a75f4-9d2a-41bf-8dd0-aec811ae1077" - }, - "0a24787c-00e3-4710-b324-90e792bfb484": { - "description": { - "en": "No", - "fr": "Non", - "ja": "いいえ", - "sv": "Nej" - }, - "link_id": "f574b2a0-6e0b-4c74-ac5b-a73ddb9593a0" - }, - "2671aef1-653a-49bf-bc74-82572b64ace9": { - "description": { - "en": "a3m - Start" - }, - "link_id": "8e3e2bf8-a543-43f9-bb2a-c3df01f112df", - "start": true - }, - "29881c21-3548-454a-9637-ebc5fd46aee0": { - "description": { - "en": "Yes", - "es": "Sí", - "fr": "Oui", - "ja": "はい", - "pt_BR": "Sim", - "sv": "Ja" - }, - "link_id": "18c37bff-fce9-4b40-a50a-022ea0386f1a" - }, - "35151db8-3a11-4b49-8865-f6697ef0ac75": { - "description": { - "en": "Yes", - "es": "Sí", - "fr": "Oui", - "ja": "はい", - "pt_BR": "Sim", - "sv": "Ja" - }, - "link_id": "2900f6d8-b64c-4f2a-8f7f-bb60a57394f6" - }, - "3a55f688-eca3-4ebc-a012-4ce68290e7b0": { - "description": { - "en": "Yes", - "es": "Sí", - "fr": "Oui", - "ja": "はい", - "pt_BR": "Sim", - "sv": "Ja" - }, - "link_id": "0fd20984-db3c-492b-a512-eedd74bacc82" - }, - "3e891cc4-39d2-4989-a001-5107a009a223": { - "description": { - "en": "No", - "fr": "Non", - "ja": "いいえ", - "sv": "Nej" - }, - "link_id": "accea2bf-ba74-4a3a-bb97-614775c74459" - }, - "612e3609-ce9a-4df6-a9a3-63d634d2d934": { - "description": { - "en": "Normalize for preservation", - "es": "Normalizar para preservación", - "fr": "Normaliser aux fins de préservation", - "pt_BR": "Normalize para preservação", - "sv": "Normalisera för bevarande" - }, - "link_id": "8ce378a5-1418-4184-bf02-328a06e1d3be" - }, - "65273f18-5b4e-4944-af4f-09be175a88e8": { - "description": { - "en": "No", - "fr": "Non", - "ja": "いいえ", - "sv": "Nej" - }, - "link_id": "ccf8ec5c-3a9a-404a-a7e7-8f567d3b36a0" - }, - "79f1f5af-7694-48a4-b645-e42790bbf870": { - "description": { - "en": "No", - "fr": "Non", - "ja": "いいえ", - "sv": "Nej" - }, - "link_id": "f8ef02c4-f585-4b0d-9b6f-3cef6fbe527f" - }, - "b7ce05f0-9d94-4b3e-86cc-d4b2c6dba546": { - "description": { - "en": "No", - "fr": "Non", - "ja": "いいえ", - "sv": "Nej" - }, - "link_id": "82ee9ad2-2c74-4c7c-853e-e4eaf68fc8b6" - }, - "c611a6ff-dfdb-46d1-b390-f366a6ea6f66": { - "description": { - "en": "Yes", - "es": "Sí", - "fr": "Oui", - "ja": "はい", - "pt_BR": "Sim", - "sv": "Ja" - }, - "link_id": "6c147aeb-20c5-47ce-9f40-7f22683cea1f" - }, - "df54fec1-dae1-4ea6-8d17-a839ee7ac4a7": { - "description": { - "en": "Yes", - "es": "Sí", - "fr": "Oui", - "ja": "はい", - "pt_BR": "Sim", - "sv": "Ja" - }, - "link_id": "4efe00da-6ed0-45dd-89ca-421b78c4b6be" - }, - "e0a39199-c62a-4a2f-98de-e9d1116460a8": { - "description": { - "en": "Skip examine contents", - "fr": "Passer l'examen des contenus", - "pt_BR": "Ignore o exame dos conteúdos", - "sv": "Hoppa över undersök innehåll" - }, - "link_id": "db99ab43-04d7-44ab-89ec-e09d7bbdc39d" - }, - "e8544c5e-9cbb-4b8f-a68b-6d9b4d7f7362": { - "description": { - "en": "Do not normalize", - "es": "No normalizar", - "fr": "Ne pas normaliser ", - "pt_BR": "Não normalize", - "sv": "Normalisera inte" - }, - "link_id": "70f41678-baa5-46e6-a71c-4b6b4d99f4a6" - }, - "e9eaef1e-c2e0-4e3b-b942-bfb537162795": { - "description": { - "en": "No", - "fr": "Non", - "ja": "いいえ", - "sv": "Nej" - }, - "link_id": "2584b25c-8d98-44b7-beca-2b3ea2ea2505" - } - }, "links": { - "01c651cb-c174-4ba4-b985-1d87a44d6754": { - "config": { - "@manager": "linkTaskManagerReplacementDicFromChoice", - "@model": "MicroServiceChoiceReplacementDic", - "replacements": [ - { - "description": { - "en": "1 - fastest compression", - "es": "1 - el modo más rápido", - "fr": "1 - mode rapide", - "ja": "1 - 最高速モード", - "pt_BR": "1 - modo mais rápido", - "sv": "1 - snabbaste läget" - }, - "id": "ecfad581-b007-4612-a0e0-fcc551f4057f", - "items": { - "AIPCompressionLevel": "1" - } - }, - { - "description": { - "en": "3 - fast compression", - "es": "3 - mode de compresión rápido", - "fr": "3 - mode de compression rapide", - "ja": "3 - 高速圧縮モード", - "pt_BR": "3 - modo de compressão rápida", - "sv": "3 - snabbkomprimeringsläge" - }, - "id": "85b2243e-ff97-4ca8-80e8-3c6b0842b360", - "items": { - "AIPCompressionLevel": "3" - } - }, - { - "description": { - "en": "5 - normal compression", - "es": "5 - modo de compresión normal", - "fr": "5 - mode de compression normal", - "ja": "5 - 通常の圧縮モード", - "pt_BR": "5 - modo de compressão normal", - "sv": "5 - normal komprimeringsläge" - }, - "id": "414da421-b83f-4648-895f-a34840e3c3f5", - "items": { - "AIPCompressionLevel": "5" - } - }, - { - "description": { - "en": "7 - maximum compression", - "es": "7 - máxima compresión", - "fr": "7 - compression maximum", - "ja": "7 - 最大圧縮", - "pt_BR": "7 - compressão máxima", - "sv": "7 - maximal komprimering" - }, - "id": "4e31f579-68bd-4be1-a10e-ec5411897121", - "items": { - "AIPCompressionLevel": "7" - } - }, - { - "description": { - "en": "9 - ultra compression", - "es": "9 - compresión ultra", - "fr": "9 - compression ultra", - "ja": "9 - 超圧縮", - "pt_BR": "9 - compressão ultra", - "sv": "9 - ultrakomprimering" - }, - "id": "6d52fd24-8c06-4c8e-997a-e427ba0acc36", - "items": { - "AIPCompressionLevel": "9" - } - } - ] + "00000e0a-2cb2-4292-a412-58ea57aa6f7e": { + "config": { + "@manager": "linkTaskManagerDirectories", + "arguments": "\"%TransferUUID%\" \"%transferDirectory%\" \"%URL%\"", + "execute": "a3m_download_transfer", + "filter_file_end": null, + "filter_file_start": null, + "filter_subdir": null, + "stderr_file": null, + "stdout_file": null }, "description": { - "en": "Select compression level", - "fr": "Sélectionnez le niveau de compression", - "ja": "圧縮レベルの選択", - "pt_BR": "Selecione o nível de compressão", - "sv": "Välj komprimeringsnivå" + "en": "a3m - Download package" }, "exit_codes": { "0": { "job_status": "Completed successfully", - "link_id": "d55b42c8-c7c5-4a40-b626-d248d2bd883f" + "link_id": "50b67418-cb8d-434d-acc9-4a8324e7fdd2" } }, "fallback_job_status": "Failed", - "fallback_link_id": null, + "fallback_link_id": "e780473a-0c10-431f-bab6-5d7238b2b70b", "group": { - "en": "Prepare AIP", - "es": "Preparar AIP", - "fr": "Préparer l'AIP", - "sv": "Förbered AIP" - } - }, - "01d64f58-8295-4b7b-9cab-8f1b153a504f": { - "config": { - "@manager": "linkTaskManagerReplacementDicFromChoice", - "@model": "MicroServiceChoiceReplacementDic", - "replacements": [ - { - "description": { - "en": "7z using bzip2", - "es": "7z usando bzip2", - "fr": "7z utilisant bzip2", - "ja": "7z（bzip2を利用）", - "pt_BR": "7z usando bzip2", - "sv": "7z användande bzip2" - }, - "id": "9475447c-9889-430c-9477-6287a9574c5b", - "items": { - "AIPCompressionAlgorithm": "7z-bzip2" - } - }, - { - "description": { - "en": "7z using lzma", - "es": "7z usando lzma", - "fr": "7z utilisant lzma", - "ja": "7z（lzmaを利用）", - "pt_BR": "7z usando lzma", - "sv": "7z using Izma" - }, - "id": "c96353b9-0d55-46cf-baa0-d7c3e180dd43", - "items": { - "AIPCompressionAlgorithm": "7z-lzma" - } - }, - { - "description": { - "en": "7z without compression", - "pt_BR": "7z sem compressão" - }, - "id": "e3906da0-41fb-4cb8-a598-6c73981b63e9", - "items": { - "AIPCompressionAlgorithm": "7z-copy" - } - }, - { - "description": { - "en": "Gzipped tar" - }, - "id": "9ce711ab-96d1-45dd-8708-de69698c1424", - "items": { - "AIPCompressionAlgorithm": "gzip-tar.gzip" - } - }, - { - "description": { - "en": "Parallel bzip2", - "es": "bzip2 paralelo", - "pt_BR": "Bzip2 paralelo", - "sv": "Parallell bzip2" - }, - "id": "f61b00a1-ef2e-4dc4-9391-111c6f42b9a7", - "items": { - "AIPCompressionAlgorithm": "pbzip2-pbzip2" - } - }, - { - "description": { - "en": "Uncompressed", - "pt_BR": "Sem comprimir", - "sv": "Okomprimerat" - }, - "id": "dc04c4c0-07ea-4796-b643-66d967ed33a4", - "items": { - "AIPCompressionAlgorithm": "None-" - } - } - ] - }, - "description": { - "en": "Select compression algorithm", - "fr": "Sélectionner l'algorithme de compression", - "ja": "圧縮アルゴリズムの選択", - "pt_BR": "Selecione o algoritmo de compressão", - "sv": "Välj komprimeringsalgoritm" - }, - "exit_codes": { - "0": { - "job_status": "Completed successfully", - "link_id": "01c651cb-c174-4ba4-b985-1d87a44d6754" - } + "en": "a3m" }, - "fallback_job_status": "Failed", - "fallback_link_id": null, - "group": { - "en": "Prepare AIP", - "es": "Preparar AIP", - "fr": "Préparer l'AIP", - "sv": "Förbered AIP" - } + "start": true }, "032cdc54-0b9b-4caf-86e8-10d63efbaec0": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "\"%SIPObjectsDirectory%\"", "execute": "check_transfer_directory_for_objects", "filter_file_end": null, @@ -406,7 +65,6 @@ "04493ab2-6cad-400d-8832-06941f121a96": { "config": { "@manager": "linkTaskManagerFiles", - "@model": "StandardTaskConfig", "arguments": "\"%relativeLocation%\" \"%fileUUID%\" \"%SIPUUID%\"", "execute": "characterize_file", "filter_file_end": null, @@ -440,48 +98,20 @@ }, "087d27be-c719-47d8-9bbb-9a7d8b609c44": { "config": { - "@manager": "linkTaskManagerReplacementDicFromChoice", - "@model": "MicroServiceChoiceReplacementDic", - "replacements": [ - { - "description": { - "en": "Yes", - "es": "Sí", - "fr": "Oui", - "ja": "はい", - "pt_BR": "Sim", - "sv": "Ja" - }, - "id": "4dec164b-79b0-4459-8505-8095af9655b5", - "items": { - "IDCommand": "True" - } - }, - { - "description": { - "en": "No", - "fr": "Non", - "ja": "いいえ", - "sv": "Nej" - }, - "id": "782bbf56-e220-48b5-9eb6-6610583f2072", - "items": { - "IDCommand": "False" - } - } + "@manager": "linkTaskManagerChoice", + "config_attr": "identify_submission_and_metadata", + "default": true, + "choices": [ + {"value": true, "link_id": "1dce8e21-7263-4cc4-aa59-968d9793b5f2"}, + {"value": false, "link_id": "33d7ac55-291c-43ae-bb42-f599ef428325"} ] }, "description": { "en": "Do you want to perform file format identification?" }, - "exit_codes": { - "0": { - "job_status": "Completed successfully", - "link_id": "1dce8e21-7263-4cc4-aa59-968d9793b5f2" - } - }, + "exit_codes": {}, "fallback_job_status": "Failed", - "fallback_link_id": "b2ef06b9-bca4-49da-bc5c-866d7b3c4bb1", + "fallback_link_id": null, "group": { "en": "Process submission documentation", "fr": "Traiter la documentation de soumission", @@ -492,7 +122,6 @@ "0a63befa-327d-4655-a021-341b639ee9ed": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%SIPName%-%SIPUUID%\"", "execute": "copy_submission_docs", "filter_file_end": null, @@ -524,41 +153,9 @@ "sv": "Förbered AIP" } }, - "0c96c798-9ace-4c05-b3cf-243cdad796b7": { - "config": { - "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", - "arguments": "\"%sharedPath%processingConfigs/%processingConfiguration%.xml\" \"%SIPDirectory%processingMCP.xml\" -n", - "execute": "cmd_cp", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null - }, - "description": { - "en": "Include default Transfer processingMCP.xml", - "pt_BR": "Incluir transferência padrão processingMCP.xml", - "sv": "Inkludera standadard överförings processing.xml" - }, - "exit_codes": { - "0": { - "job_status": "Completed successfully", - "link_id": "bd899573-694e-4d33-8c9b-df0af802437d" - } - }, - "fallback_job_status": "Failed", - "fallback_link_id": "e780473a-0c10-431f-bab6-5d7238b2b70b", - "group": { - "en": "Include default Transfer processingMCP.xml", - "pt_BR": "Incluir transferência padrão processingMCP.xml", - "sv": "Inkludera standadard överförings processing.xml" - } - }, "0fd20984-db3c-492b-a512-eedd74bacc82": { "config": { "@manager": "linkTaskManagerFiles", - "@model": "StandardTaskConfig", "arguments": "\"%relativeLocation%\" \"%fileUUID%\" \"%SIPUUID%\" \"%sharedPath%\" \"preservation\"", "execute": "policy_check", "filter_file_end": null, @@ -595,7 +192,6 @@ "100a75f4-9d2a-41bf-8dd0-aec811ae1077": { "config": { "@manager": "linkTaskManagerFiles", - "@model": "StandardTaskConfig", "arguments": "\"%relativeLocation%\" \"%SIPDirectory%\" \"%fileUUID%\"", "execute": "examine_contents", "filter_file_end": null, @@ -632,7 +228,6 @@ "11033dbd-e4d4-4dd6-8bcf-48c424e222e3": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%objects/submissionDocumentation/\" \"%SIPUUID%\" \"%date%\" \"%taskUUID%\" \"SIPDirectory\" \"sip_id\" \"%SIPDirectory%\"", "execute": "sanitize_object_names", "filter_file_end": null, @@ -664,7 +259,6 @@ "14d86bf1-f7a9-4d3e-9eda-f8b1f5583ea7": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "\"%SIPUUID%\" \"%SIPDirectory%%AIPFilename%\"", "execute": "a3m_store_aip", "filter_file_end": null, @@ -696,10 +290,11 @@ "153c5f41-3cfb-47ba-9150-2dd44ebc27df": { "config": { "@manager": "linkTaskManagerChoice", - "@model": "MicroServiceChainChoice", - "chain_choices": [ - "3a55f688-eca3-4ebc-a012-4ce68290e7b0", - "b7ce05f0-9d94-4b3e-86cc-d4b2c6dba546" + "config_attr": "perform_policy_checks_on_preservation_derivatives", + "default": true, + "choices": [ + {"value": true, "link_id": "0fd20984-db3c-492b-a512-eedd74bacc82"}, + {"value": false, "link_id": "82ee9ad2-2c74-4c7c-853e-e4eaf68fc8b6"} ] }, "description": { @@ -721,7 +316,6 @@ "15a2df8a-7b45-4c11-b6fa-884c9b7e5c67": { "config": { "@manager": "linkTaskManagerFiles", - "@model": "StandardTaskConfig", "arguments": "\"%fileUUID%\"", "execute": "manual_normalization_identify_files_included", "filter_file_end": null, @@ -756,7 +350,6 @@ "173d310c-8e40-4669-9a69-6d4c8ffd0396": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%metadata/submissionDocumentation\" \"%SIPDirectory%objects/submissionDocumentation\"", "execute": "move_or_merge", "filter_file_end": null, @@ -789,7 +382,6 @@ "18c37bff-fce9-4b40-a50a-022ea0386f1a": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "--amdSec --baseDirectoryPath \"%SIPDirectory%\" --baseDirectoryPathString \"SIPDirectory\" --fileGroupIdentifier \"%SIPUUID%\" --fileGroupType \"sip_id\" --xmlFile \"%SIPDirectory%METS.%SIPUUID%.xml\" --createNormativeStructmap", "execute": "create_mets_v2", "filter_file_end": null, @@ -825,7 +417,6 @@ "1a136608-ae7b-42b4-bf2f-de0e514cfd47": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "\"%TransferUUID%\" \"%SIPDirectory%metadata/rights.csv\"", "execute": "rights_from_csv", "filter_file_end": null, @@ -859,7 +450,6 @@ "1b1a4565-b501-407b-b40f-2f20889423f1": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "\"%TransferUUID%\" \"%SIPDirectory%metadata/file_labels.csv\"", "execute": "load_labels_from_csv", "filter_file_end": null, @@ -893,7 +483,6 @@ "1ba589db-88d1-48cf-bb1a-a5f9d2b17378": { "config": { "@manager": "linkTaskManagerFiles", - "@model": "StandardTaskConfig", "arguments": "\"%fileUUID%\" \"%relativeLocation%\" \"%date%\" \"%taskUUID%\"", "execute": "virus_scan", "filter_file_end": null, @@ -926,7 +515,6 @@ "1c2550f1-3fc0-45d8-8bc4-4c06d720283b": { "config": { "@manager": "linkTaskManagerFiles", - "@model": "StandardTaskConfig", "arguments": "\"%fileUUID%\" \"%relativeLocation%\" \"%date%\" \"%taskUUID%\"", "execute": "virus_scan", "filter_file_end": null, @@ -961,8 +549,7 @@ "1cb7e228-6e94-4c93-bf70-430af99b9264": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", - "arguments": "\"%TransferUUID%\" \"%transferDirectory%\" \"%date%\" \"%taskUUID%\" \"%DeletePackage%\"", + "arguments": "\"%TransferUUID%\" \"%transferDirectory%\" \"%date%\" \"%taskUUID%\" \"%config:delete_packages_after_extraction%\"", "execute": "extract_contents", "filter_file_end": null, "filter_file_start": null, @@ -997,7 +584,6 @@ "1cd3b36a-5252-4a69-9b1c-3b36829288ab": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "--SIPDirectory \"%SIPDirectory%\" --serviceDirectory \"objects/service/\" --objectsDirectory \"objects/\" --SIPUUID \"%SIPUUID%\" --date \"%date%\"", "execute": "check_for_service_directory", "filter_file_end": null, @@ -1033,8 +619,7 @@ "1dce8e21-7263-4cc4-aa59-968d9793b5f2": { "config": { "@manager": "linkTaskManagerFiles", - "@model": "StandardTaskConfig", - "arguments": "\"%IDCommand%\" \"%relativeLocation%\" \"%fileUUID%\"", + "arguments": "\"%relativeLocation%\" \"%fileUUID%\"", "execute": "identify_file_format", "filter_file_end": null, "filter_file_start": null, @@ -1067,7 +652,6 @@ "208d441b-6938-44f9-b54a-bd73f05bc764": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\"", "execute": "verify_sip_compliance", "filter_file_end": null, @@ -1086,7 +670,7 @@ "exit_codes": { "0": { "job_status": "Completed successfully", - "link_id": "df02cac1-f582-4a86-b7cf-da98a58e279e" + "link_id": "f3be1ee1-8881-465d-80a6-a6f093d40ec2" } }, "fallback_job_status": "Failed", @@ -1102,8 +686,7 @@ "2522d680-c7d9-4d06-8b11-a28d8bd8a71f": { "config": { "@manager": "linkTaskManagerFiles", - "@model": "StandardTaskConfig", - "arguments": "\"%IDCommand%\" \"%relativeLocation%\" \"%fileUUID%\" --disable-reidentify", + "arguments": "\"%relativeLocation%\" \"%fileUUID%\" --disable-reidentify", "execute": "identify_file_format", "filter_file_end": null, "filter_file_start": null, @@ -1137,7 +720,6 @@ "2584b25c-8d98-44b7-beca-2b3ea2ea2505": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "\"%SIPObjectsDirectory%\" \"%TransferUUID%\" \"%date%\" \"%taskUUID%\" \"transferDirectory\" \"transfer_id\" \"%SIPDirectory%\"", "execute": "sanitize_object_names", "filter_file_end": null, @@ -1170,7 +752,6 @@ "2900f6d8-b64c-4f2a-8f7f-bb60a57394f6": { "config": { "@manager": "linkTaskManagerFiles", - "@model": "StandardTaskConfig", "arguments": "\"%taskUUID%\" \"%fileUUID%\"", "execute": "transcribe_file", "filter_file_end": null, @@ -1203,7 +784,6 @@ "2a62f025-83ec-4f23-adb4-11d5da7ad8c2": { "config": { "@manager": "linkTaskManagerFiles", - "@model": "StandardTaskConfig", "arguments": "--filePath \"%relativeLocation%\" --fileUUID \"%fileUUID%\" --eventIdentifierUUID \"%taskUUID%\" --date \"%date%\"", "execute": "update_size_and_checksum", "filter_file_end": null, @@ -1238,8 +818,7 @@ "2dd53959-8106-457d-a385-fee57fc93aa9": { "config": { "@manager": "linkTaskManagerFiles", - "@model": "StandardTaskConfig", - "arguments": "\"%IDCommand%\" \"%relativeLocation%\" \"%fileUUID%\" --disable-reidentify", + "arguments": "\"%relativeLocation%\" \"%fileUUID%\" --disable-reidentify", "execute": "identify_file_format", "filter_file_end": null, "filter_file_start": null, @@ -1273,7 +852,6 @@ "303a65f6-a16f-4a06-807b-cb3425a30201": { "config": { "@manager": "linkTaskManagerFiles", - "@model": "StandardTaskConfig", "arguments": "\"%relativeLocation%\" \"%fileUUID%\" \"%TransferUUID%\"", "execute": "characterize_file", "filter_file_end": null, @@ -1310,7 +888,6 @@ "307edcde-ad10-401c-92c4-652917c993ed": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "--sipUUID \"%TransferUUID%\" --basePath \"%SIPDirectory%\" --xmlFile \"%SIPDirectory%\"metadata/submissionDocumentation/METS.xml --basePathString \"transferDirectory\"", "execute": "create_transfer_mets", "filter_file_end": null, @@ -1347,7 +924,6 @@ "33d7ac55-291c-43ae-bb42-f599ef428325": { "config": { "@manager": "linkTaskManagerFiles", - "@model": "StandardTaskConfig", "arguments": "\"%relativeLocation%\" \"%fileUUID%\" \"%SIPUUID%\"", "execute": "characterize_file", "filter_file_end": null, @@ -1382,7 +958,6 @@ "370aca94-65ab-4f2a-9d7d-294a62c8b7ba": { "config": { "@manager": "linkTaskManagerFiles", - "@model": "StandardTaskConfig", "arguments": "--filePath \"%relativeLocation%\" --fileUUID \"%fileUUID%\" --eventIdentifierUUID \"%taskUUID%\" --date \"%date%\"", "execute": "update_size_and_checksum", "filter_file_end": null, @@ -1419,7 +994,6 @@ "377f8ebb-7989-4a68-9361-658079ff8138": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%sharedPath%failed/.\" \"%TransferUUID%\" \"%sharedPath%\"", "execute": "move_transfer", "filter_file_end": null, @@ -1454,7 +1028,6 @@ "39a128e3-c35d-40b7-9363-87f75091e1ff": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "\"%TransferUUID%\" \"%SIPUUID%\"", "execute": "create_sip_from_transfer_objects", "filter_file_end": null, @@ -1491,7 +1064,6 @@ "3e25bda6-5314-4bb4-aa1e-90900dce887d": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%%SIPName%-%SIPUUID%\" \"%SIPDirectory%\" \"%SIPUUID%\"", "execute": "bag_with_empty_directories", "filter_file_end": null, @@ -1509,7 +1081,7 @@ "exit_codes": { "0": { "job_status": "Completed successfully", - "link_id": "01d64f58-8295-4b7b-9cab-8f1b153a504f" + "link_id": "d55b42c8-c7c5-4a40-b626-d248d2bd883f" } }, "fallback_job_status": "Failed", @@ -1524,7 +1096,6 @@ "3f543585-fa4f-4099-9153-dd6d53572f5c": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "\"%SIPUUID%\" \"%SIPDirectory%%AIPFilename%\"", "execute": "verify_aip", "filter_file_end": null, @@ -1560,7 +1131,6 @@ "438dc1cf-9813-44b5-a0a3-58e09ae73b8a": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\"", "execute": "verify_transfer_compliance", "filter_file_end": null, @@ -1593,7 +1163,6 @@ "47dd6ea6-1ee7-4462-8b84-3fc4c1eeeb7f": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%metadata/submissionDocumentation\"", "execute": "check_for_submission_documentation", "filter_file_end": null, @@ -1627,7 +1196,6 @@ "4edfe7e4-82ff-4c0a-ba5f-29f1ee14e17a": { "config": { "@manager": "linkTaskManagerFiles", - "@model": "StandardTaskConfig", "arguments": "--sipUUID \"%SIPUUID%\" --sipDirectory \"%SIPDirectory%\" --filePath \"%relativeLocation%\" --fileUUID \"%fileUUID%\" --eventIdentifierUUID \"%taskUUID%\" --date \"%date%\" --use \"submissionDocumentation\"", "execute": "assign_file_uuids", "filter_file_end": null, @@ -1662,7 +1230,6 @@ "4efe00da-6ed0-45dd-89ca-421b78c4b6be": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "\"%SIPObjectsDirectory%\" -o \"%SIPDirectory%metadata/directory_tree.txt\"", "execute": "cmd_tree", "filter_file_end": null, @@ -1696,7 +1263,6 @@ "50b67418-cb8d-434d-acc9-4a8324e7fdd2": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\"", "execute": "remove_hidden_files_and_directories", "filter_file_end": null, @@ -1728,7 +1294,6 @@ "523c97cc-b267-4cfb-8209-d99e523bf4b3": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "\"%clientAssetsDirectory%README/README.html\" \"%SIPDirectory%README.html\"", "execute": "cmd_cp", "filter_file_end": null, @@ -1761,10 +1326,11 @@ "56eebd45-5600-4768-a8c2-ec0114555a3d": { "config": { "@manager": "linkTaskManagerChoice", - "@model": "MicroServiceChainChoice", - "chain_choices": [ - "df54fec1-dae1-4ea6-8d17-a839ee7ac4a7", - "e9eaef1e-c2e0-4e3b-b942-bfb537162795" + "config_attr": "generate_transfer_structure_report", + "default": true, + "choices": [ + {"value": true, "link_id": "4efe00da-6ed0-45dd-89ca-421b78c4b6be"}, + {"value": false, "link_id": "2584b25c-8d98-44b7-beca-2b3ea2ea2505"} ] }, "description": { @@ -1788,7 +1354,6 @@ "576f1f43-a130-4c15-abeb-c272ec458d33": { "config": { "@manager": "linkTaskManagerFiles", - "@model": "StandardTaskConfig", "arguments": "--fileUUID \"%fileUUID%\" --inputFile \"%relativeLocation%\" --sipDirectory \"%SIPDirectory%\"", "execute": "remove_files_without_premis_metadata", "filter_file_end": null, @@ -1820,7 +1385,6 @@ "5b0042a2-2244-475c-85d5-41e4b11e65d6": { "config": { "@manager": "linkTaskManagerFiles", - "@model": "StandardTaskConfig", "arguments": "\"%relativeLocation%\" \"%fileUUID%\" \"%SIPUUID%\" \"%sharedPath%\" \"preservation\"", "execute": "validate_file", "filter_file_end": null, @@ -1859,7 +1423,6 @@ "5d780c7d-39d0-4f4a-922b-9d1b0d217bca": { "config": { "@manager": "linkTaskManagerFiles", - "@model": "StandardTaskConfig", "arguments": "\"%relativeLocation%\" \"%fileUUID%\"", "execute": "remove_unneeded_files", "filter_file_end": null, @@ -1891,7 +1454,6 @@ "5fbc344c-19c8-48be-a753-02dac987428c": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "775 \"%SIPDirectory%%AIPFilename%\"", "execute": "cmd_chmod", "filter_file_end": null, @@ -1924,7 +1486,6 @@ "63f35161-ba76-4a43-8cfa-c38c6a2d5b2f": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "\"%SIPLogsDirectory%\" \"%SIPObjectsDirectory%\"", "execute": "remove_directories", "filter_file_end": null, @@ -1956,8 +1517,7 @@ "6441980c-b64b-447e-abc7-9351a2547f6a": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", - "arguments": "\"%SIPDirectory%\" \"%TransferUUID%\" --include-dirs \"%AssignUUIDsToDirectories%\"", + "arguments": "\"%SIPDirectory%\" \"%TransferUUID%\"", "execute": "assign_uuids_to_directories", "filter_file_end": null, "filter_file_start": null, @@ -1995,7 +1555,6 @@ "6c147aeb-20c5-47ce-9f40-7f22683cea1f": { "config": { "@manager": "linkTaskManagerFiles", - "@model": "StandardTaskConfig", "arguments": "\"%relativeLocation%\" \"%fileUUID%\" \"%TransferUUID%\" \"%sharedPath%\" \"original\"", "execute": "policy_check", "filter_file_end": null, @@ -2033,7 +1592,6 @@ "70f41678-baa5-46e6-a71c-4b6b4d99f4a6": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%processingDirectory%.\" \"%SIPUUID%\" \"%sharedPath%\"", "execute": "move_sip", "filter_file_end": null, @@ -2066,10 +1624,11 @@ "70fc7040-d4fb-4d19-a0e6-792387ca1006": { "config": { "@manager": "linkTaskManagerChoice", - "@model": "MicroServiceChainChoice", - "chain_choices": [ - "c611a6ff-dfdb-46d1-b390-f366a6ea6f66", - "3e891cc4-39d2-4989-a001-5107a009a223" + "config_attr": "perform_policy_checks_on_originals", + "default": true, + "choices": [ + {"value": true, "link_id": "6c147aeb-20c5-47ce-9f40-7f22683cea1f"}, + {"value": false, "link_id": "accea2bf-ba74-4a3a-bb97-614775c74459"} ] }, "description": { @@ -2092,7 +1651,6 @@ "746b1f47-2dad-427b-8915-8b0cb7acccd8": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%%SIPName%-%SIPUUID%\" \"%SIPLogsDirectory%\" \"%SIPObjectsDirectory%\"", "execute": "remove_directories", "filter_file_end": null, @@ -2124,10 +1682,11 @@ "7509e7dc-1e1b-4dce-8d21-e130515fce73": { "config": { "@manager": "linkTaskManagerChoice", - "@model": "MicroServiceChainChoice", - "chain_choices": [ - "612e3609-ce9a-4df6-a9a3-63d634d2d934", - "e8544c5e-9cbb-4b8f-a68b-6d9b4d7f7362" + "config_attr": "normalize", + "default": true, + "choices": [ + {"value": true, "link_id": "8ce378a5-1418-4184-bf02-328a06e1d3be"}, + {"value": false, "link_id": "70f41678-baa5-46e6-a71c-4b6b4d99f4a6"} ] }, "description": { @@ -2151,7 +1710,6 @@ "75fb5d67-5efa-4232-b00b-d85236de0d3f": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\"", "execute": "manual_normalization_remove_mn_directories", "filter_file_end": null, @@ -2183,7 +1741,6 @@ "78b7adff-861d-4450-b6dd-3fabe96a849e": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\"", "execute": "manual_normalization_check_for_manual_normalization_directory", "filter_file_end": null, @@ -2220,46 +1777,20 @@ }, "7a024896-c4f7-4808-a240-44c87c762bc5": { "config": { - "@manager": "linkTaskManagerReplacementDicFromChoice", - "@model": "MicroServiceChoiceReplacementDic", - "replacements": [ - { - "description": { - "en": "Yes", - "es": "Sí", - "fr": "Oui", - "ja": "はい", - "pt_BR": "Sim", - "sv": "Ja" - }, - "id": "5b3c8268-5b33-4b70-b1aa-0e4540fe03d1", - "items": { - "IDCommand": "True" - } - }, - { - "description": { - "en": "No, use existing data", - "es": "No, usa los datos existentes" - }, - "id": "3c1faec7-7e1e-4cdd-b3bd-e2f05f4baa9b", - "items": { - "IDCommand": "False" - } - } + "@manager": "linkTaskManagerChoice", + "config_attr": "identify_before_normalization", + "default": true, + "choices": [ + {"value": true, "link_id": "2dd53959-8106-457d-a385-fee57fc93aa9"}, + {"value": false, "link_id": "7509e7dc-1e1b-4dce-8d21-e130515fce73"} ] }, "description": { - "en": "Do you want to perform file format identification?" - }, - "exit_codes": { - "0": { - "job_status": "Completed successfully", - "link_id": "2dd53959-8106-457d-a385-fee57fc93aa9" - } + "en": "Do you want to perform file format identification (normalization)?" }, + "exit_codes": {}, "fallback_job_status": "Failed", - "fallback_link_id": "2dd53959-8106-457d-a385-fee57fc93aa9", + "fallback_link_id": null, "group": { "en": "Normalize", "es": "Normalizar", @@ -2271,7 +1802,6 @@ "7d33f228-0fa8-4f4c-a66b-24f8e264c214": { "config": { "@manager": "linkTaskManagerFiles", - "@model": "StandardTaskConfig", "arguments": "\"%fileUUID%\" \"%relativeLocation%\" \"%date%\" \"%taskUUID%\"", "execute": "virus_scan", "filter_file_end": null, @@ -2305,7 +1835,6 @@ "828528c2-2eb9-4514-b5ca-dfd1f7cb5b8c": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%sharedPath%failed/.\" \"%SIPUUID%\" \"%sharedPath%\"", "execute": "move_sip", "filter_file_end": null, @@ -2340,10 +1869,11 @@ "82ee9ad2-2c74-4c7c-853e-e4eaf68fc8b6": { "config": { "@manager": "linkTaskManagerChoice", - "@model": "MicroServiceChainChoice", - "chain_choices": [ - "35151db8-3a11-4b49-8865-f6697ef0ac75", - "0a24787c-00e3-4710-b324-90e792bfb484" + "config_attr": "transcribe_files", + "default": true, + "choices": [ + {"value": true, "link_id": "2900f6d8-b64c-4f2a-8f7f-bb60a57394f6"}, + {"value": false, "link_id": "f574b2a0-6e0b-4c74-ac5b-a73ddb9593a0"} ] }, "description": { @@ -2362,7 +1892,6 @@ "8bc92801-4308-4e3b-885b-1a89fdcd3014": { "config": { "@manager": "linkTaskManagerFiles", - "@model": "StandardTaskConfig", "arguments": "\"%fileUUID%\" \"%relativeLocation%\" \"%date%\" \"%taskUUID%\"", "execute": "virus_scan", "filter_file_end": null, @@ -2396,7 +1925,6 @@ "8c8bac29-4102-4fd2-9d0a-a3bd2e607566": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "\"%TransferUUID%\" \"%SIPDirectory%metadata/metadata.json\"", "execute": "json_metadata_to_csv", "filter_file_end": null, @@ -2430,7 +1958,6 @@ "8ce378a5-1418-4184-bf02-328a06e1d3be": { "config": { "@manager": "linkTaskManagerFiles", - "@model": "StandardTaskConfig", "arguments": "\"%fileUUID%\" \"%relativeLocation%\" \"%SIPDirectory%\" \"%SIPUUID%\" \"%taskUUID%\" \"original\"", "execute": "normalize", "filter_file_end": null, @@ -2470,37 +1997,9 @@ "sv": "Normalisera" } }, - "8e3e2bf8-a543-43f9-bb2a-c3df01f112df": { - "config": { - "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", - "arguments": "\"%TransferUUID%\" \"%transferDirectory%\" \"%URL%\"", - "execute": "a3m_download_transfer", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null - }, - "description": { - "en": "a3m - Download package" - }, - "exit_codes": { - "0": { - "job_status": "Completed successfully", - "link_id": "50b67418-cb8d-434d-acc9-4a8324e7fdd2" - } - }, - "fallback_job_status": "Failed", - "fallback_link_id": "e780473a-0c10-431f-bab6-5d7238b2b70b", - "group": { - "en": "a3m" - } - }, "91ca6f1f-feb5-485d-99d2-25eed195e330": { "config": { "@manager": "linkTaskManagerFiles", - "@model": "StandardTaskConfig", "arguments": "\"%SIPUUID%\" \"%SIPName%\" \"%SIPDirectory%\" \"%fileUUID%\" \"%relativeLocation%\" \"%date%\"", "execute": "manual_normalization_create_metadata_and_restructure", "filter_file_end": null, @@ -2532,7 +2031,6 @@ "91dc1ab1-487e-4121-a6c5-d8441da7a422": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "-d \"%SIPDirectory%%AIPFilename%\"", "execute": "cmd_test", "filter_file_end": null, @@ -2570,7 +2068,6 @@ "970b7d17-7a6b-4d51-808b-c94b78c0d97f": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "\"%SIPUUID%\" \"%relativeLocation%metadata/dc.json\"", "execute": "load_dublin_core", "filter_file_end": null, @@ -2605,7 +2102,6 @@ "9e9b522a-77ab-4c17-ab08-5a4256f49d59": { "config": { "@manager": "linkTaskManagerFiles", - "@model": "StandardTaskConfig", "arguments": "--sipUUID \"%SIPUUID%\" --sipDirectory \"%SIPDirectory%\" --filePath \"%relativeLocation%\" --fileUUID \"%fileUUID%\" --eventIdentifierUUID \"%taskUUID%\" --date \"%date%\" --use \"preservation\"", "execute": "assign_file_uuids", "filter_file_end": null, @@ -2639,7 +2135,6 @@ "a536828c-be65-4088-80bd-eb511a0a063d": { "config": { "@manager": "linkTaskManagerFiles", - "@model": "StandardTaskConfig", "arguments": "\"%relativeLocation%\" \"%fileUUID%\" \"%TransferUUID%\"", "execute": "validate_file", "filter_file_end": null, @@ -2674,8 +2169,7 @@ "aaa929e4-5c35-447e-816a-033a66b9b90b": { "config": { "@manager": "linkTaskManagerFiles", - "@model": "StandardTaskConfig", - "arguments": "\"%IDCommand%\" \"%relativeLocation%\" \"%fileUUID%\" --disable-reidentify", + "arguments": "\"%relativeLocation%\" \"%fileUUID%\" --disable-reidentify", "execute": "identify_file_format", "filter_file_end": null, "filter_file_start": null, @@ -2710,10 +2204,11 @@ "accea2bf-ba74-4a3a-bb97-614775c74459": { "config": { "@manager": "linkTaskManagerChoice", - "@model": "MicroServiceChainChoice", - "chain_choices": [ - "06f03bb3-121d-4c85-bec7-abbc5320a409", - "e0a39199-c62a-4a2f-98de-e9d1116460a8" + "config_attr": "examine_contents", + "default": true, + "choices": [ + {"value": true, "link_id": "100a75f4-9d2a-41bf-8dd0-aec811ae1077"}, + {"value": false, "link_id": "db99ab43-04d7-44ab-89ec-e09d7bbdc39d"} ] }, "description": { @@ -2739,7 +2234,6 @@ "b0ffcd90-eb26-4caf-8fab-58572d205f04": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "\"%SIPUUID%\" \"%SIPDirectory%metadata/metadata.json\"", "execute": "json_metadata_to_csv", "filter_file_end": null, @@ -2772,7 +2266,6 @@ "b21018df-f67d-469a-9ceb-ac92ac68654e": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%objects/metadata/\" \"%SIPUUID%\" \"%date%\" \"%taskUUID%\" \"SIPDirectory\" \"sip_id\" \"%SIPDirectory%\"", "execute": "sanitize_object_names", "filter_file_end": null, @@ -2804,8 +2297,7 @@ "b2444a6e-c626-4487-9abc-1556dd89a8ae": { "config": { "@manager": "linkTaskManagerFiles", - "@model": "StandardTaskConfig", - "arguments": "\"%IDCommand%\" \"%relativeLocation%\" \"%fileUUID%\"", + "arguments": "\"%relativeLocation%\" \"%fileUUID%\"", "execute": "identify_file_format", "filter_file_end": null, "filter_file_start": null, @@ -2838,7 +2330,6 @@ "b2ef06b9-bca4-49da-bc5c-866d7b3c4bb1": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "\"fail\" \"%SIPUUID%\"", "execute": "failed_sip_cleanup", "filter_file_end": null, @@ -2875,7 +2366,6 @@ "b6b0fe37-aa26-40bd-8be8-d3acebf3ccf8": { "config": { "@manager": "linkTaskManagerFiles", - "@model": "StandardTaskConfig", "arguments": "--filePath \"%relativeLocation%\" --fileUUID \"%fileUUID%\" --eventIdentifierUUID \"%taskUUID%\" --date \"%date%\"", "execute": "update_size_and_checksum", "filter_file_end": null, @@ -2910,7 +2400,6 @@ "b944ec7f-7f99-491f-986d-58914c9bb4fa": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "\"%TransferUUID%\"", "execute": "has_packages", "filter_file_end": null, @@ -2950,7 +2439,6 @@ "bd792750-a55b-42e9-903a-8c898bb77df1": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "\"%TransferUUID%\"", "execute": "has_packages", "filter_file_end": null, @@ -2989,35 +2477,12 @@ }, "bd899573-694e-4d33-8c9b-df0af802437d": { "config": { - "@manager": "linkTaskManagerReplacementDicFromChoice", - "@model": "MicroServiceChoiceReplacementDic", - "replacements": [ - { - "description": { - "en": "Yes", - "es": "Sí", - "fr": "Oui", - "ja": "はい", - "pt_BR": "Sim", - "sv": "Ja" - }, - "id": "2dc3f487-e4b0-4e07-a4b3-6216ed24ca14", - "items": { - "AssignUUIDsToDirectories": "True" - } - }, - { - "description": { - "en": "No", - "fr": "Non", - "ja": "いいえ", - "sv": "Nej" - }, - "id": "891f60d0-1ba8-48d3-b39e-dd0934635d29", - "items": { - "AssignUUIDsToDirectories": "False" - } - } + "@manager": "linkTaskManagerChoice", + "config_attr": "assign_uuids_to_directories", + "default": true, + "choices": [ + {"value": true, "link_id": "6441980c-b64b-447e-abc7-9351a2547f6a"}, + {"value": false, "link_id": "dc144ff4-ad74-4a6e-ac15-b0beedcaf662"} ] }, "description": { @@ -3026,18 +2491,9 @@ "pt_BR": "Atribuir UUIDs aos diretórios?", "sv": "Tilldela mappar UUID:er?" }, - "exit_codes": { - "0": { - "job_status": "Completed successfully", - "link_id": "6441980c-b64b-447e-abc7-9351a2547f6a" - }, - "1": { - "job_status": "Failed", - "link_id": "6441980c-b64b-447e-abc7-9351a2547f6a" - } - }, + "exit_codes": {}, "fallback_job_status": "Failed", - "fallback_link_id": "6441980c-b64b-447e-abc7-9351a2547f6a", + "fallback_link_id": null, "group": { "en": "Assign file UUIDs and checksums", "es": "Asignar UUIDs y sumas de verificación a los ficheros", @@ -3050,7 +2506,6 @@ "bdce640d-6e94-49fe-9300-3192a7e5edac": { "config": { "@manager": "linkTaskManagerFiles", - "@model": "StandardTaskConfig", "arguments": "\"%relativeLocation%\" \"%fileUUID%\"", "execute": "remove_unneeded_files", "filter_file_end": null, @@ -3084,7 +2539,6 @@ "c5ecb5a9-d697-4188-844f-9a756d8734fa": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "\"%SIPObjectsDirectory%\" \"%TransferUUID%\" \"%date%\" \"%taskUUID%\" \"transferDirectory\" \"transfer_id\" \"%SIPDirectory%\"", "execute": "sanitize_object_names", "filter_file_end": null, @@ -3118,7 +2572,6 @@ "ccf8ec5c-3a9a-404a-a7e7-8f567d3b36a0": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "--amdSec --baseDirectoryPath \"%SIPDirectory%\" --baseDirectoryPathString \"SIPDirectory\" --fileGroupIdentifier \"%SIPUUID%\" --fileGroupType \"sip_id\" --xmlFile \"%SIPDirectory%METS.%SIPUUID%.xml\"", "execute": "create_mets_v2", "filter_file_end": null, @@ -3154,7 +2607,6 @@ "d0c463c2-da4c-4a70-accb-c4ce96ac5194": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\" \"%clientAssetsDirectory%mets/mets.xsd\"", "execute": "verify_mets", "filter_file_end": null, @@ -3172,7 +2624,7 @@ "exit_codes": { "0": { "job_status": "Completed successfully", - "link_id": "0c96c798-9ace-4c05-b3cf-243cdad796b7" + "link_id": "bd899573-694e-4d33-8c9b-df0af802437d" } }, "fallback_job_status": "Failed", @@ -3187,10 +2639,11 @@ "d0dfa5fc-e3c2-4638-9eda-f96eea1070e0": { "config": { "@manager": "linkTaskManagerChoice", - "@model": "MicroServiceChainChoice", - "chain_choices": [ - "29881c21-3548-454a-9637-ebc5fd46aee0", - "65273f18-5b4e-4944-af4f-09be175a88e8" + "config_attr": "document_empty_directories", + "default": true, + "choices": [ + {"value": true, "link_id": "18c37bff-fce9-4b40-a50a-022ea0386f1a"}, + {"value": false, "link_id": "ccf8ec5c-3a9a-404a-a7e7-8f567d3b36a0"} ] }, "description": { @@ -3211,7 +2664,6 @@ "d51a7ed8-9cf4-424b-9671-85fd8b5b95aa": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "\"%TransferUUID%\" \"%clientAssetsDirectory%premis/premis.xsd\" \"%SIPDirectory%metadata/premis.xml\"", "execute": "load_premis_events_from_xml", "filter_file_end": null, @@ -3243,8 +2695,7 @@ "d55b42c8-c7c5-4a40-b626-d248d2bd883f": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", - "arguments": "\"%AIPCompressionAlgorithm%\" \"%AIPCompressionLevel%\" \"%SIPDirectory%\" \"%SIPName%\" \"%SIPUUID%\"", + "arguments": "\"%config:aip_compression_algorithm%\" \"%config:aip_compression_level%\" \"%SIPDirectory%\" \"%SIPName%\" \"%SIPUUID%\"", "execute": "compress_aip", "filter_file_end": null, "filter_file_start": null, @@ -3278,7 +2729,6 @@ "db99ab43-04d7-44ab-89ec-e09d7bbdc39d": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "--sipUUID \"%TransferUUID%\" --xmlFile \"%SIPDirectory%\"metadata/transfer_metadata.xml", "execute": "create_transfer_metadata", "filter_file_end": null, @@ -3314,7 +2764,6 @@ "dba3028d-2029-4a87-9992-f6335d890528": { "config": { "@manager": "linkTaskManagerFiles", - "@model": "StandardTaskConfig", "arguments": "--fileUUID \"%fileUUID%\" --inputFile \"%relativeLocation%\" --sipDirectory \"%SIPDirectory%\"", "execute": "remove_files_without_premis_metadata", "filter_file_end": null, @@ -3347,7 +2796,6 @@ "dc144ff4-ad74-4a6e-ac15-b0beedcaf662": { "config": { "@manager": "linkTaskManagerFiles", - "@model": "StandardTaskConfig", "arguments": "--transferUUID \"%TransferUUID%\" --sipDirectory \"%SIPDirectory%\" --filePath \"%relativeLocation%\" --fileUUID \"%fileUUID%\" --eventIdentifierUUID \"%taskUUID%\" --date \"%date%\"", "execute": "assign_file_uuids", "filter_file_end": null, @@ -3384,7 +2832,6 @@ "dc9d4991-aefa-4d7e-b7b5-84e3c4336e74": { "config": { "@manager": "linkTaskManagerFiles", - "@model": "StandardTaskConfig", "arguments": "--sipUUID \"%SIPUUID%\" --sipDirectory \"%SIPDirectory%\" --filePath \"%relativeLocation%\" --fileUUID \"%fileUUID%\" --eventIdentifierUUID \"%taskUUID%\" --date \"%date%\" --use \"metadata\" --disable-update-filegrpuse", "execute": "assign_file_uuids", "filter_file_end": null, @@ -3419,10 +2866,11 @@ "dec97e3c-5598-4b99-b26e-f87a435a6b7f": { "config": { "@manager": "linkTaskManagerChoice", - "@model": "MicroServiceChainChoice", - "chain_choices": [ - "01d80b27-4ad1-4bd1-8f8d-f819f18bf685", - "79f1f5af-7694-48a4-b645-e42790bbf870" + "config_attr": "extract_packages", + "default": true, + "choices": [ + {"value": true, "link_id": "1cb7e228-6e94-4c93-bf70-430af99b9264"}, + {"value": false, "link_id": "f8ef02c4-f585-4b0d-9b6f-3cef6fbe527f"} ] }, "description": { @@ -3445,41 +2893,9 @@ "sv": "Extrahera paket" } }, - "df02cac1-f582-4a86-b7cf-da98a58e279e": { - "config": { - "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", - "arguments": "\"%sharedPath%processingConfigs/default.xml\" \"%relativeLocation%processingMCP.xml\" -n", - "execute": "cmd_cp", - "filter_file_end": null, - "filter_file_start": null, - "filter_subdir": null, - "stderr_file": null, - "stdout_file": null - }, - "description": { - "en": "Include default SIP processingMCP.xml", - "pt_BR": "Incluir padrão SIP processingMCP.xml", - "sv": "Inkludera standard SIP processingMCP.xml" - }, - "exit_codes": { - "0": { - "job_status": "Completed successfully", - "link_id": "f3be1ee1-8881-465d-80a6-a6f093d40ec2" - } - }, - "fallback_job_status": "Failed", - "fallback_link_id": "b2ef06b9-bca4-49da-bc5c-866d7b3c4bb1", - "group": { - "en": "Include default SIP processingMCP.xml", - "pt_BR": "Incluir padrão SIP processingMCP.xml", - "sv": "Inkludera standard SIP processingMCP.xml" - } - }, "e4b0c713-988a-4606-82ea-4b565936d9a7": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%metadata\" \"%SIPDirectory%objects/metadata\"", "execute": "move_or_merge", "filter_file_end": null, @@ -3512,7 +2928,6 @@ "e76aec15-5dfa-4b14-9405-735863e3a6fa": { "config": { "@manager": "linkTaskManagerFiles", - "@model": "StandardTaskConfig", "arguments": "--filePath \"%relativeLocation%\" --fileUUID \"%fileUUID%\" --eventIdentifierUUID \"%taskUUID%\" --date \"%date%\"", "execute": "update_size_and_checksum", "filter_file_end": null, @@ -3547,7 +2962,6 @@ "e780473a-0c10-431f-bab6-5d7238b2b70b": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "\"%TransferUUID%\"", "execute": "failed_transfer_cleanup", "filter_file_end": null, @@ -3584,7 +2998,6 @@ "ea0e8838-ad3a-4bdd-be14-e5dba5a4ae0c": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "\"%SIPDirectory%\"", "execute": "restructure_for_compliance", "filter_file_end": null, @@ -3619,7 +3032,6 @@ "ee438694-815f-4b74-97e1-8e7dde2cc6d5": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "--sipDirectory \"%SIPDirectory%\" --sipUUID \"%SIPUUID%\" --sharedPath \"%sharedPath%\"", "execute": "copy_transfers_metadata_and_logs", "filter_file_end": null, @@ -3653,48 +3065,20 @@ }, "f09847c2-ee51-429a-9478-a860477f6b8d": { "config": { - "@manager": "linkTaskManagerReplacementDicFromChoice", - "@model": "MicroServiceChoiceReplacementDic", - "replacements": [ - { - "description": { - "en": "Yes", - "es": "Sí", - "fr": "Oui", - "ja": "はい", - "pt_BR": "Sim", - "sv": "Ja" - }, - "id": "d97297c7-2b49-4cfe-8c9f-0613d63ed763", - "items": { - "IDCommand": "True" - } - }, - { - "description": { - "en": "No", - "fr": "Non", - "ja": "いいえ", - "sv": "Nej" - }, - "id": "1f77af0a-2f7a-468f-af8c-653a9e61ca4f", - "items": { - "IDCommand": "False" - } - } + "@manager": "linkTaskManagerChoice", + "config_attr": "identify_transfer", + "default": true, + "choices": [ + {"value": true, "link_id": "2522d680-c7d9-4d06-8b11-a28d8bd8a71f"}, + {"value": false, "link_id": "b944ec7f-7f99-491f-986d-58914c9bb4fa"} ] }, "description": { - "en": "Do you want to perform file format identification?" - }, - "exit_codes": { - "0": { - "job_status": "Completed successfully", - "link_id": "2522d680-c7d9-4d06-8b11-a28d8bd8a71f" - } + "en": "Do you want to perform file format identification (Transfer)?" }, + "exit_codes": {}, "fallback_job_status": "Failed", - "fallback_link_id": "e780473a-0c10-431f-bab6-5d7238b2b70b", + "fallback_link_id": null, "group": { "en": "Identify file format", "es": "Identificar formato de fichero", @@ -3703,68 +3087,9 @@ "sv": "Identifiera filformat" } }, - "f19926dd-8fb5-4c79-8ade-c83f61f55b40": { - "config": { - "@manager": "linkTaskManagerReplacementDicFromChoice", - "@model": "MicroServiceChoiceReplacementDic", - "replacements": [ - { - "description": { - "en": "Yes", - "es": "Sí", - "fr": "Oui", - "ja": "はい", - "pt_BR": "Sim", - "sv": "Ja" - }, - "id": "85b1e45d-8f98-4cae-8336-72f40e12cbef", - "items": { - "DeletePackage": "True" - } - }, - { - "description": { - "en": "No", - "fr": "Non", - "ja": "いいえ", - "sv": "Nej" - }, - "id": "72e8443e-a8eb-49a8-ba5f-76d52f960bde", - "items": { - "DeletePackage": "False" - } - } - ] - }, - "description": { - "en": "Delete package after extraction?", - "es": "¿Eliminar el paquete después de su extracción?", - "fr": "Supprimer les paquets après l'extraction?", - "ja": "抽出後にパッケージを削除しますか？", - "pt_BR": "Apagar o pacote após a extração?", - "sv": "Radera paket efter extrahering?" - }, - "exit_codes": { - "0": { - "job_status": "Completed successfully", - "link_id": "1cb7e228-6e94-4c93-bf70-430af99b9264" - } - }, - "fallback_job_status": "Failed", - "fallback_link_id": null, - "group": { - "en": "Extract packages", - "es": "Extraer paquetes", - "fr": "Extraire les paquets ", - "ja": "パッケージの抽出", - "pt_BR": "Extrair pacotes", - "sv": "Extrahera paket" - } - }, "f1bfce12-b637-443f-85f8-b6450ca01a13": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "\"%relativeLocation%\" \"%TransferUUID%\"", "execute": "verify_checksum", "filter_file_end": null, @@ -3797,7 +3122,6 @@ "f378ec85-adcc-4ee6-ada2-bc90cfe20efb": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "\"%TransferUUID%\" \"%relativeLocation%metadata/dc.json\"", "execute": "save_dublin_core", "filter_file_end": null, @@ -3831,7 +3155,6 @@ "f3be1ee1-8881-465d-80a6-a6f093d40ec2": { "config": { "@manager": "linkTaskManagerFiles", - "@model": "StandardTaskConfig", "arguments": "\"%relativeLocation%\" \"%fileUUID%\"", "execute": "remove_unneeded_files", "filter_file_end": null, @@ -3862,7 +3185,6 @@ "f574b2a0-6e0b-4c74-ac5b-a73ddb9593a0": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "\"%SIPUUID%\" \"%SIPDirectory%metadata/submissionDocumentation\" \"%sharedPath%\"", "execute": "copy_transfer_submission_documentation", "filter_file_end": null, @@ -3897,7 +3219,6 @@ "f8ef02c4-f585-4b0d-9b6f-3cef6fbe527f": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "arguments": "\"%TransferUUID%\" \"%sharedPath%\"", "execute": "store_file_modification_dates", "filter_file_end": null, diff --git a/a3m/cli/client/__main__.py b/a3m/cli/client/__main__.py index 64d92553c..9f3f0c1d7 100644 --- a/a3m/cli/client/__main__.py +++ b/a3m/cli/client/__main__.py @@ -2,9 +2,12 @@ import datetime import logging import sys +from typing import List +from typing import Optional import click from django.conf import settings +from google.protobuf.descriptor import FieldDescriptor from rich.console import Console from rich.panel import Panel from rich.table import Table @@ -27,9 +30,16 @@ @click.option( "--wait-for-ready", is_flag=True, help="Block until server becomes available." ) +@click.option( + "--processing-config", + "-p", + multiple=True, + metavar="CONFIG_PAIR", + help='Processing configuration pair (form "name:value"), e.g.: "normalize=no".', +) @click.option("--no-input", is_flag=True, help="Disable interactive mode.") @click.pass_context -def main(ctx, uri, name, address, wait_for_ready, no_input): +def main(ctx, uri, name, address, processing_config, wait_for_ready, no_input): """a3m - Lightweight Archivematica. Creates an Archival Information Package (AIP) from the contents in URI. @@ -56,8 +66,10 @@ def main(ctx, uri, name, address, wait_for_ready, no_input): else: name = click.prompt("Enter transfer name") + processing_config = _prepare_config(processing_config) + with ClientWrapper(address, wait_for_ready) as cw: - resp = cw.client.submit(uri, name) + resp = cw.client.submit(uri, name, processing_config) click.secho(f"AIP {resp.id} is being generated...") resp = cw.client.wait_until_complete(resp.id) @@ -76,6 +88,70 @@ def main(ctx, uri, name, address, wait_for_ready, no_input): click.secho("Processing completed successfully!", fg="green") +def _to_int(value: str) -> Optional[int]: + try: + return int(value) + except ValueError: + None + + +def _prepare_config(user_pairs: Optional[List[str]] = None) -> a3m_pb2.ProcessingConfig: + """Consolidate ``a3m_pb2.ProcessingConfig`` defaults and user-provided. + + A3M-TODO: defaults should be set on the server! + + ``user_pairs`` is a list of strings following the format ``key=value``. + Those matching processing configuration attributes will be evaluated as + indicated by the field type, e.g.: + + * ``normalize=yes`` (boolean), or + * ``aip_compression_level=1`` (integer), or + * ``aip_compression_algorithm=1`` (enum) + (it is not possible at the moment to use the enum names) + + A comprehensive list can be found in the definition of the + ``ProcessingConfig`` message in the proto file. + """ + config = a3m_pb2.ProcessingConfig( + assign_uuids_to_directories=True, + examine_contents=True, + generate_transfer_structure_report=True, + document_empty_directories=True, + extract_packages=True, + delete_packages_after_extraction=False, + identify_transfer=True, + identify_submission_and_metadata=True, + identify_before_normalization=True, + normalize=True, + transcribe_files=True, + perform_policy_checks_on_originals=True, + perform_policy_checks_on_preservation_derivatives=True, + aip_compression_level=1, + aip_compression_algorithm=a3m_pb2.ProcessingConfig.S7_COPY, + ) + for item in user_pairs: + head, sep, tail = item.partition("=") + if head and sep: + try: + field = a3m_pb2.ProcessingConfig.DESCRIPTOR.fields_by_name[head] + except KeyError: + continue + if field.type == FieldDescriptor.TYPE_BOOL: + enabled = tail.lower() in ("yes", "true", "1", "on") + setattr(config, head, enabled) + elif field.type == FieldDescriptor.TYPE_INT32: + if (value := _to_int(tail)) is not None: + setattr(config, head, value) + elif field.type == FieldDescriptor.TYPE_ENUM: + if (value := _to_int(tail)) is not None: + setattr(config, head, value) + else: + raise NotImplementedError( + f"{head} has an unsupported type ({field.type})" + ) + return config + + def _print_failed_jobs(client: Client, jobs): """Prints failed jobs and associated tasks for a failed package.""" if not jobs: diff --git a/a3m/cli/client/wrapper.py b/a3m/cli/client/wrapper.py index 71c3cd9c2..5ccc9dbb7 100644 --- a/a3m/cli/client/wrapper.py +++ b/a3m/cli/client/wrapper.py @@ -7,11 +7,11 @@ class ClientWrapper(ContextDecorator): - """A context manager that provides a a3m client. + """A context manager that provides a a3m client or client-server instance. - When the server address is undefined, it launches an embedded server - instance and sets up the client accordingly. Used resources are - automatically cleaned up. + Use ``address`` to indicate the location of the a3m server. When undefined, + this wrapper launches an embedded server and sets up the client accordingly. + Used resources are automatically cleaned up. """ BIND_LOCAL_ADDRESS = "localhost:0" @@ -35,7 +35,6 @@ def __exit__(self, exc_type, exc_value, exc_traceback): return False def _create_server(self): - """Create a3m server.""" self.server = None if self.address is not None: diff --git a/a3m/client/clientScripts/a3m_download_transfer.py b/a3m/client/clientScripts/a3m_download_transfer.py index 72600027d..04888fe9e 100644 --- a/a3m/client/clientScripts/a3m_download_transfer.py +++ b/a3m/client/clientScripts/a3m_download_transfer.py @@ -44,9 +44,9 @@ def _create_tmpdir(suffix, purpose=None): def _archived(path): command = ["sf", "-json", path] exit_code, stdout, stderr = executeOrRun("command", command, capture_output=True) - if exit_code > 0: + if exit_code != 0: raise RetrievalError( - "Extraction failed, Siegfried quit with exit code {exit_code}" + f"Extraction failed, Siegfried quit with exit code {exit_code}" ) idresults = json.loads(stdout) puid = idresults["files"][0]["matches"][0]["id"] @@ -156,6 +156,7 @@ def main(job, transfer_id, transfer_path, url): return 1 except RetrievalError as err: job.pyprint(f"Error retrievent contents: {err}", file=sys.stderr) + return 1 if is_bag(transfer_path): job.pyprint("Bags not supported yet!", file=sys.stderr) diff --git a/a3m/client/clientScripts/assign_uuids_to_directories.py b/a3m/client/clientScripts/assign_uuids_to_directories.py index 9c45ab985..9a9fc1492 100644 --- a/a3m/client/clientScripts/assign_uuids_to_directories.py +++ b/a3m/client/clientScripts/assign_uuids_to_directories.py @@ -34,7 +34,6 @@ from a3m.archivematicaFunctions import format_subdir_path from a3m.archivematicaFunctions import get_dir_uuids -from a3m.archivematicaFunctions import str2bool from a3m.main.models import Directory from a3m.main.models import Transfer @@ -68,16 +67,6 @@ def wrapped(*_args, **kwargs): return wrapped -def _exit_if_not_include_dirs(include_dirs): - """Quit processing if include_dirs is not truthy.""" - if not include_dirs: - logger.debug( - "Configuration indicates that directories in this Transfer" - " should not be given UUIDs." - ) - raise DirsUUIDsWarning - - def _get_transfer_mdl(transfer_uuid): """Get the ``Transfer`` model with UUID ``transfer_uuid``. Also update it in the db to indicate that this transfer has UUIDs assigned to the @@ -108,13 +97,11 @@ def _get_subdir_paths(root_path): @exit_on_known_exception -def main(job, transfer_path, transfer_uuid, include_dirs): +def main(job, transfer_path, transfer_uuid): """Assign UUIDs to all of the directories (and subdirectories, i.e., all unique directory paths) in the absolute system path ``transfer_path``, such - being the root directory of the transfer with UUID ``transfer_uuid``. Do - this only if ``include_dirs`` is ``True``. + being the root directory of the transfer with UUID ``transfer_uuid``. """ - _exit_if_not_include_dirs(include_dirs) Directory.create_many( get_dir_uuids(_get_subdir_paths(transfer_path), logger, printfn=job.pyprint), _get_transfer_mdl(transfer_uuid), @@ -128,14 +115,6 @@ def call(jobs): "transfer_path", type=str, help="The path to the Transfer on disk." ) parser.add_argument("transfer_uuid", type=str, help="The UUID of the Transfer.") - parser.add_argument( - "--include-dirs", - action="store", - type=str2bool, - dest="include_dirs", - default="No", - ) - with transaction.atomic(): for job in jobs: with job.JobContext(logger=logger): diff --git a/a3m/client/clientScripts/compress_aip.py b/a3m/client/clientScripts/compress_aip.py index c8d8c566c..8226f8b6f 100644 --- a/a3m/client/clientScripts/compress_aip.py +++ b/a3m/client/clientScripts/compress_aip.py @@ -7,6 +7,7 @@ from a3m import databaseFunctions from a3m.executeOrRunSubProcess import executeOrRun from a3m.main.models import SIP +from a3m.server.rpc.proto import a3m_pb2 def update_unit(sip_uuid, compressed_location): @@ -34,9 +35,29 @@ def compress_aip( ep d87d5845-bd07-4200-b1a4-928e0cb6e1e4 """ + if compression_level == "0": + compression_level = "1" + + # Default is uncompressed. + compression = int(compression) + a3m_pb2.ProcessingConfig.AIPCompressionAlgorithm.Name(compression) + if compression == a3m_pb2.ProcessingConfig.UNSPECIFIED: + compression = a3m_pb2.ProcessingConfig.UNCOMPRESSED + + # Translation to make compress_aip happy. + mapping = { + a3m_pb2.ProcessingConfig.UNCOMPRESSED: ("None", ""), + a3m_pb2.ProcessingConfig.TAR: ("gzip", "tar.gzip"), # A3M-TODO: support + a3m_pb2.ProcessingConfig.TAR_BZIP2: ("pbzip2", "pbzip2"), + a3m_pb2.ProcessingConfig.TAR_GZIP: ("gzip", "tar.gzip"), + a3m_pb2.ProcessingConfig.S7_COPY: ("7z", "copy"), + a3m_pb2.ProcessingConfig.S7_BZIP2: ("7z", "bzip2"), + a3m_pb2.ProcessingConfig.S7_LZMA: ("7z", "lzma"), + } + try: - program, compression_algorithm = compression.split("-") - except ValueError: + program, compression_algorithm = mapping[compression] + except KeyError: msg = f"Invalid program-compression algorithm: {compression}" job.pyprint(msg, file=sys.stderr) return 255 @@ -141,8 +162,8 @@ def compress_aip( def call(jobs): parser = argparse.ArgumentParser(description="Compress an AIP.") - parser.add_argument("compression", type=str, help="%AIPCompressionAlgorithm%") - parser.add_argument("compression_level", type=str, help="%AIPCompressionLevel%") + parser.add_argument("compression", type=str) + parser.add_argument("compression_level", type=str) parser.add_argument("sip_directory", type=str, help="%SIPDirectory%") parser.add_argument("sip_name", type=str, help="%SIPName%") parser.add_argument("sip_uuid", type=str, help="%SIPUUID%") diff --git a/a3m/client/clientScripts/create_sip_from_transfer_objects.py b/a3m/client/clientScripts/create_sip_from_transfer_objects.py index 89064e9d8..3ff233abf 100644 --- a/a3m/client/clientScripts/create_sip_from_transfer_objects.py +++ b/a3m/client/clientScripts/create_sip_from_transfer_objects.py @@ -98,11 +98,6 @@ def main(job, transfer_id, sip_id): dst = sip_dir / "metadata" / "dc.json" shutil.copy(str(src), str(dst)) - # Copy processingMCP.xml file - shutil.copy( - str(transfer_dir / "processingMCP.xml"), str(sip_dir / "processingMCP.xml") - ) - def call(jobs): with transaction.atomic(): diff --git a/a3m/client/clientScripts/identify_file_format.py b/a3m/client/clientScripts/identify_file_format.py index 0745fad24..663ebac85 100644 --- a/a3m/client/clientScripts/identify_file_format.py +++ b/a3m/client/clientScripts/identify_file_format.py @@ -12,31 +12,6 @@ from a3m.main.models import File from a3m.main.models import FileFormatVersion from a3m.main.models import FileID -from a3m.main.models import UnitVariable - - -def _save_id_preference(file_, value): - """ - Saves whether file format identification is being used. - - This is necessary in order to allow post-extraction identification to work. - The replacement dict will be saved to the special 'replacementDict' unit - variable, which will be transformed back into a passVar when a new chain in - the same unit is begun. - """ - value = str(value) - - # The unit_uuid foreign key can point to a transfer or SIP, and this tool - # runs in both. - # Check the SIP first - if it hasn't been assigned yet, then this is being - # run during the transfer. - unit = file_.sip or file_.transfer - - rd = {"%IDCommand%": value} - - UnitVariable.objects.create( - unituuid=unit.pk, variable="replacementDict", variablevalue=str(rd) - ) def write_identification_event(file_uuid, command, format=None, success=True): @@ -99,12 +74,7 @@ def _default_idcommand(): return IDCommand.active.first() -def main(job, enabled, file_path, file_uuid, disable_reidentify): - enabled = True if enabled == "True" else False - if not enabled: - job.print_output("Skipping file format identification") - return 0 - +def main(job, file_path, file_uuid, disable_reidentify): command = _default_idcommand() if command is None: job.write_error("Unable to determine IDCommand.\n") @@ -129,10 +99,6 @@ def main(job, enabled, file_path, file_uuid, disable_reidentify): ) return 0 - # Save whether identification was enabled by the user for use in a later - # chain. - _save_id_preference(file_, enabled) - exitcode, output, err = executeOrRun( command.script_type, command.script, @@ -193,13 +159,6 @@ def main(job, enabled, file_path, file_uuid, disable_reidentify): def call(jobs): parser = argparse.ArgumentParser(description="Identify file formats.") - - # Since AM19 the accepted values are "True" or "False" since the ability to - # choose the command from the workflow has been removed. Instead, this - # script will look up in FPR what's the preferred command. - # This argument may be renamed later. - parser.add_argument("idcommand", type=str, help="%IDCommand%") - parser.add_argument("file_path", type=str, help="%relativeLocation%") parser.add_argument("file_uuid", type=str, help="%fileUUID%") parser.add_argument( @@ -215,7 +174,6 @@ def call(jobs): job.set_status( main( job, - args.idcommand, args.file_path, args.file_uuid, args.disable_reidentify, diff --git a/a3m/client/clientScripts/verify_sip_compliance.py b/a3m/client/clientScripts/verify_sip_compliance.py index 5eb0f801a..b47625d74 100644 --- a/a3m/client/clientScripts/verify_sip_compliance.py +++ b/a3m/client/clientScripts/verify_sip_compliance.py @@ -27,7 +27,7 @@ "metadata/submissionDocumentation", ) -ALLOWABLE_FILES = ("processingMCP.xml",) +ALLOWABLE_FILES = () def checkDirectory(job, directory, ret=0): diff --git a/a3m/client/clientScripts/verify_transfer_compliance.py b/a3m/client/clientScripts/verify_transfer_compliance.py index 4cb39784d..01579ca95 100644 --- a/a3m/client/clientScripts/verify_transfer_compliance.py +++ b/a3m/client/clientScripts/verify_transfer_compliance.py @@ -26,7 +26,7 @@ "metadata/submissionDocumentation", ) -ALLOWABLE_FILES = ("processingMCP.xml",) +ALLOWABLE_FILES = tuple() def verifyDirectoriesExist(job, SIPDir, ret=0): diff --git a/a3m/executeOrRunSubProcess.py b/a3m/executeOrRunSubProcess.py index fdc9b0f75..b57eea246 100644 --- a/a3m/executeOrRunSubProcess.py +++ b/a3m/executeOrRunSubProcess.py @@ -24,7 +24,12 @@ def launchSubProcess( - command, stdIn="", printing=True, arguments=[], env_updates={}, capture_output=False + command, + stdIn="", + printing=False, + arguments=[], + env_updates={}, + capture_output=False, ): """ Launches a subprocess using ``command``, where ``command`` is either: @@ -124,7 +129,7 @@ def launchSubProcess( def createAndRunScript( - text, stdIn="", printing=True, arguments=[], env_updates={}, capture_output=True + text, stdIn="", printing=False, arguments=[], env_updates={}, capture_output=True ): # Output the text to a /tmp/ file scriptPath = "/tmp/" + uuid.uuid4().__str__() diff --git a/a3m/main/models.py b/a3m/main/models.py index 1eff38bb3..1c23d8c57 100644 --- a/a3m/main/models.py +++ b/a3m/main/models.py @@ -318,25 +318,6 @@ def agents(self): agent_lookups = Agent.objects.default_agents_query_keywords() return Agent.objects.filter(agent_lookups) - def set_processing_configuration(self, processing_configuration): - UnitVariable.objects.update_processing_configuration( - "Transfer", self.uuid, processing_configuration - ) - - @property - def processing_configuration(self): - try: - unit_variable = UnitVariable.objects.get( - unittype="Transfer", - unituuid=self.uuid, - variable="processingConfiguration", - ) - except UnitVariable.DoesNotExist: - result = None - else: - result = unit_variable.variablevalue - return result or "default" - class Identifier(models.Model): """Identifiers used by File, Directory SIP models. Used for Handle System @@ -1218,13 +1199,6 @@ def update_variable(self, unit_type, unit_uuid, variable, value, link_id=None): unittype=unit_type, unituuid=unit_uuid, variable=variable, defaults=defaults ) - def update_processing_configuration( - self, unit_type, unit_uuid, processing_configuration - ): - return self.update_variable( - unit_type, unit_uuid, "processingConfiguration", processing_configuration - ) - class UnitVariable(models.Model): id = models.UUIDField( diff --git a/a3m/server/__init__.py b/a3m/server/__init__.py index 362af0abc..3ee32ca95 100644 --- a/a3m/server/__init__.py +++ b/a3m/server/__init__.py @@ -8,7 +8,6 @@ * `jobs.base.Job` and subclasses handle execution of a single link of the workflow. * `jobs.client.ClientScriptJob` for jobs to be executed via MCPClient script * `jobs.client.DecisionJob` for workflow decision points - * `jobs.client.LocalJob` for jobs that are executed on MCPServer directly * `jobs.chain.JobChain` handles passing context between jobs, and determining the next `Job` to be executed based on the workflow chain * `tasks.Task` corresponds to a single command to be executed by MCPClient diff --git a/a3m/server/jobs/__init__.py b/a3m/server/jobs/__init__.py index 20b4f8b67..3b633c754 100644 --- a/a3m/server/jobs/__init__.py +++ b/a3m/server/jobs/__init__.py @@ -9,27 +9,20 @@ The `Job` class is a base class for other job types. There are various concrete types of jobs, handled by subclasses: * `ClientScriptJob`, handling Jobs to be execute on MCPClient - * `DecisionJob`, handling workflow decision points - * `LocalJob`, handling work done directly on MCPServer + * `NextLinkDecisionJob`, handling workflow decision points """ from a3m.server.jobs.base import Job from a3m.server.jobs.chain import JobChain from a3m.server.jobs.client import ClientScriptJob from a3m.server.jobs.client import DirectoryClientScriptJob from a3m.server.jobs.client import FilesClientScriptJob -from a3m.server.jobs.decisions import DecisionJob -from a3m.server.jobs.decisions import NextChainDecisionJob -from a3m.server.jobs.decisions import UpdateContextDecisionJob -from a3m.server.jobs.local import LocalJob +from a3m.server.jobs.decisions import NextLinkDecisionJob __all__ = ( "ClientScriptJob", - "DecisionJob", "DirectoryClientScriptJob", "FilesClientScriptJob", "Job", "JobChain", - "LocalJob", - "NextChainDecisionJob", - "UpdateContextDecisionJob", + "NextLinkDecisionJob", ) diff --git a/a3m/server/jobs/chain.py b/a3m/server/jobs/chain.py index 8273e6f43..50a4cbf62 100644 --- a/a3m/server/jobs/chain.py +++ b/a3m/server/jobs/chain.py @@ -9,14 +9,13 @@ one by looking at the workflow. """ import logging +from typing import Iterator -from django.utils import timezone - +from a3m.server.jobs.base import Job from a3m.server.jobs.client import ClientScriptJob from a3m.server.jobs.client import DirectoryClientScriptJob from a3m.server.jobs.client import FilesClientScriptJob -from a3m.server.jobs.decisions import NextChainDecisionJob -from a3m.server.jobs.decisions import UpdateContextDecisionJob +from a3m.server.jobs.decisions import NextLinkDecisionJob logger = logging.getLogger(__name__) @@ -34,9 +33,7 @@ def get_job_class_for_link(link): elif manager_name == "linkTaskManagerFiles": job_class = FilesClientScriptJob elif manager_name == "linkTaskManagerChoice": - job_class = NextChainDecisionJob - elif manager_name == "linkTaskManagerReplacementDicFromChoice": - job_class = UpdateContextDecisionJob + job_class = NextLinkDecisionJob else: raise ValueError(f"Unknown manager type {manager_name}") @@ -55,14 +52,12 @@ class JobChain: * next_link, a workflow link that can be set to redirect the job chain """ - def __init__(self, package, chain, workflow, starting_link=None): + def __init__(self, package, workflow, starting_link): """Create an instance of a chain, based on the workflow chain given.""" self.package = package - self.chain = chain self.workflow = workflow - self.started_on = timezone.now() - self.initial_link = starting_link or self.chain.link + self.initial_link = starting_link self.current_link = None self.next_link = self.initial_link @@ -71,13 +66,12 @@ def __init__(self, package, chain, workflow, starting_link=None): self.context = self.package.context.copy() logger.debug( - "Creating JobChain %s for package %s (initial link %s)", - chain.id, + "Creating JobChain for package %s (initial link %s)", package.uuid, self.initial_link, ) - def __iter__(self): + def __iter__(self) -> Iterator[Job]: return self def __next__(self): @@ -90,20 +84,21 @@ def __next__(self): except KeyError: next_link = None - if next_link: - self.current_link = next_link - job_class = get_job_class_for_link(self.current_link) - self.current_job = job_class(self, self.current_link, self.package) - return self.current_job - else: + # End of chain. + if not next_link: self.current_link = None self.current_job = None self.chain_completed() raise StopIteration - @property - def id(self): - return self.chain.id + # Ensure we have a Link instance instead of its identifier. + if isinstance(next_link, str): + next_link = self.workflow.get_link(next_link) + + self.current_link = next_link + job_class = get_job_class_for_link(self.current_link) + self.current_job = job_class(self, self.current_link, self.package) + return self.current_job def job_completed(self): logger.debug( @@ -119,6 +114,4 @@ def job_completed(self): def chain_completed(self): """Log chain completion""" - logger.debug( - "Done with chain %s for package %s", self.chain.id, self.package.uuid - ) + logger.debug("Done with chain for package %s", self.package.uuid) diff --git a/a3m/server/jobs/client.py b/a3m/server/jobs/client.py index aa799dd29..5d52fe97e 100644 --- a/a3m/server/jobs/client.py +++ b/a3m/server/jobs/client.py @@ -176,12 +176,7 @@ class FilesClientScriptJob(ClientScriptJob): @property def filter_subdir(self): - """Returns directory to filter files on. - - In Archivematica, it was possible to override this per package in a - UnitVariable. In practice this was done only twice in the workflow of - Maildir transfers which a3m does not handle. - """ + """Returns directory to filter files on.""" return self.link.config.get("filter_subdir", "") def submit_tasks(self): diff --git a/a3m/server/jobs/decisions.py b/a3m/server/jobs/decisions.py index 96df4bdb7..e6ad5f0b4 100644 --- a/a3m/server/jobs/decisions.py +++ b/a3m/server/jobs/decisions.py @@ -1,180 +1,47 @@ """ -Jobs relating to user decisions. +Jobs relating to configurable decisions. """ -import abc import logging -from collections import OrderedDict -from a3m.server.db import auto_close_old_connections from a3m.server.jobs.base import Job -from a3m.server.processing_config import load_preconfigured_choice -from a3m.server.processing_config import load_processing_xml logger = logging.getLogger(__name__) -class DecisionJob(Job, metaclass=abc.ABCMeta): - """A Job that handles a workflow decision point. - - The `run` method checks if a choice has been preconfigured. If so, - it executes as a normal job. If should fail otherwise. - """ - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - @property - def workflow(self): - return self.link.workflow +class NextLinkDecisionJob(Job): + """A job that determines the next link to be executed.""" def run(self, *args, **kwargs): super().run(*args, **kwargs) logger.debug("Running %s (package %s)", self.description, self.package.uuid) + # Reload the package, in case the path has changed self.package.reload() self.save_to_db() - preconfigured_choice = self.get_preconfigured_choice() - if not preconfigured_choice: - logger.error("Link not configured: %s", self.link.id) - return - - return self.decide(preconfigured_choice) - - def get_preconfigured_choice(self): - """Check the processing XML file for a pre-selected choice. - - Returns a value for choices if found, None otherwise. - """ - return load_preconfigured_choice( - self.package.current_path, self.link.id, self.workflow - ) - - @abc.abstractmethod - def get_choices(self): - """Returns a dict of value: description choices.""" - - @abc.abstractmethod - def decide(self, choice): - """Make a choice, resulting in this job being completed and the - next one started. - """ - - -class NextChainDecisionJob(DecisionJob): - """ - A type of workflow decision that determines the next chain to be executed, - by UUID. - """ - - def get_choices(self): - choices = OrderedDict() - for chain_id in self.link.config["chain_choices"]: - try: - chain = self.workflow.get_chain(chain_id) - except KeyError: - continue - choices[chain_id] = chain["description"] - - return choices - - @auto_close_old_connections() - def decide(self, choice): - # TODO: fix circular imports :( - from a3m.server.jobs import JobChain - - if choice not in self.get_choices(): - raise ValueError(f"{choice} is not one of the available choices") + self.job_chain.next_link = self.decide() - chain = self.workflow.get_chain(choice) - logger.debug("Using user selected chain %s for link %s", chain.id, self.link.id) - - self.mark_complete() - - job_chain = JobChain(self.package, chain, self.workflow) - return next(job_chain, None) - - -class UpdateContextDecisionJob(DecisionJob): - """A job that updates the job chain context based on a user choice.""" + return next(self.job_chain, None) - # TODO: This type of job is mostly copied from the previous - # linkTaskManagerReplacementDicFromChoice, and it seems to have multiple - # ways of executing. It could use some cleanup. + def decide(self): + config = self.link.config - @auto_close_old_connections() - def run(self, *args, **kwargs): - # Intentionally don't call super() here - logger.debug("Running %s (package %s)", self.description, self.package.uuid) - # Reload the package, in case the path has changed - self.package.reload() - self.save_to_db() + config_value = self.get_configured_value(config["config_attr"]) + if config_value is None: + config_value = config["default"] - preconfigured_context = self.load_preconfigured_context() - if not preconfigured_context: - logger.error("Link not configured: %s", self.link.id) - return + next_id = None + for item in config["choices"]: + if item["value"] == config_value: + next_id = item["link_id"] + break - logger.debug( - "Job %s got preconfigured context %s", self.uuid, preconfigured_context - ) - self.job_chain.context.update(preconfigured_context) + logger.debug("Using user selected link %s", next_id) self.mark_complete() - return next(self.job_chain, None) - def _format_items(self, items): - """Wrap replacement items with the ``%`` wildcard character.""" - return {f"%{key}%": value for key, value in items.items()} - - def load_preconfigured_context(self): - choice_id = self.link.id - - processing_xml = load_processing_xml(self.package.current_path, self.workflow) - if processing_xml is None: - return - - for preconfiguredChoice in processing_xml.findall(".//preconfiguredChoice"): - if preconfiguredChoice.find("appliesTo").text != choice_id: - continue - desired_choice = preconfiguredChoice.find("goToChain").text - - try: - link = self.workflow.get_link(choice_id) - except KeyError: - return None - - for replacement in link.config["replacements"]: - if replacement["id"] != desired_choice: - continue - # In our JSON-encoded document, the items in - # the replacements are not wrapped, do it here. - # Needed by ReplacementDict. - return self._format_items(replacement["items"]) - - def get_choices(self): - choices = OrderedDict() - # TODO: this is kind of an odd side effect here; refactor - self.choice_items = [] - - for index, item in enumerate(self.link.config["replacements"]): - # item description is already translated in workflow - choices[str(index)] = item["description"] - self.choice_items.append(self._format_items(item["items"])) - - return choices - - @auto_close_old_connections() - def decide(self, choice): - # TODO: DRY with sibling classes - if choice not in self.get_choices(): - raise ValueError(f"{choice} is not one of the available choices") - - choice_index = int(choice) - items = self.choice_items[choice_index] - - self.job_chain.context.update(items) - self.mark_complete() + return next_id - return next(self.job_chain, None) + def get_configured_value(self, attr_name): + return getattr(self.package.config, attr_name, None) diff --git a/a3m/server/jobs/local.py b/a3m/server/jobs/local.py deleted file mode 100644 index e24575221..000000000 --- a/a3m/server/jobs/local.py +++ /dev/null @@ -1,24 +0,0 @@ -""" -Jobs executed locally in MCPServer. -""" -import abc -import logging - -from a3m.server.db import auto_close_old_connections -from a3m.server.jobs.base import Job - - -logger = logging.getLogger(__name__) - - -class LocalJob(Job, metaclass=abc.ABCMeta): - """Base class for jobs that are executed directly.""" - - @auto_close_old_connections() - def run(self, *args, **kwargs): - super().run(*args, **kwargs) - logger.debug("Running %s (package %s)", self.description, self.package.uuid) - - # Reload the package, in case the path has changed - self.package.reload() - self.save_to_db() diff --git a/a3m/server/packages.py b/a3m/server/packages.py index 31c9d2ea1..36d827aff 100644 --- a/a3m/server/packages.py +++ b/a3m/server/packages.py @@ -80,9 +80,10 @@ class Package: package is in. """ - def __init__(self, name, url, transfer, sip): + def __init__(self, name, url, config, transfer, sip): self.name = name self.url = url + self.config = config self.transfer = transfer self.sip = sip self.stage = Stage.TRANSFER @@ -96,7 +97,7 @@ def __repr__(self): @classmethod @auto_close_old_connections() - def create_package(cls, package_queue, executor, workflow, name, url): + def create_package(cls, package_queue, executor, workflow, name, url, config): """Launch transfer and return its object immediately.""" if not name: raise ValueError("No transfer name provided.") @@ -110,7 +111,6 @@ def create_package(cls, package_queue, executor, workflow, name, url): transfer = models.Transfer.objects.create( uuid=transfer_id, currentlocation=transfer_dir ) - transfer.set_processing_configuration("default") logger.debug("Transfer object created: %s", transfer.pk) sip_id = str(uuid4()) @@ -121,7 +121,7 @@ def create_package(cls, package_queue, executor, workflow, name, url): sip.transfer_id = transfer_id logger.debug("SIP object created: %s", sip.pk) - package = cls(name, url, transfer, sip) + package = cls(name, url, config, transfer, sip) params = (package, package_queue, workflow) future = executor.submit(Package.trigger_workflow, *params) @@ -135,12 +135,11 @@ def create_package(cls, package_queue, executor, workflow, name, url): def trigger_workflow(package, package_queue, workflow): logger.debug("Package %s: starting workflow processing", package.uuid) - # It should be "2671aef1-653a-49bf-bc74-82572b64ace9". - chain = workflow.get_initiator_chain() - if chain is None: + initiator_link = workflow.get_initiator() + if initiator_link is None: raise ValueError("Workflow initiator not found") - job_chain = JobChain(package, chain, workflow) + job_chain = JobChain(package, workflow, initiator_link) package_queue.schedule_job(next(job_chain)) @@ -219,27 +218,6 @@ def base_queryset(self): else: return models.File.objects.filter(transfer_id=self.transfer.pk) - @auto_close_old_connections() - def set_variable(self, key, value, chain_link_id): - """Sets a UnitVariable, which tracks choices made by users during processing.""" - # TODO: refactor this concept - if not value: - value = "" - else: - value = str(value) - - unit_var, created = models.UnitVariable.objects.update_or_create( - unittype=self.unit_variable_type, - unituuid=self.subid, - variable=key, - defaults=dict(variablevalue=value, microservicechainlink=chain_link_id), - ) - if created: - message = "New UnitVariable %s created for %s: %s (MSCL: %s)" - else: - message = "Existing UnitVariable %s for %s updated to %s (MSCL" " %s)" - logger.debug(message, key, self.subid, value, chain_link_id) - def start_ingest(self): """Signal this package so it becomes a SIP.""" self.stage = Stage.INGEST @@ -253,7 +231,6 @@ def reload(self): else: transfer = models.Transfer.objects.get(uuid=self.transfer.pk) self.current_path = transfer.currentlocation - self.processing_configuration = transfer.processing_configuration def get_replacement_mapping(self): mapping = BASE_REPLACEMENTS.copy() @@ -274,6 +251,15 @@ def get_replacement_mapping(self): } ) + mapping.update( + { + fr"%config:{config_attr.name}%": str( + getattr(self.config, config_attr.name) + ) + for config_attr in a3m_pb2.ProcessingConfig.DESCRIPTOR.fields + } + ) + if self.stage is Stage.INGEST: mapping.update( { @@ -286,7 +272,6 @@ def get_replacement_mapping(self): { self.replacement_path_string: self.current_path, r"%unitType%": self.unit_variable_type, - r"%processingConfiguration%": self.processing_configuration, r"%URL%": self.url, } ) diff --git a/a3m/server/processing_config.py b/a3m/server/processing_config.py deleted file mode 100644 index bb547f227..000000000 --- a/a3m/server/processing_config.py +++ /dev/null @@ -1,114 +0,0 @@ -"""Processing configuration. - -This module lists the processing configuration fields where the user has the -ability to establish predefined choices via the user interface, and handles -processing config file operations. -""" -import logging -from pathlib import Path - -from django.conf import settings -from lxml import etree - - -logger = logging.getLogger(__name__) - -PROCESSING_XML_FILE = "processingMCP.xml" - - -class ProcessingConfigError(Exception): - pass - - -def parse_processing_xml(path, workflow): - """Parse, validate and return processing config XML document. - - Validation is performed against the workflow dataset. - """ - if not path.is_file(): - raise ProcessingConfigError(f"Configuration file not found: {path}") - - try: - processing_xml = etree.parse(str(path)) - except etree.LxmlError: - raise ProcessingConfigError( - f"Error parsing XML at {path} for pre-configured choice" - ) - - preconfigured_applies = [] - - # Confirm that all pre-configured choices match a real workflow entity. - for preconfigured_choice in processing_xml.findall(".//preconfiguredChoice"): - applies_to = preconfigured_choice.find("appliesTo").text - go_to_chain = preconfigured_choice.find("goToChain").text - try: - link = workflow.get_link(applies_to) - except KeyError: - raise ProcessingConfigError( - f"Pre-configured choice {applies_to} (appliesTo) not found in workflow" - ) - - matched = False - manager = link.config["@manager"] - if manager == "linkTaskManagerReplacementDicFromChoice": - for replacement in link.config["replacements"]: - if go_to_chain == replacement["id"]: - matched = True - else: - try: - workflow.get_chain(go_to_chain) - except KeyError: - pass - else: - matched = True - if not matched: - raise ProcessingConfigError( - f"Pre-configured choice {go_to_chain} (goToChain) not found in workflow" - ) - preconfigured_applies.append(applies_to) - - # Confirm that all workflow decions are addressed. - for link_id in workflow.get_links(): - try: - link = workflow.get_link(link_id) - except KeyError: - continue - manager = link.config["@manager"] - if manager not in ( - "linkTaskManagerChoice", - "linkTaskManagerReplacementDicFromChoice", - ): - continue - if link_id not in preconfigured_applies: - raise ProcessingConfigError( - f"Workflow link {link_id} ({manager}) is not pre-configured by {path.name}" - ) - - return processing_xml - - -def validate_processing_configs(workflow): - """Validate all local processing configurations.""" - config_dir = Path(settings.SHARED_DIRECTORY, "processingConfigs") - matches = config_dir.glob("*.xml") - if not matches: - raise ProcessingConfigError(f"{config_dir} is not set up") - for path in matches: - parse_processing_xml(path, workflow) - - -def load_processing_xml(package_path, workflow): - """Load the processing configuration made available in the package.""" - return parse_processing_xml(Path(package_path, PROCESSING_XML_FILE), workflow) - - -def load_preconfigured_choice(package_path, workflow_link_id, workflow): - choice = None - - processing_xml = load_processing_xml(package_path, workflow) - if processing_xml is not None: - for preconfigured_choice in processing_xml.findall(".//preconfiguredChoice"): - if preconfigured_choice.find("appliesTo").text == str(workflow_link_id): - choice = preconfigured_choice.find("goToChain").text - - return choice diff --git a/a3m/server/rpc/client.py b/a3m/server/rpc/client.py index ed3d4a637..27d0f29d5 100644 --- a/a3m/server/rpc/client.py +++ b/a3m/server/rpc/client.py @@ -22,6 +22,8 @@ class Client: + """a3m gRPC API client.""" + def __init__( self, channel: Channel, @@ -50,8 +52,8 @@ def _unary_call(self, api_method, request): def version_metadata(): return ((_VERSION_METADATA_KEY, __version__),) - def submit(self, url: str, name: str): - request = a3m_pb2.SubmitRequest(name=name, url=url) + def submit(self, url: str, name: str, config: a3m_pb2.ProcessingConfig = None): + request = a3m_pb2.SubmitRequest(name=name, url=url, config=config) return self._unary_call(self.transfer_stub.Submit, request) def read(self, package_id: str): diff --git a/a3m/server/rpc/proto/a3m.proto b/a3m/server/rpc/proto/a3m.proto index 5b9296935..3fa5f67c0 100644 --- a/a3m/server/rpc/proto/a3m.proto +++ b/a3m/server/rpc/proto/a3m.proto @@ -18,6 +18,7 @@ service Transfer { message SubmitRequest { string name = 1; string url = 2; + ProcessingConfig config = 3; } message SubmitReply { @@ -75,3 +76,36 @@ message Task { string stdout = 7; string stderr = 8; } + +message ProcessingConfig { + bool assign_uuids_to_directories = 1; + bool examine_contents = 2; + bool generate_transfer_structure_report = 3; + bool document_empty_directories = 4; + bool extract_packages = 5; + bool delete_packages_after_extraction = 6; + bool identify_transfer = 7; + bool identify_submission_and_metadata = 8; + bool identify_before_normalization = 9; + bool normalize = 10; + bool transcribe_files = 11; + bool perform_policy_checks_on_originals = 12; + bool perform_policy_checks_on_preservation_derivatives = 13; + + // AIP compression level (1 is the fastest, 9 is the smallest). + int32 aip_compression_level = 14; + + // AIP compression algorithm + AIPCompressionAlgorithm aip_compression_algorithm = 15; + + enum AIPCompressionAlgorithm { + UNSPECIFIED = 0; + UNCOMPRESSED = 1; // It breaks in verify_aip. + TAR = 2; // Not supported yet! + TAR_BZIP2 = 3; + TAR_GZIP = 4; + S7_COPY = 5; + S7_BZIP2 = 6; + S7_LZMA = 7; + } +} diff --git a/a3m/server/rpc/proto/a3m_pb2.py b/a3m/server/rpc/proto/a3m_pb2.py index c0405f1cc..619af8826 100644 --- a/a3m/server/rpc/proto/a3m_pb2.py +++ b/a3m/server/rpc/proto/a3m_pb2.py @@ -1,5 +1,6 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: a3m/server/rpc/proto/a3m.proto +"""Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection @@ -17,7 +18,7 @@ syntax="proto3", serialized_options=None, create_key=_descriptor._internal_create_key, - serialized_pb=b'\n\x1e\x61\x33m/server/rpc/proto/a3m.proto\x12\x03\x61\x33m"*\n\rSubmitRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0b\n\x03url\x18\x02 \x01(\t"\x19\n\x0bSubmitReply\x12\n\n\x02id\x18\x01 \x01(\t"\x19\n\x0bReadRequest\x12\n\n\x02id\x18\x01 \x01(\t"T\n\tReadReply\x12"\n\x06status\x18\x01 \x01(\x0e\x32\x12.a3m.PackageStatus\x12\x0b\n\x03job\x18\x02 \x01(\t\x12\x16\n\x04jobs\x18\x03 \x03(\x0b\x32\x08.a3m.Job""\n\x10ListTasksRequest\x12\x0e\n\x06job_id\x18\x01 \x01(\t"*\n\x0eListTasksReply\x12\x18\n\x05tasks\x18\x01 \x03(\x0b\x32\t.a3m.Task"\xa1\x01\n\x03Job\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\r\n\x05group\x18\x03 \x01(\t\x12\x0f\n\x07link_id\x18\x04 \x01(\t\x12\x1f\n\x06status\x18\x05 \x01(\x0e\x32\x0f.a3m.Job.Status"?\n\x06Status\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x0c\n\x08\x43OMPLETE\x10\x01\x12\x0e\n\nPROCESSING\x10\x02\x12\n\n\x06\x46\x41ILED\x10\x03"\x8e\x01\n\x04Task\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0f\n\x07\x66ile_id\x18\x02 \x01(\t\x12\x11\n\texit_code\x18\x03 \x01(\x05\x12\x10\n\x08\x66ilename\x18\x04 \x01(\t\x12\x11\n\texecution\x18\x05 \x01(\t\x12\x11\n\targuments\x18\x06 \x01(\t\x12\x0e\n\x06stdout\x18\x07 \x01(\t\x12\x0e\n\x06stderr\x18\x08 \x01(\t*G\n\rPackageStatus\x12\n\n\x06\x46\x41ILED\x10\x00\x12\x0c\n\x08REJECTED\x10\x01\x12\x0c\n\x08\x43OMPLETE\x10\x02\x12\x0e\n\nPROCESSING\x10\x03\x32\xa3\x01\n\x08Transfer\x12\x30\n\x06Submit\x12\x12.a3m.SubmitRequest\x1a\x10.a3m.SubmitReply"\x00\x12*\n\x04Read\x12\x10.a3m.ReadRequest\x1a\x0e.a3m.ReadReply"\x00\x12\x39\n\tListTasks\x12\x15.a3m.ListTasksRequest\x1a\x13.a3m.ListTasksReply"\x00\x62\x06proto3', + serialized_pb=b'\n\x1e\x61\x33m/server/rpc/proto/a3m.proto\x12\x03\x61\x33m"Q\n\rSubmitRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0b\n\x03url\x18\x02 \x01(\t\x12%\n\x06\x63onfig\x18\x03 \x01(\x0b\x32\x15.a3m.ProcessingConfig"\x19\n\x0bSubmitReply\x12\n\n\x02id\x18\x01 \x01(\t"\x19\n\x0bReadRequest\x12\n\n\x02id\x18\x01 \x01(\t"T\n\tReadReply\x12"\n\x06status\x18\x01 \x01(\x0e\x32\x12.a3m.PackageStatus\x12\x0b\n\x03job\x18\x02 \x01(\t\x12\x16\n\x04jobs\x18\x03 \x03(\x0b\x32\x08.a3m.Job""\n\x10ListTasksRequest\x12\x0e\n\x06job_id\x18\x01 \x01(\t"*\n\x0eListTasksReply\x12\x18\n\x05tasks\x18\x01 \x03(\x0b\x32\t.a3m.Task"\xa1\x01\n\x03Job\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\r\n\x05group\x18\x03 \x01(\t\x12\x0f\n\x07link_id\x18\x04 \x01(\t\x12\x1f\n\x06status\x18\x05 \x01(\x0e\x32\x0f.a3m.Job.Status"?\n\x06Status\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x0c\n\x08\x43OMPLETE\x10\x01\x12\x0e\n\nPROCESSING\x10\x02\x12\n\n\x06\x46\x41ILED\x10\x03"\x8e\x01\n\x04Task\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0f\n\x07\x66ile_id\x18\x02 \x01(\t\x12\x11\n\texit_code\x18\x03 \x01(\x05\x12\x10\n\x08\x66ilename\x18\x04 \x01(\t\x12\x11\n\texecution\x18\x05 \x01(\t\x12\x11\n\targuments\x18\x06 \x01(\t\x12\x0e\n\x06stdout\x18\x07 \x01(\t\x12\x0e\n\x06stderr\x18\x08 \x01(\t"\xe3\x05\n\x10ProcessingConfig\x12#\n\x1b\x61ssign_uuids_to_directories\x18\x01 \x01(\x08\x12\x18\n\x10\x65xamine_contents\x18\x02 \x01(\x08\x12*\n"generate_transfer_structure_report\x18\x03 \x01(\x08\x12"\n\x1a\x64ocument_empty_directories\x18\x04 \x01(\x08\x12\x18\n\x10\x65xtract_packages\x18\x05 \x01(\x08\x12(\n delete_packages_after_extraction\x18\x06 \x01(\x08\x12\x19\n\x11identify_transfer\x18\x07 \x01(\x08\x12(\n identify_submission_and_metadata\x18\x08 \x01(\x08\x12%\n\x1didentify_before_normalization\x18\t \x01(\x08\x12\x11\n\tnormalize\x18\n \x01(\x08\x12\x18\n\x10transcribe_files\x18\x0b \x01(\x08\x12*\n"perform_policy_checks_on_originals\x18\x0c \x01(\x08\x12\x39\n1perform_policy_checks_on_preservation_derivatives\x18\r \x01(\x08\x12\x1d\n\x15\x61ip_compression_level\x18\x0e \x01(\x05\x12P\n\x19\x61ip_compression_algorithm\x18\x0f \x01(\x0e\x32-.a3m.ProcessingConfig.AIPCompressionAlgorithm"\x8a\x01\n\x17\x41IPCompressionAlgorithm\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x10\n\x0cUNCOMPRESSED\x10\x01\x12\x07\n\x03TAR\x10\x02\x12\r\n\tTAR_BZIP2\x10\x03\x12\x0c\n\x08TAR_GZIP\x10\x04\x12\x0b\n\x07S7_COPY\x10\x05\x12\x0c\n\x08S7_BZIP2\x10\x06\x12\x0b\n\x07S7_LZMA\x10\x07*G\n\rPackageStatus\x12\n\n\x06\x46\x41ILED\x10\x00\x12\x0c\n\x08REJECTED\x10\x01\x12\x0c\n\x08\x43OMPLETE\x10\x02\x12\x0e\n\nPROCESSING\x10\x03\x32\xa3\x01\n\x08Transfer\x12\x30\n\x06Submit\x12\x12.a3m.SubmitRequest\x1a\x10.a3m.SubmitReply"\x00\x12*\n\x04Read\x12\x10.a3m.ReadRequest\x1a\x0e.a3m.ReadReply"\x00\x12\x39\n\tListTasks\x12\x15.a3m.ListTasksRequest\x1a\x13.a3m.ListTasksReply"\x00\x62\x06proto3', ) _PACKAGESTATUS = _descriptor.EnumDescriptor( @@ -62,8 +63,8 @@ ], containing_type=None, serialized_options=None, - serialized_start=612, - serialized_end=683, + serialized_start=1393, + serialized_end=1464, ) _sym_db.RegisterEnumDescriptor(_PACKAGESTATUS) @@ -116,11 +117,90 @@ ], containing_type=None, serialized_options=None, - serialized_start=402, - serialized_end=465, + serialized_start=441, + serialized_end=504, ) _sym_db.RegisterEnumDescriptor(_JOB_STATUS) +_PROCESSINGCONFIG_AIPCOMPRESSIONALGORITHM = _descriptor.EnumDescriptor( + name="AIPCompressionAlgorithm", + full_name="a3m.ProcessingConfig.AIPCompressionAlgorithm", + filename=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + values=[ + _descriptor.EnumValueDescriptor( + name="UNSPECIFIED", + index=0, + number=0, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key, + ), + _descriptor.EnumValueDescriptor( + name="UNCOMPRESSED", + index=1, + number=1, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key, + ), + _descriptor.EnumValueDescriptor( + name="TAR", + index=2, + number=2, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key, + ), + _descriptor.EnumValueDescriptor( + name="TAR_BZIP2", + index=3, + number=3, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key, + ), + _descriptor.EnumValueDescriptor( + name="TAR_GZIP", + index=4, + number=4, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key, + ), + _descriptor.EnumValueDescriptor( + name="S7_COPY", + index=5, + number=5, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key, + ), + _descriptor.EnumValueDescriptor( + name="S7_BZIP2", + index=6, + number=6, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key, + ), + _descriptor.EnumValueDescriptor( + name="S7_LZMA", + index=7, + number=7, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key, + ), + ], + containing_type=None, + serialized_options=None, + serialized_start=1253, + serialized_end=1391, +) +_sym_db.RegisterEnumDescriptor(_PROCESSINGCONFIG_AIPCOMPRESSIONALGORITHM) + _SUBMITREQUEST = _descriptor.Descriptor( name="SubmitRequest", @@ -168,6 +248,25 @@ file=DESCRIPTOR, create_key=_descriptor._internal_create_key, ), + _descriptor.FieldDescriptor( + name="config", + full_name="a3m.SubmitRequest.config", + index=2, + number=3, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + ), ], extensions=[], nested_types=[], @@ -178,7 +277,7 @@ extension_ranges=[], oneofs=[], serialized_start=39, - serialized_end=81, + serialized_end=120, ) @@ -218,8 +317,8 @@ syntax="proto3", extension_ranges=[], oneofs=[], - serialized_start=83, - serialized_end=108, + serialized_start=122, + serialized_end=147, ) @@ -259,8 +358,8 @@ syntax="proto3", extension_ranges=[], oneofs=[], - serialized_start=110, - serialized_end=135, + serialized_start=149, + serialized_end=174, ) @@ -338,8 +437,8 @@ syntax="proto3", extension_ranges=[], oneofs=[], - serialized_start=137, - serialized_end=221, + serialized_start=176, + serialized_end=260, ) @@ -379,8 +478,8 @@ syntax="proto3", extension_ranges=[], oneofs=[], - serialized_start=223, - serialized_end=257, + serialized_start=262, + serialized_end=296, ) @@ -420,8 +519,8 @@ syntax="proto3", extension_ranges=[], oneofs=[], - serialized_start=259, - serialized_end=301, + serialized_start=298, + serialized_end=340, ) @@ -531,14 +630,16 @@ ], extensions=[], nested_types=[], - enum_types=[_JOB_STATUS], + enum_types=[ + _JOB_STATUS, + ], serialized_options=None, is_extendable=False, syntax="proto3", extension_ranges=[], oneofs=[], - serialized_start=304, - serialized_end=465, + serialized_start=343, + serialized_end=504, ) @@ -711,15 +812,329 @@ syntax="proto3", extension_ranges=[], oneofs=[], - serialized_start=468, - serialized_end=610, + serialized_start=507, + serialized_end=649, ) + +_PROCESSINGCONFIG = _descriptor.Descriptor( + name="ProcessingConfig", + full_name="a3m.ProcessingConfig", + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name="assign_uuids_to_directories", + full_name="a3m.ProcessingConfig.assign_uuids_to_directories", + index=0, + number=1, + type=8, + cpp_type=7, + label=1, + has_default_value=False, + default_value=False, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + ), + _descriptor.FieldDescriptor( + name="examine_contents", + full_name="a3m.ProcessingConfig.examine_contents", + index=1, + number=2, + type=8, + cpp_type=7, + label=1, + has_default_value=False, + default_value=False, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + ), + _descriptor.FieldDescriptor( + name="generate_transfer_structure_report", + full_name="a3m.ProcessingConfig.generate_transfer_structure_report", + index=2, + number=3, + type=8, + cpp_type=7, + label=1, + has_default_value=False, + default_value=False, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + ), + _descriptor.FieldDescriptor( + name="document_empty_directories", + full_name="a3m.ProcessingConfig.document_empty_directories", + index=3, + number=4, + type=8, + cpp_type=7, + label=1, + has_default_value=False, + default_value=False, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + ), + _descriptor.FieldDescriptor( + name="extract_packages", + full_name="a3m.ProcessingConfig.extract_packages", + index=4, + number=5, + type=8, + cpp_type=7, + label=1, + has_default_value=False, + default_value=False, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + ), + _descriptor.FieldDescriptor( + name="delete_packages_after_extraction", + full_name="a3m.ProcessingConfig.delete_packages_after_extraction", + index=5, + number=6, + type=8, + cpp_type=7, + label=1, + has_default_value=False, + default_value=False, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + ), + _descriptor.FieldDescriptor( + name="identify_transfer", + full_name="a3m.ProcessingConfig.identify_transfer", + index=6, + number=7, + type=8, + cpp_type=7, + label=1, + has_default_value=False, + default_value=False, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + ), + _descriptor.FieldDescriptor( + name="identify_submission_and_metadata", + full_name="a3m.ProcessingConfig.identify_submission_and_metadata", + index=7, + number=8, + type=8, + cpp_type=7, + label=1, + has_default_value=False, + default_value=False, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + ), + _descriptor.FieldDescriptor( + name="identify_before_normalization", + full_name="a3m.ProcessingConfig.identify_before_normalization", + index=8, + number=9, + type=8, + cpp_type=7, + label=1, + has_default_value=False, + default_value=False, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + ), + _descriptor.FieldDescriptor( + name="normalize", + full_name="a3m.ProcessingConfig.normalize", + index=9, + number=10, + type=8, + cpp_type=7, + label=1, + has_default_value=False, + default_value=False, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + ), + _descriptor.FieldDescriptor( + name="transcribe_files", + full_name="a3m.ProcessingConfig.transcribe_files", + index=10, + number=11, + type=8, + cpp_type=7, + label=1, + has_default_value=False, + default_value=False, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + ), + _descriptor.FieldDescriptor( + name="perform_policy_checks_on_originals", + full_name="a3m.ProcessingConfig.perform_policy_checks_on_originals", + index=11, + number=12, + type=8, + cpp_type=7, + label=1, + has_default_value=False, + default_value=False, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + ), + _descriptor.FieldDescriptor( + name="perform_policy_checks_on_preservation_derivatives", + full_name="a3m.ProcessingConfig.perform_policy_checks_on_preservation_derivatives", + index=12, + number=13, + type=8, + cpp_type=7, + label=1, + has_default_value=False, + default_value=False, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + ), + _descriptor.FieldDescriptor( + name="aip_compression_level", + full_name="a3m.ProcessingConfig.aip_compression_level", + index=13, + number=14, + type=5, + cpp_type=1, + label=1, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + ), + _descriptor.FieldDescriptor( + name="aip_compression_algorithm", + full_name="a3m.ProcessingConfig.aip_compression_algorithm", + index=14, + number=15, + type=14, + cpp_type=8, + label=1, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + ), + ], + extensions=[], + nested_types=[], + enum_types=[ + _PROCESSINGCONFIG_AIPCOMPRESSIONALGORITHM, + ], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=652, + serialized_end=1391, +) + +_SUBMITREQUEST.fields_by_name["config"].message_type = _PROCESSINGCONFIG _READREPLY.fields_by_name["status"].enum_type = _PACKAGESTATUS _READREPLY.fields_by_name["jobs"].message_type = _JOB _LISTTASKSREPLY.fields_by_name["tasks"].message_type = _TASK _JOB.fields_by_name["status"].enum_type = _JOB_STATUS _JOB_STATUS.containing_type = _JOB +_PROCESSINGCONFIG.fields_by_name[ + "aip_compression_algorithm" +].enum_type = _PROCESSINGCONFIG_AIPCOMPRESSIONALGORITHM +_PROCESSINGCONFIG_AIPCOMPRESSIONALGORITHM.containing_type = _PROCESSINGCONFIG DESCRIPTOR.message_types_by_name["SubmitRequest"] = _SUBMITREQUEST DESCRIPTOR.message_types_by_name["SubmitReply"] = _SUBMITREPLY DESCRIPTOR.message_types_by_name["ReadRequest"] = _READREQUEST @@ -728,6 +1143,7 @@ DESCRIPTOR.message_types_by_name["ListTasksReply"] = _LISTTASKSREPLY DESCRIPTOR.message_types_by_name["Job"] = _JOB DESCRIPTOR.message_types_by_name["Task"] = _TASK +DESCRIPTOR.message_types_by_name["ProcessingConfig"] = _PROCESSINGCONFIG DESCRIPTOR.enum_types_by_name["PackageStatus"] = _PACKAGESTATUS _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -819,6 +1235,17 @@ ) _sym_db.RegisterMessage(Task) +ProcessingConfig = _reflection.GeneratedProtocolMessageType( + "ProcessingConfig", + (_message.Message,), + { + "DESCRIPTOR": _PROCESSINGCONFIG, + "__module__": "a3m.server.rpc.proto.a3m_pb2" + # @@protoc_insertion_point(class_scope:a3m.ProcessingConfig) + }, +) +_sym_db.RegisterMessage(ProcessingConfig) + _TRANSFER = _descriptor.ServiceDescriptor( name="Transfer", @@ -827,8 +1254,8 @@ index=0, serialized_options=None, create_key=_descriptor._internal_create_key, - serialized_start=686, - serialized_end=849, + serialized_start=1467, + serialized_end=1630, methods=[ _descriptor.MethodDescriptor( name="Submit", diff --git a/a3m/server/runner.py b/a3m/server/runner.py index 4021c0221..b040ca8d4 100644 --- a/a3m/server/runner.py +++ b/a3m/server/runner.py @@ -32,7 +32,6 @@ from a3m.server import shared_dirs from a3m.server.db import migrate from a3m.server.jobs import Job -from a3m.server.processing_config import validate_processing_configs from a3m.server.queues import PackageQueue from a3m.server.rpc.proto import a3m_pb2 from a3m.server.rpc.proto import a3m_pb2_grpc @@ -55,6 +54,13 @@ class ServerStage(enum.Enum): class Server: + """a3m server. + + It runs the gRPC API server and the workflow engine, using independent pools + of threads. It accepts a :class:`a3m.server.workflow.Workflow` which can be + customized as needed. + """ + def __init__( self, bind_address: str, @@ -158,8 +164,13 @@ def create_server( grpc_workers, debug=False, ): + """Create a3m server ready to use. + + It bootstraps some bits locally needed, like the database, the local + processing directory or the pool of threads. It wraps + :class:`a3m.server.runner.Server`. + """ workflow = load_default_workflow() - validate_processing_configs(workflow) shared_dirs.create() diff --git a/a3m/server/shared_dirs.py b/a3m/server/shared_dirs.py index a769e9651..76adfd9cc 100644 --- a/a3m/server/shared_dirs.py +++ b/a3m/server/shared_dirs.py @@ -9,102 +9,6 @@ logger = logging.getLogger(__name__) -DEFAULT_PROCESSING_CONFIG = """ - - - - 01c651cb-c174-4ba4-b985-1d87a44d6754 - 414da421-b83f-4648-895f-a34840e3c3f5 - - - - accea2bf-ba74-4a3a-bb97-614775c74459 - e0a39199-c62a-4a2f-98de-e9d1116460a8 - - - - 087d27be-c719-47d8-9bbb-9a7d8b609c44 - 4dec164b-79b0-4459-8505-8095af9655b5 - - - - 7509e7dc-1e1b-4dce-8d21-e130515fce73 - 612e3609-ce9a-4df6-a9a3-63d634d2d934 - - - - f19926dd-8fb5-4c79-8ade-c83f61f55b40 - 85b1e45d-8f98-4cae-8336-72f40e12cbef - - - - 82ee9ad2-2c74-4c7c-853e-e4eaf68fc8b6 - 0a24787c-00e3-4710-b324-90e792bfb484 - - - - f09847c2-ee51-429a-9478-a860477f6b8d - d97297c7-2b49-4cfe-8c9f-0613d63ed763 - - - - 56eebd45-5600-4768-a8c2-ec0114555a3d - df54fec1-dae1-4ea6-8d17-a839ee7ac4a7 - - - - 70fc7040-d4fb-4d19-a0e6-792387ca1006 - 3e891cc4-39d2-4989-a001-5107a009a223 - - - - 01d64f58-8295-4b7b-9cab-8f1b153a504f - 9475447c-9889-430c-9477-6287a9574c5b - - - - 7a024896-c4f7-4808-a240-44c87c762bc5 - 5b3c8268-5b33-4b70-b1aa-0e4540fe03d1 - - - - 153c5f41-3cfb-47ba-9150-2dd44ebc27df - b7ce05f0-9d94-4b3e-86cc-d4b2c6dba546 - - - - bd899573-694e-4d33-8c9b-df0af802437d - 2dc3f487-e4b0-4e07-a4b3-6216ed24ca14 - - - - d0dfa5fc-e3c2-4638-9eda-f96eea1070e0 - 65273f18-5b4e-4944-af4f-09be175a88e8 - - - - dec97e3c-5598-4b99-b26e-f87a435a6b7f - 01d80b27-4ad1-4bd1-8f8d-f819f18bf685 - - - -""" -BUILTIN_CONFIGS = {"default": DEFAULT_PROCESSING_CONFIG} - - -def install_builtin_config(name): - """ - Install the original version of a builtin processing configuration - """ - config = BUILTIN_CONFIGS[name] - path = os.path.join(settings.SHARED_DIRECTORY, "processingConfigs", f"{name}.xml") - if not os.path.isfile(path): - with open(path, "w") as file_descriptor: - file_descriptor.write(config) - - return config - - def create(): dirs = ( "currentlyProcessing/transfer", @@ -112,7 +16,6 @@ def create(): "completed", "failed", "policies", - "processingConfigs", "tmp", ) for dirname in dirs: @@ -121,7 +24,3 @@ def create(): continue logger.debug("Creating directory: %s", dirname) os.makedirs(dirname, mode=0o770) - - # Populate processing configurations - for config in BUILTIN_CONFIGS: - install_builtin_config(config) diff --git a/a3m/server/transfer_service.py b/a3m/server/transfer_service.py index dfc283afd..9f71b0451 100644 --- a/a3m/server/transfer_service.py +++ b/a3m/server/transfer_service.py @@ -26,6 +26,7 @@ def Submit(self, request, context): self.workflow, request.name, request.url, + request.config, ) except Exception as err: logger.warning("TransferService.Submit handler error: %s", err) diff --git a/a3m/server/workflow.py b/a3m/server/workflow.py index bf6cb1819..7a8f1624d 100644 --- a/a3m/server/workflow.py +++ b/a3m/server/workflow.py @@ -52,38 +52,26 @@ def _invert_job_statuses(): class Workflow: def __init__(self, parsed_obj): self._src = parsed_obj - self._decode_chains() self._decode_links() def __str__(self): - return "Chains {}, links {}".format(len(self.chains), len(self.links)) - - def _decode_chains(self): - self.chains = {} - for chain_id, chain_obj in self._src["chains"].items(): - self.chains[chain_id] = Chain(chain_id, chain_obj, self) + return "Links {}".format(len(self.links)) def _decode_links(self): self.links = {} for link_id, link_obj in self._src["links"].items(): self.links[link_id] = Link(link_id, link_obj, self) - def get_chains(self): - return self.chains - def get_links(self): return self.links - def get_chain(self, chain_id): - return self.chains[chain_id] - def get_link(self, link_id): return self.links[link_id] - def get_initiator_chain(self): - for _, chain in self.chains.items(): - if chain.initiator: - return chain + def get_initiator(self): + for _, link in self.links.items(): + if link.is_initiator: + return link class BaseLink: @@ -106,31 +94,6 @@ def workflow(self): return self._workflow -class Chain(BaseLink): - def __init__(self, id_, attrs, workflow): - self.id = id_ - self._src = attrs - self._workflow = workflow - self._decode_translations() - - def __repr__(self): - return f"Chain <{self.id}>" - - def __getitem__(self, key): - return self._src[key] - - def _decode_translations(self): - self._src["description"] = self._decode_translation(self._src["description"]) - - @property - def link(self): - return self._workflow.get_link(self._src["link_id"]) - - @property - def initiator(self): - return self._src.get("start", False) - - class Link(BaseLink): def __init__(self, id_, attrs, workflow): self.id = id_ @@ -169,6 +132,11 @@ def _decode_translations(self): def config(self): return self._src["config"] + @property + def is_initiator(self): + """Check if the link is indicated as an initiator link.""" + return self._src.get("start", False) + @property def is_terminal(self): """Check if the link is indicated as a terminal link.""" diff --git a/docs/_static/custom.css b/docs/_static/custom.css new file mode 100644 index 000000000..b4ccbf8ec --- /dev/null +++ b/docs/_static/custom.css @@ -0,0 +1,7 @@ +#reference .py { + margin-bottom: 1em !important; +} + +#reference .sig-param { + color: #666 !important; +} diff --git a/docs/changelog.rst b/docs/changelog.rst index 6662a437b..9cacac781 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,8 @@ Changelog ========= +- :release:`0.5.0 <2020-10-27>` +- :feature:`51` Removed reingest capabilities. - :release:`0.4.0 <2020-10-20>` - :feature:`-` Removed UnitVariable links. - :feature:`-` Removed access normalization paths. diff --git a/docs/development.rst b/docs/development.rst new file mode 100644 index 000000000..8c7b8ae5d --- /dev/null +++ b/docs/development.rst @@ -0,0 +1,66 @@ +=========== +Development +=========== + +Python SDK +---------- + +You may have already learned that a3m comes with two executables: **a3m** and +**a3md**. These are command-line interfaces wrapping a number of Python +abstractions that we are also making available to software developers planning +to build new applications embedding or communicating with a3m. + +:func:`a3m.server.runner.create_server` is a function that helps you create your +own instance of :class:`a3m.server.runner.Server`, the gRPC server. + +Use :class:`a3m.server.rpc.client.Client` to communicate with it. +:class:`a3m.cli.client.wrapper.ClientWrapper` is a context manager that makes +easier to access to both an embedded server and its client instance. + +For more details, see: https://gist.github.com/sevein/2e5cf115c153df1cfc24f0f9d67f6d2a. + +.. warning:: + + These APIs are still unstable, expect changes! + +The following is an example of a web application that uses the development kit +to embed a3m and make it available to web clients. + +.. literalinclude:: ../examples/webapp.py + + +gRPC API +-------- + +Whether you are embedding a3m or communicating with remote instances, its gRPC +API is the underlying communication system and you should be able to put it in +practice given any of the languages supported by the `gRPC +stack `_. + +gRPC uses Protocol Buffers as the Interface Definition Language (IDL) for +describing both the service interface and the structure of the payload messages. + +So far the whole definition of messages and services fits in a single file that +we share below. Writing your custom client isn't hard because the stubs are +automatically generated. Alternatively, it is possible to use a client such as +`grpccurl `_ which dynamically browses +our service schema. + +.. _idl: + +.. literalinclude:: ../a3m/server/rpc/proto/a3m.proto + :language: protobuf + + +Reference +--------- + +.. autofunction:: a3m.server.runner.create_server + +.. autoclass:: a3m.server.runner.Server + :undoc-members: + +.. autoclass:: a3m.server.rpc.client.Client + :undoc-members: + +.. autoclass:: a3m.cli.client.wrapper.ClientWrapper diff --git a/docs/docker.rst b/docs/docker.rst index 880469623..fbfd0d136 100644 --- a/docs/docker.rst +++ b/docs/docker.rst @@ -5,8 +5,8 @@ Docker Our Docker image is extremely convenient because it includes many software dependencies that you would need to install manually otherwise. -User our Docker image as it is -============================== +Using our Docker image +====================== Off the shelf, the Docker image brings an environment with a3m and its dependencies installed and ready to use. Below is an example of using the @@ -44,8 +44,8 @@ Don't forget to clean up before leaving:: separately. -Building a custom Docker image -============================== +Custom images +============= Our image ``ghcr.io/artefactual-labs/a3m`` can be used as a parent image. Say you're building a new application embedding a3m and you need a few extra diff --git a/docs/index.rst b/docs/index.rst index e22c9c439..26b319b7b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -14,4 +14,5 @@ Current release: |version|. installation usage settings + development docker diff --git a/docs/overview.rst b/docs/overview.rst index dcb119c1c..c38c6523c 100644 --- a/docs/overview.rst +++ b/docs/overview.rst @@ -22,7 +22,7 @@ roadmap has not been set in stone, but the authors have plans. A proof of concept was developed to demonstrate some of the design principles like integrability or simplicity. However, the arfifacts that a3m produces have not been thoughtfully verified yet and there are some known issues. - + * **Operability.** Producing standalone executables and integrating with cloud-based environments. Dependency management. @@ -125,7 +125,7 @@ configuration steps. cli(gRPC
client) srv(gRPC
server) we(Workflow
engine) - + cli-- operates -->srv srv-- controls -->we end @@ -140,8 +140,8 @@ Embedded mode ------------- A design principle in a3m is *composability*. It wants to become a building -block for system integrators. Please read the :ref:`development kit usage -page` to learn how to use our programming interfaces. +block for system integrators. Please read the :doc:`development kit usage +page` to learn how to use our programming interfaces. There are multiple use cases. For example, you may want to build an application that connects to a message broker used to receive preservation requests and @@ -182,4 +182,4 @@ a3m and control it via its API. a3m(a3m) ele-- gRPC -->a3m style a3m fill:#fff - end \ No newline at end of file + end diff --git a/docs/settings.rst b/docs/settings.rst index 393c3513c..6c0440cda 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -1,17 +1,58 @@ Settings ======== -Users can provide settings via the ``/etc/a3m/a3m.conf`` configuration file, -e.g.:: +Users can provide service settings via the ``/etc/a3m/a3m.conf`` configuration +file, e.g.:: [a3m] debug = False Environment strings are also supported and they are evaluated last, e.g.:: - env A3M_DEBUG=yes + env A3M_DEBUG=yes a3m ... -The full list of environment strings can be found in the -``a3m.settings.common`` module. This module can be customized as needed. +Configuration settings are not properly described yet, but here's the list: + +* ``debug`` (boolean) +* ``batch_size`` (int) +* ``concurrent_packages`` (int) +* ``rpc_threads`` (int) +* ``worker_threads`` (int) +* ``shared_directory`` (string) +* ``temp_directory`` (string) +* ``processing_directory`` (string) +* ``rejected_directory`` (string) +* ``capture_client_script_output`` (boolean) +* ``removable_files`` (string) +* ``secret_key`` (string) +* ``prometheus_bind_address`` (string) +* ``prometheus_bind_port`` (string) +* ``time_zone`` (string) +* ``clamav_server`` (string) +* ``clamav_pass_by_stream`` (boolean) +* ``clamav_client_timeout`` (float) +* ``clamav_client_backend`` (string) +* ``clamav_client_max_file_size`` (float) +* ``clamav_client_max_scan_size`` (float) +* ``virus_scanning_enabled`` (boolean) +* ``db_engine`` (string) +* ``db_name`` (string) +* ``db_user`` (string) +* ``db_password`` (string) +* ``db_host`` (string) +* ``db_port`` (string) +* ``rpc_bind_address`` (string) +* ``s3_enabled`` (boolean) +* ``s3_endpoint_url`` (string) +* ``s3_region_name`` (string) +* ``s3_access_key_id`` (string) +* ``s3_secret_access_key`` (string) +* ``s3_use_ssl`` (boolean) +* ``s3_addressing_style`` (string) +* ``s3_signature_version`` (string) +* ``s3_bucket`` (string) + +For greater flexibility, it is also possible to alter the applicatin settings +module manually. This is how our :mod:`a3m.settings.common` module looks like: .. literalinclude:: ../a3m/settings/common.py diff --git a/docs/usage.rst b/docs/usage.rst index 4f19db9ae..26d750892 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -51,57 +51,28 @@ and these are its contents:: ├── failed │   ├── 0d117bed-2124-48a2-b9d7-f32514d39c1e ├── policies - ├── processingConfigs - │   └── default.xml └── tmp -.. _devkit: +Processing configuration +------------------------ -Development kit ---------------- +a3m abandons the XML-based processing configuration document used by +Archivematica. Instead, users are asked to submit the configuration as part +of their transfer requests. -**a3m** and **a3md** are both command-line interfaces wrapping a number of -abstractions that are also available to Python developers. This is useful if -you are planning to build an application embedding a3m, e.g. a processing -worker that receives tasks off a message queue like Redis. +With our client, ``--processing-config`` can be used multiple times to indicate +the desired settings:: -:func:`a3m.server.runner.create_server` is a function that helps you create your -own instance of :class:`a3m.server.runner.Server`, the gRPC server. + a3m --name="Test" --processing-config="normalize=no" http://... -Use :class:`a3m.server.rpc.client.Client` to communicate with it. -:class:`a3m.cli.client.wrapper.ClientWrapper` is a context manager that makes -easier to access to both an embedded server and its client instance. +The Python client can do similarly:: -For more details, see: https://gist.github.com/sevein/2e5cf115c153df1cfc24f0f9d67f6d2a. + c = Client(...) + c.submit( + url="URL...", name="Name...", + config=a3m_pb2.ProcessingConfig(normalize=False)) -.. warning:: - - These APIs are still unstable, expect changes! - -The following is an example of a web application that uses the development kit -to embed a3m and make it available to web clients. - -.. literalinclude:: ../examples/webapp.py - -Service definition ------------------- - -gRPC uses Protocol Buffers as the Interface Definition Language (IDL) for -describing both the service interface and the structure of the payload messages. - -.. literalinclude:: ../a3m/server/rpc/proto/a3m.proto - :language: protobuf - -Reference ---------- - -.. autofunction:: a3m.server.runner.create_server - -.. autoclass:: a3m.server.runner.Server - :undoc-members: - -.. autoclass:: a3m.server.rpc.client.Client - :undoc-members: - -.. autoclass:: a3m.cli.client.wrapper.ClientWrapper +The full list of settings or their defaults are not described yet but it can be +found in our :ref:`service definition file `, under the +``ProcessingConfiguration`` message type. diff --git a/requirements-dev.txt b/requirements-dev.txt index 6b7a6c2ca..523afdd6b 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -12,8 +12,8 @@ attrs==20.2.0 # via -r requirements.txt, jsonschema, pytest babel==2.8.0 # via sphinx bagit==1.7.0 # via -r requirements.txt black==20.8b1 # via -r requirements-dev.in -boto3==1.16.0 # via -r requirements.txt -botocore==1.19.0 # via -r requirements.txt, boto3, s3transfer +boto3==1.16.4 # via -r requirements.txt +botocore==1.19.4 # via -r requirements.txt, boto3, s3transfer certifi==2020.6.20 # via -r requirements.txt, requests cfgv==3.2.0 # via pre-commit chardet==3.0.4 # via -r requirements.txt, requests @@ -29,10 +29,10 @@ filelock==3.0.12 # via tox, virtualenv flake8==3.8.4 # via -r requirements-dev.in future==0.18.2 # via -r requirements.txt, metsrw googleapis-common-protos==1.52.0 # via -r requirements.txt, grpcio-status -grpcio-reflection==1.32.0 # via -r requirements.txt -grpcio-status==1.32.0 # via -r requirements.txt -grpcio-tools==1.32.0 # via -r requirements-dev.in -grpcio==1.32.0 # via -r requirements.txt, grpcio-reflection, grpcio-status, grpcio-tools +grpcio-reflection==1.33.1 # via -r requirements.txt +grpcio-status==1.33.1 # via -r requirements.txt +grpcio-tools==1.33.1 # via -r requirements-dev.in +grpcio==1.33.1 # via -r requirements.txt, grpcio-reflection, grpcio-status, grpcio-tools identify==1.5.6 # via pre-commit idna==2.10 # via -r requirements.txt, requests, yarl imagesize==1.2.0 # via sphinx @@ -62,21 +62,21 @@ protobuf==3.13.0 # via -r requirements.txt, googleapis-common-protos, g py==1.9.0 # via pytest, tox pycodestyle==2.6.0 # via flake8 pyflakes==2.2.0 # via flake8 -pygments==2.7.1 # via -r requirements.txt, rich, sphinx +pygments==2.7.2 # via -r requirements.txt, rich, sphinx pylint==2.6.0 # via -r requirements-dev.in pyparsing==2.4.7 # via packaging pyrsistent==0.17.3 # via -r requirements.txt, jsonschema pytest-cov==2.10.1 # via -r requirements-dev.in -pytest-django==4.0.0 # via -r requirements-dev.in +pytest-django==4.1.0 # via -r requirements-dev.in pytest-mock==3.3.1 # via -r requirements-dev.in pytest==6.1.1 # via -r requirements-dev.in, pytest-cov, pytest-django, pytest-mock python-dateutil==2.8.1 # via -r requirements.txt, botocore pytz==2020.1 # via -r requirements.txt, babel, django pyyaml==5.3.1 # via pre-commit, vcrpy -regex==2020.10.15 # via black +regex==2020.10.23 # via black releases==1.6.3 # via -r requirements-dev.in requests==2.24.0 # via -r requirements.txt, sphinx -rich==9.0.1 # via -r requirements.txt +rich==9.1.0 # via -r requirements.txt s3transfer==0.3.3 # via -r requirements.txt, boto3 semantic-version==2.6.0 # via releases six==1.15.0 # via -r requirements.txt, astroid, grpcio, jsonschema, metsrw, opf-fido, packaging, pip-tools, protobuf, python-dateutil, tenacity, tox, vcrpy, virtualenv @@ -98,7 +98,7 @@ typing-extensions==3.7.4.3 # via -r requirements.txt, black, mypy, rich unidecode==1.1.1 # via -r requirements.txt urllib3==1.25.11 # via -r requirements.txt, botocore, requests vcrpy==4.1.1 # via -r requirements-dev.in -virtualenv==20.0.35 # via pre-commit, tox +virtualenv==20.1.0 # via pre-commit, tox vulture==2.1 # via -r requirements-dev.in wrapt==1.12.1 # via astroid, vcrpy yarl==1.6.2 # via vcrpy diff --git a/requirements.txt b/requirements.txt index 56c52c847..1e451d2d7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,8 +8,8 @@ ammcpc==0.1.3 # via -r requirements.in appdirs==1.4.4 # via -r requirements.in attrs==20.2.0 # via jsonschema bagit==1.7.0 # via -r requirements.in -boto3==1.16.0 # via -r requirements.in -botocore==1.19.0 # via boto3, s3transfer +boto3==1.16.4 # via -r requirements.in +botocore==1.19.4 # via boto3, s3transfer certifi==2020.6.20 # via requests chardet==3.0.4 # via requests clamd==1.0.2 # via -r requirements.in @@ -19,9 +19,9 @@ commonmark==0.9.1 # via rich django==2.2.16 # via -r requirements.in future==0.18.2 # via metsrw googleapis-common-protos==1.52.0 # via -r requirements.in, grpcio-status -grpcio-reflection==1.32.0 # via -r requirements.in -grpcio-status==1.32.0 # via -r requirements.in -grpcio==1.32.0 # via -r requirements.in, grpcio-reflection, grpcio-status +grpcio-reflection==1.33.1 # via -r requirements.in +grpcio-status==1.33.1 # via -r requirements.in +grpcio==1.33.1 # via -r requirements.in, grpcio-reflection, grpcio-status idna==2.10 # via requests jmespath==0.10.0 # via boto3, botocore jsonschema==3.2.0 # via -r requirements.in @@ -31,12 +31,12 @@ olefile==0.46 # via opf-fido opf-fido==1.4.1 # via -r requirements.in prometheus-client==0.8.0 # via -r requirements.in protobuf==3.13.0 # via googleapis-common-protos, grpcio-reflection, grpcio-status -pygments==2.7.1 # via rich +pygments==2.7.2 # via rich pyrsistent==0.17.3 # via jsonschema python-dateutil==2.8.1 # via botocore pytz==2020.1 # via django requests==2.24.0 # via -r requirements.in -rich==9.0.1 # via -r requirements.in +rich==9.1.0 # via -r requirements.in s3transfer==0.3.3 # via boto3 six==1.15.0 # via grpcio, jsonschema, metsrw, opf-fido, protobuf, python-dateutil, tenacity sqlparse==0.4.1 # via django diff --git a/setup.cfg b/setup.cfg index deee007cf..6db71f169 100644 --- a/setup.cfg +++ b/setup.cfg @@ -48,7 +48,7 @@ install_requires = requests~=2.24 appdirs~=1.4 click~=7.1 - rich~=5.2 + rich~=9.0 tenacity~=6.2 boto3~=1.14 jsonschema~=3.2 @@ -76,6 +76,8 @@ exclude = .tox, .git, __pycache__, .cache, build, dist, *.pyc, *.egg-info, .eggs application-import-names = flake8 select = C, E, F, W, B, B950 ignore = E203, E402, E501, E722, W503, W605 +per-file-ignores = + *.pyi: F401 [coverage:run] diff --git a/tests/server/fixtures/workflow-integration-test.json b/tests/server/fixtures/workflow-integration-test.json index 5bcf7e306..52264d98e 100644 --- a/tests/server/fixtures/workflow-integration-test.json +++ b/tests/server/fixtures/workflow-integration-test.json @@ -1,30 +1,8 @@ { - "chains": { - "3816f689-65a8-4ad0-ac27-74292a70b093": { - "description": { - "en": "Chain 1" - }, - "link_id": "3b5dd6a5-b951-4e44-b00d-1180e5557beb", - "start": true - }, - "7b814362-c679-43c4-a2e2-1ba59957cd18": { - "description": { - "en": "Chain 2 (choice 1)" - }, - "link_id": "de6eb412-0029-4dbd-9bfa-7311697d6012" - }, - "f8c952a2-d7bc-4851-8613-2935726e2b9d": { - "description": { - "en": "Chain 3 (choice 2)" - }, - "link_id": "f8e4c1ee-3e43-4caa-a664-f6b6bd8f156e" - } - }, "links": { "3b5dd6a5-b951-4e44-b00d-1180e5557beb": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "execute": "link1_command", "arguments": "\"%processingDirectory%\" \"%TransferUUID%\"", "filter_file_start": null, @@ -46,12 +24,12 @@ "fallback_link_id": null, "group": { "en": "Group 1" - } + }, + "start": true }, "47bf2a2c-8d72-4f36-96d0-53b53a2bbc3f": { "config": { "@manager": "linkTaskManagerFiles", - "@model": "StandardTaskConfig", "execute": "link2_command", "arguments": "\"%fileUUID%\"", "filter_file_start": null, @@ -78,7 +56,6 @@ "5678bbab-c0ea-4b3c-9de9-addc92d0de50": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "execute": "link3_command", "arguments": "\"%processingDirectory%\" \"%SIPDirectory%\" \"%SIPName%-%TransferUUID%\"", "filter_file_start": null, @@ -105,7 +82,6 @@ "c38f7b32-6f0c-48a5-a5f6-6dbe97ca75ba": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "execute": "link4_command", "arguments": "\"%processingDirectory%\" \"%SIPDirectory%\" \"%SIPName%-%TransferUUID%\"", "filter_file_start": null, @@ -132,10 +108,11 @@ "d875dcf3-5e0e-4546-a66d-b2580c7a1a75": { "config": { "@manager": "linkTaskManagerChoice", - "@model": "MicroServiceChainChoice", - "chain_choices": [ - "7b814362-c679-43c4-a2e2-1ba59957cd18", - "f8c952a2-d7bc-4851-8613-2935726e2b9d" + "config_attr": "run_fifth_link", + "default": true, + "choices": [ + {"value": true, "link_id": "de6eb412-0029-4dbd-9bfa-7311697d6012"}, + {"value": false, "link_id": "f8e4c1ee-3e43-4caa-a664-f6b6bd8f156e"} ] }, "description": { @@ -150,31 +127,17 @@ }, "de6eb412-0029-4dbd-9bfa-7311697d6012": { "config": { - "@manager": "linkTaskManagerReplacementDicFromChoice", - "@model": "MicroServiceChoiceReplacementDic", - "replacements": [ - { - "description": { - "en": "Yes" - }, - "id": "51e395b9-1b74-419c-b013-3283b7fe39ff", - "items": { - "TestValue": "7" - } - }, - { - "description": { - "en": "No" - }, - "id": "5b6615b8-a54e-40ef-955d-d8af01797b7b", - "items": { - "TestValue": "3" - } - } - ] + "@manager": "linkTaskManagerDirectories", + "execute": "link6_command", + "arguments": "\"%SIPName%\"", + "filter_file_start": null, + "filter_file_end": null, + "filter_subdir": null, + "stdout_file": null, + "stderr_file": null }, "description": { - "en": "Chain 2 first link" + "en": "Sixth link" }, "exit_codes": { "0": { @@ -191,7 +154,6 @@ "f8e4c1ee-3e43-4caa-a664-f6b6bd8f156e": { "config": { "@manager": "linkTaskManagerDirectories", - "@model": "StandardTaskConfig", "execute": "final_command", "arguments": "\"%processingDirectory%\" \"%SIPDirectory%\" \"%SIPName%-%TransferUUID%\"", "filter_file_start": null, diff --git a/tests/server/fixtures/workflow-sample.json b/tests/server/fixtures/workflow-sample.json deleted file mode 100644 index 403590dcd..000000000 --- a/tests/server/fixtures/workflow-sample.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "chains": { - "03df8d8c-34d3-4924-86d4-f4e706c1522b": { - "description": { - "en": "Chain test" - }, - "link_id": "03df8d8c-34d3-4924-86d4-f4e706c1522b", - "start": true - } - }, - "links": { - "03df8d8c-34d3-4924-86d4-f4e706c1522b": { - "config": { - "@manager": "linkTaskManagerFiles", - "@model": "StandardTaskConfig", - "execute": "command", - "arguments": "--debug", - "filter_file_start": null, - "filter_file_end": null, - "filter_subdir": null, - "stdout_file": null, - "stderr_file": null - }, - "description": { - "en": "Go!" - }, - "exit_codes": { - "0": { - "job_status": "Completed successfully", - "link_id": null - } - }, - "fallback_job_status": "Failed", - "fallback_link_id": null, - "group": { - "en": "Group" - } - } - } -} diff --git a/tests/server/test_integration.py b/tests/server/test_integration.py index a6daed1ef..23e953467 100644 --- a/tests/server/test_integration.py +++ b/tests/server/test_integration.py @@ -2,40 +2,24 @@ import os import threading import uuid -from io import StringIO import pytest from django.utils import timezone -from lxml import etree from a3m.main import models from a3m.server.jobs import DirectoryClientScriptJob from a3m.server.jobs import FilesClientScriptJob from a3m.server.jobs import JobChain -from a3m.server.jobs import NextChainDecisionJob -from a3m.server.jobs import UpdateContextDecisionJob +from a3m.server.jobs import NextLinkDecisionJob from a3m.server.packages import Package from a3m.server.queues import PackageQueue +from a3m.server.rpc.proto.a3m_pb2 import ProcessingConfig from a3m.server.tasks import TaskBackend from a3m.server.workflow import load as load_workflow FIXTURES_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures") INTEGRATION_TEST_PATH = os.path.join(FIXTURES_DIR, "workflow-integration-test.json") -TEST_PROCESSING_CONFIG = etree.parse( - StringIO( - """ - - - - de6eb412-0029-4dbd-9bfa-7311697d6012 - 51e395b9-1b74-419c-b013-3283b7fe39ff - - - -""" - ) -) class EchoBackend(TaskBackend): @@ -80,6 +64,7 @@ def package(request): return Package( "package-1", "file:///tmp/foobar-1.gz", + ProcessingConfig(), models.Transfer.objects.create(pk=uuid.uuid4()), models.SIP.objects.create(pk=uuid.uuid4()), ) @@ -121,17 +106,11 @@ def test_workflow_integration( mock_get_task_backend = mocker.patch( "a3m.server.jobs.client.get_task_backend", return_value=echo_backend ) - mock_load_preconfigured_choice = mocker.patch( - "a3m.server.jobs.decisions.load_preconfigured_choice" - ) - mock_load_processing_xml = mocker.patch( - "a3m.server.jobs.decisions.load_processing_xml" - ) mocker.patch.object(package, "files", return_value=dummy_file_replacements) # Schedule the first job - first_workflow_chain = workflow.get_chains()["3816f689-65a8-4ad0-ac27-74292a70b093"] - first_job_chain = JobChain(package, first_workflow_chain, workflow) + initiator_link = workflow.get_initiator() + first_job_chain = JobChain(package, workflow, initiator_link) job = next(first_job_chain) package_queue.schedule_job(job) @@ -183,7 +162,7 @@ def test_workflow_integration( assert package_queue.job_queue.qsize() == 1 job = future.result() - # Process the fourth job (OutputDecisionJob) + # Process the fourth job (DirectoryClientScriptJob) future = package_queue.process_one_job(timeout=1.0) concurrent.futures.wait([future], timeout=1.0) @@ -194,35 +173,21 @@ def test_workflow_integration( assert package_queue.job_queue.qsize() == 1 job = future.result() - # Setup preconfigured choice for next job - mock_load_preconfigured_choice.return_value = "7b814362-c679-43c4-a2e2-1ba59957cd18" - - # Process the fifth job (NextChainDecisionJob) + # Process the fifth job (NextLinkDecisionJob) future = package_queue.process_one_job(timeout=1.0) concurrent.futures.wait([future], timeout=1.0) - assert isinstance(job, NextChainDecisionJob) + assert isinstance(job, NextLinkDecisionJob) assert job.exit_code == 0 # Next job in chain should be queued assert package_queue.job_queue.qsize() == 1 job = future.result() - # We should be on chain 2 now - assert job.job_chain is not first_job_chain - assert job.job_chain.chain.id == "7b814362-c679-43c4-a2e2-1ba59957cd18" - - # Setup preconfigured choice for next job - mock_load_processing_xml.return_value = TEST_PROCESSING_CONFIG - # Process the sixth job (UpdateContextDecisionJob) future = package_queue.process_one_job(timeout=1.0) concurrent.futures.wait([future], timeout=1.0) - assert isinstance(job, UpdateContextDecisionJob) - assert job.exit_code == 0 - assert job.job_chain.context[r"%TestValue%"] == "7" - # Out job chain should have been redirected to the final link assert job.job_chain.current_link.id == "f8e4c1ee-3e43-4caa-a664-f6b6bd8f156e" assert package_queue.job_queue.qsize() == 1 diff --git a/tests/server/test_package.py b/tests/server/test_package.py index f5dfb846b..85152d875 100644 --- a/tests/server/test_package.py +++ b/tests/server/test_package.py @@ -9,6 +9,7 @@ from a3m.main import models from a3m.server.packages import Package from a3m.server.queues import PackageQueue +from a3m.server.rpc.proto.a3m_pb2 import ProcessingConfig from a3m.server.workflow import load as load_workflow FIXTURES_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures") @@ -30,7 +31,12 @@ def package_queue(request): @pytest.fixture def package(request, db, package_queue, workflow): return Package.create_package( - package_queue, package_queue.executor, workflow, "name", "file:///tmp/foobar.gz" + package_queue, + package_queue.executor, + workflow, + "name", + "file:///tmp/foobar.gz", + ProcessingConfig(), ) diff --git a/tests/server/test_queues.py b/tests/server/test_queues.py index 9afefcd07..448bfac91 100644 --- a/tests/server/test_queues.py +++ b/tests/server/test_queues.py @@ -9,6 +9,7 @@ from a3m.server.jobs import Job from a3m.server.packages import Package from a3m.server.queues import PackageQueue +from a3m.server.rpc.proto.a3m_pb2 import ProcessingConfig from a3m.server.workflow import Link @@ -70,14 +71,22 @@ def __init__(self, pk): @pytest.fixture def package(request): return Package( - "package-1", "file:///tmp/foobar-1.gz", FakeUnit("abc"), FakeUnit("def") + "package-1", + "file:///tmp/foobar-1.gz", + ProcessingConfig(), + FakeUnit("abc"), + FakeUnit("def"), ) @pytest.fixture def package_2(request): return Package( - "package-2", "file:///tmp/foobar-2.gz", FakeUnit("ghi"), FakeUnit("jkl") + "package-2", + "file:///tmp/foobar-2.gz", + ProcessingConfig(), + FakeUnit("ghi"), + FakeUnit("jkl"), ) diff --git a/tests/server/test_workflow.py b/tests/server/test_workflow.py index 208d180f2..6d3e97609 100644 --- a/tests/server/test_workflow.py +++ b/tests/server/test_workflow.py @@ -48,24 +48,13 @@ def test_load_invalid_json(): "path", ( os.path.join(ASSETS_DIR, "workflow.json"), - os.path.join(FIXTURES_DIR, "workflow-sample.json"), + os.path.join(FIXTURES_DIR, "workflow-integration-test.json"), ), ) def test_load_valid_document(path): with open(path) as fp: wf = workflow.load(fp) - chains = wf.get_chains() - assert len(chains) > 0 - first_chain = next(iter(chains.values())) - assert isinstance(first_chain, workflow.Chain) - assert str(first_chain) == first_chain.id - assert repr(first_chain) == f"Chain <{first_chain.id}>" - assert isinstance(first_chain.link, workflow.Link) - assert isinstance(first_chain.link, workflow.BaseLink) - assert isinstance(first_chain["description"], workflow.TranslationLabel) - assert first_chain["description"]._src == first_chain._src["description"]._src - links = wf.get_links() assert len(links) > 0 first_link = next(iter(links.values())) @@ -74,7 +63,7 @@ def test_load_valid_document(path): assert first_link.config == first_link._src["config"] # Workflow __str__ method - assert str(wf) == "Chains {}, links {}".format(len(chains), len(links)) + assert str(wf) == "Links {}".format(len(links)) # Test normalization of job statuses. link = next(iter(links.values())) @@ -103,7 +92,7 @@ def test_link_browse_methods(mocker): def test_get_schema(): schema = workflow._get_schema() - assert schema["$id"] == "https://www.archivematica.org/labs/workflow/schema/v1.json" + assert schema["$id"] == "https://a3m.readthedocs.io/workflow/schema/v1.json" def test_get_schema_not_found(mocker):