diff --git a/src/MCPClient/lib/clientScripts/archivematicaCreateMETSMetadataXML.py b/src/MCPClient/lib/clientScripts/archivematicaCreateMETSMetadataXML.py index f9f8d3b39b..d8dd762c1e 100755 --- a/src/MCPClient/lib/clientScripts/archivematicaCreateMETSMetadataXML.py +++ b/src/MCPClient/lib/clientScripts/archivematicaCreateMETSMetadataXML.py @@ -26,6 +26,7 @@ import namespaces as ns import requests from databaseFunctions import insertIntoEvents +from django.core.exceptions import ValidationError from importlib_metadata import version from lxml import etree from main import models @@ -62,7 +63,7 @@ def process_xml_metadata(mets, sip_dir, sip_uuid, sip_type, xml_validation): sip_id=sip_uuid, currentlocation=f"%SIPDirectory%{xml_rel_path}".encode(), ) - except models.File.DoesNotExist: + except (models.File.DoesNotExist, ValidationError): xml_metadata_errors.append(f"No uuid for file: {xml_rel_path}") continue valid, errors = _validate_xml(tree, schema_uri) diff --git a/src/MCPClient/lib/clientScripts/archivematicaCreateMETSReingest.py b/src/MCPClient/lib/clientScripts/archivematicaCreateMETSReingest.py index a3b6d2c5d3..468b303a51 100755 --- a/src/MCPClient/lib/clientScripts/archivematicaCreateMETSReingest.py +++ b/src/MCPClient/lib/clientScripts/archivematicaCreateMETSReingest.py @@ -7,6 +7,7 @@ import create_mets_v2 as createmets2 import metsrw import namespaces as ns +from django.core.exceptions import ValidationError from lxml import etree from main import models @@ -562,12 +563,12 @@ def update_metadata_csv(job, mets, metadata_csv, sip_uuid, sip_dir, state): file_obj = models.File.objects.get( sip_id=str(sip_uuid), originallocation__endswith="%" + f ) - except models.File.DoesNotExist: + except (models.File.DoesNotExist, ValidationError): try: file_obj = models.File.objects.get( sip_id=sip_uuid, currentlocation__endswith="%" + f ) - except models.File.DoesNotExist: + except (models.File.DoesNotExist, ValidationError): pass if file_obj is not None: fsentry = mets.get_file(file_uuid=str(file_obj.uuid)) diff --git a/src/MCPClient/lib/clientScripts/archivematicaCreateMETSRightsDspaceMDRef.py b/src/MCPClient/lib/clientScripts/archivematicaCreateMETSRightsDspaceMDRef.py index ef9bc401bb..84a9665ad5 100755 --- a/src/MCPClient/lib/clientScripts/archivematicaCreateMETSRightsDspaceMDRef.py +++ b/src/MCPClient/lib/clientScripts/archivematicaCreateMETSRightsDspaceMDRef.py @@ -24,6 +24,7 @@ import lxml.etree as etree import namespaces as ns +from django.core.exceptions import ValidationError from main.models import File # dashboard @@ -64,7 +65,7 @@ def archivematicaCreateMETSRightsDspaceMDRef( mets = File.objects.get( currentlocation=path.encode(), transfer_id=transferUUID ) - except File.DoesNotExist: + except (File.DoesNotExist, ValidationError): pass else: metsFileUUID = mets.uuid @@ -91,7 +92,7 @@ def archivematicaCreateMETSRightsDspaceMDRef( f = File.objects.get( currentlocation=path.encode(), transfer_id=transferUUID ) - except File.DoesNotExist: + except (File.DoesNotExist, ValidationError): pass else: metsFileUUID = f.uuid diff --git a/src/MCPClient/lib/clientScripts/archivematicaCreateMETSTrim.py b/src/MCPClient/lib/clientScripts/archivematicaCreateMETSTrim.py index 82dfa4bf2d..aceecb7e30 100755 --- a/src/MCPClient/lib/clientScripts/archivematicaCreateMETSTrim.py +++ b/src/MCPClient/lib/clientScripts/archivematicaCreateMETSTrim.py @@ -26,6 +26,7 @@ import lxml.etree as etree import namespaces as ns +from django.core.exceptions import ValidationError from main.models import File # dashboard @@ -124,7 +125,7 @@ def getTrimFileDmdSec(job, baseDirectoryPath, sipUUID, fileUUID): filegrpuuid=fileUUID, filegrpuse="TRIM file metadata", ) - except File.DoesNotExist: + except (File.DoesNotExist, ValidationError): job.pyprint("no metadata for original file: ", fileUUID, file=sys.stderr) return None else: @@ -163,7 +164,7 @@ def getTrimFileAmdSec(job, baseDirectoryPath, sipUUID, fileUUID): filegrpuuid=fileUUID, filegrpuse="TRIM file metadata", ) - except File.DoesNotExist: + except (File.DoesNotExist, ValidationError): job.pyprint("no metadata for original file: ", fileUUID, file=sys.stderr) return None else: diff --git a/src/MCPClient/lib/clientScripts/archivematica_clamscan.py b/src/MCPClient/lib/clientScripts/archivematica_clamscan.py index 8869773b09..0802f32046 100755 --- a/src/MCPClient/lib/clientScripts/archivematica_clamscan.py +++ b/src/MCPClient/lib/clientScripts/archivematica_clamscan.py @@ -39,6 +39,7 @@ from custom_handlers import get_script_logger from databaseFunctions import insertIntoEvents from main.models import Event, File +from django.core.exceptions import ValidationError logger = get_script_logger("archivematica.mcp.client.clamscan") @@ -291,7 +292,7 @@ def get_size(file_uuid, path): if file_uuid != "None": try: return File.objects.get(uuid=file_uuid).size - except File.DoesNotExist: + except (File.DoesNotExist, ValidationError): pass # Our fallback. try: diff --git a/src/MCPClient/lib/clientScripts/assign_uuids_to_directories.py b/src/MCPClient/lib/clientScripts/assign_uuids_to_directories.py index af68aa4f97..3b1801fb31 100755 --- a/src/MCPClient/lib/clientScripts/assign_uuids_to_directories.py +++ b/src/MCPClient/lib/clientScripts/assign_uuids_to_directories.py @@ -41,6 +41,7 @@ from archivematicaFunctions import get_dir_uuids, format_subdir_path, str2bool from django.db import transaction +from django.core.exceptions import ValidationError logger = get_script_logger("archivematica.mcp.client.assignUUIDsToDirectories") @@ -92,7 +93,7 @@ def _get_transfer_mdl(transfer_uuid): transfer_mdl.diruuids = True transfer_mdl.save() return transfer_mdl - except Transfer.DoesNotExist: + except (Transfer.DoesNotExist, ValidationError): logger.warning("There is no transfer with UUID %s", transfer_uuid) raise DirsUUIDsException diff --git a/src/MCPClient/lib/clientScripts/bind_pids.py b/src/MCPClient/lib/clientScripts/bind_pids.py index 494cf51456..12bbb8f867 100755 --- a/src/MCPClient/lib/clientScripts/bind_pids.py +++ b/src/MCPClient/lib/clientScripts/bind_pids.py @@ -48,6 +48,7 @@ django.setup() from django.db import transaction from lxml import etree +from django.core.exceptions import ValidationError # dashboard from main.models import DashboardSetting, Directory, SIP @@ -102,7 +103,7 @@ def _add_pid_to_mdl_identifiers(mdl, config): def _get_sip(sip_uuid): try: return SIP.objects.get(uuid=sip_uuid) - except SIP.DoesNotExist: + except (SIP.DoesNotExist, ValidationError): raise BindPIDsException diff --git a/src/MCPClient/lib/clientScripts/characterize_file.py b/src/MCPClient/lib/clientScripts/characterize_file.py index 4fd53363a6..18aeeb77b4 100755 --- a/src/MCPClient/lib/clientScripts/characterize_file.py +++ b/src/MCPClient/lib/clientScripts/characterize_file.py @@ -25,6 +25,7 @@ from dicts import replace_string_values, ReplacementDict from lib import setup_dicts +from django.core.exceptions import ValidationError def concurrent_instances(): @@ -43,7 +44,7 @@ def main(job, file_path, file_uuid, sip_uuid): try: format = FormatVersion.active.get(fileformatversion__file_uuid=file_uuid) - except FormatVersion.DoesNotExist: + except (FormatVersion.DoesNotExist, ValidationError): rules = format = None if format: diff --git a/src/MCPClient/lib/clientScripts/create_mets_v2.py b/src/MCPClient/lib/clientScripts/create_mets_v2.py index e95c04b10c..e969beea36 100755 --- a/src/MCPClient/lib/clientScripts/create_mets_v2.py +++ b/src/MCPClient/lib/clientScripts/create_mets_v2.py @@ -75,7 +75,7 @@ from change_names import change_name from bagit import Bag, BagError - +from django.core.exceptions import ValidationError SIP_DIR_VAR = r"%SIPDirectory%" @@ -1070,7 +1070,7 @@ def createFileSec( } try: f = File.objects.get(**kwargs) - except File.DoesNotExist: + except (File.DoesNotExist, ValidationError): job.pyprint( 'No uuid for file: "', directoryPathSTR, '"', file=sys.stderr ) @@ -1203,7 +1203,7 @@ def createFileSec( # Derived files should be in the original file's group try: d = Derivation.objects.get(derived_file_id=f.uuid) - except Derivation.DoesNotExist: + except (Derivation.DoesNotExist, ValidationError): job.pyprint( "Fatal error: unable to locate a Derivation object" " where the derived file is {}".format(f.uuid) diff --git a/src/MCPClient/lib/clientScripts/create_sip_from_transfer_objects.py b/src/MCPClient/lib/clientScripts/create_sip_from_transfer_objects.py index b7d5efe0f8..2f6d58389f 100755 --- a/src/MCPClient/lib/clientScripts/create_sip_from_transfer_objects.py +++ b/src/MCPClient/lib/clientScripts/create_sip_from_transfer_objects.py @@ -33,6 +33,7 @@ # archivematicaCommon import archivematicaFunctions +from django.core.exceptions import ValidationError def call(jobs): @@ -117,7 +118,7 @@ def call(jobs): unituuid=transferUUID, variable="activeAgent", ) - except UnitVariable.DoesNotExist: + except (UnitVariable.DoesNotExist, ValidationError): unit_variable = None if unit_variable: try: diff --git a/src/MCPClient/lib/clientScripts/create_transfer_mets.py b/src/MCPClient/lib/clientScripts/create_transfer_mets.py index 6093c61d7d..9574da9202 100755 --- a/src/MCPClient/lib/clientScripts/create_transfer_mets.py +++ b/src/MCPClient/lib/clientScripts/create_transfer_mets.py @@ -24,6 +24,7 @@ import uuid import django +from django.core.exceptions import ValidationError from django.db.models import Prefetch from lxml import etree @@ -85,7 +86,7 @@ def write_mets(mets_path, transfer_dir_path, base_path_placeholder, transfer_uui try: transfer = Transfer.objects.get(uuid=transfer_uuid) - except Transfer.DoesNotExist: + except (Transfer.DoesNotExist, ValidationError): logger.info("No record in database for transfer: %s", transfer_uuid) raise diff --git a/src/MCPClient/lib/clientScripts/dip_generation_helper.py b/src/MCPClient/lib/clientScripts/dip_generation_helper.py index 22688bc421..45474b35b1 100755 --- a/src/MCPClient/lib/clientScripts/dip_generation_helper.py +++ b/src/MCPClient/lib/clientScripts/dip_generation_helper.py @@ -6,14 +6,10 @@ import django from agentarchives import archivesspace from custom_handlers import get_script_logger +from django.core.exceptions import ValidationError from django.db.models import Q from main import models -# dashboard -# archivematicaCommon -# Third party dependencies, alphabetical by import source -# initialize Django (required for Django 1.7) - django.setup() from django.db import transaction @@ -105,7 +101,7 @@ def parse_archivesspace_ids(sip_path, sip_uuid): | Q(originallocation=b"%SIPDirectory%objects/" + filename.encode()), sip_id=sip_uuid, ) - except models.File.DoesNotExist: + except (models.File.DoesNotExist, ValidationError): logger.error("%s not found in database, skipping", filename) continue except models.File.MultipleObjectsReturned: diff --git a/src/MCPClient/lib/clientScripts/extract_maildir_attachments.py b/src/MCPClient/lib/clientScripts/extract_maildir_attachments.py index 597c123f01..0c7bedce69 100755 --- a/src/MCPClient/lib/clientScripts/extract_maildir_attachments.py +++ b/src/MCPClient/lib/clientScripts/extract_maildir_attachments.py @@ -29,6 +29,7 @@ django.setup() from django.db import transaction +from django.core.exceptions import ValidationError # dashboard from main.models import File @@ -84,7 +85,7 @@ def getFileUUIDofSourceFile(transferUUID, sourceFilePath): transfer_id=transferUUID, currentlocation__startswith=sourceFilePath, ).uuid - except File.DoesNotExist: + except (File.DoesNotExist, ValidationError): return "" diff --git a/src/MCPClient/lib/clientScripts/index_aip.py b/src/MCPClient/lib/clientScripts/index_aip.py index d6ffb9af5c..2e7d8a99f8 100755 --- a/src/MCPClient/lib/clientScripts/index_aip.py +++ b/src/MCPClient/lib/clientScripts/index_aip.py @@ -9,6 +9,7 @@ import identifier_functions import storageService as storage_service from custom_handlers import get_script_logger +from django.core.exceptions import ValidationError from main.models import UnitVariable # dashboard @@ -70,7 +71,7 @@ def index_aip(job): unittype="SIP", unituuid=sip_uuid, variable="AIPsinAIC" ) aips_in_aic = uv.variablevalue - except UnitVariable.DoesNotExist: + except (UnitVariable.DoesNotExist, ValidationError): pass # Delete ES index before creating new one if reingesting if "REIN" in sip_type: diff --git a/src/MCPClient/lib/clientScripts/manual_normalization_create_metadata_and_restructure.py b/src/MCPClient/lib/clientScripts/manual_normalization_create_metadata_and_restructure.py index 91eac31bf8..f96da55893 100755 --- a/src/MCPClient/lib/clientScripts/manual_normalization_create_metadata_and_restructure.py +++ b/src/MCPClient/lib/clientScripts/manual_normalization_create_metadata_and_restructure.py @@ -25,6 +25,7 @@ # archivematicaCommon import databaseFunctions import fileOperations +from django.core.exceptions import ValidationError def main(job): @@ -61,7 +62,7 @@ def main(job): filegrpuse="original", sip_id=SIPUUID, ) - except (File.DoesNotExist, File.MultipleObjectsReturned) as e: + except (File.DoesNotExist, File.MultipleObjectsReturned, ValidationError) as e: # Original file was not found, or there is more than one original file with # the same filename (differing extensions) # Look for a CSV that will specify the mapping @@ -84,7 +85,7 @@ def main(job): printfn=job.pyprint, ) if original is None: - if isinstance(e, File.DoesNotExist): + if isinstance(e, File.DoesNotExist, ValidationError): job.print_error( "No matching file for: {}".format( filePath.replace(SIPDirectory, "%SIPDirectory%") @@ -106,7 +107,7 @@ def main(job): sip_id=SIPUUID, ) else: - if isinstance(e, File.DoesNotExist): + if isinstance(e, File.DoesNotExist, ValidationError): job.print_error( "No matching file for: ", filePath.replace(SIPDirectory, "%SIPDirectory%", 1), @@ -158,7 +159,7 @@ def main(job): "Updated the eventOutcomeDetailNote of an existing normalization" " Event for file {}. Not creating a Derivation object".format(fileUUID) ) - except Event.DoesNotExist: + except (Event.DoesNotExist, ValidationError): # No normalization event was created in normalize.py - probably manually # normalized during Ingest derivationEventUUID = str(uuid.uuid4()) diff --git a/src/MCPClient/lib/clientScripts/manual_normalization_move_access_files_to_dip.py b/src/MCPClient/lib/clientScripts/manual_normalization_move_access_files_to_dip.py index 8db6e32afd..750991ee05 100755 --- a/src/MCPClient/lib/clientScripts/manual_normalization_move_access_files_to_dip.py +++ b/src/MCPClient/lib/clientScripts/manual_normalization_move_access_files_to_dip.py @@ -32,6 +32,7 @@ # --sipUUID "%SIPUUID%" --sipDirectory "%SIPDirectory%" --filePath "%relativeLocation%" from optparse import OptionParser +from django.core.exceptions import ValidationError def main(job): @@ -69,9 +70,9 @@ def main(job): } try: f = File.objects.get(currentlocation__startswith=filePathLike, **kwargs) - except (File.DoesNotExist, File.MultipleObjectsReturned): + except (File.DoesNotExist, File.MultipleObjectsReturned, ValidationError): f = File.objects.get(currentlocation=filePathLike2.encode(), **kwargs) - except (File.DoesNotExist, File.MultipleObjectsReturned) as e: + except (File.DoesNotExist, File.MultipleObjectsReturned, ValidationError) as e: # Original file was not found, or there is more than one original file with # the same filename (differing extensions) # Look for a CSV that will specify the mapping @@ -90,7 +91,7 @@ def main(job): csv_path, "access", access_file, unitIdentifier, printfn=job.pyprint ) if original is None: - if isinstance(e, File.DoesNotExist): + if isinstance(e, File.DoesNotExist, ValidationError): job.print_error( "No matching file for: {}".format( opts.filePath.replace(opts.sipDirectory, "%SIPDirectory%") @@ -113,7 +114,7 @@ def main(job): } f = File.objects.get(**kwargs) else: - if isinstance(e, File.DoesNotExist): + if isinstance(e, File.DoesNotExist, ValidationError): job.print_error( "No matching file for: ", opts.filePath.replace(opts.SIPDirectory, "%SIPDirectory%", 1), diff --git a/src/MCPClient/lib/clientScripts/move_to_backlog.py b/src/MCPClient/lib/clientScripts/move_to_backlog.py index 4c774c2940..49c1342f80 100755 --- a/src/MCPClient/lib/clientScripts/move_to_backlog.py +++ b/src/MCPClient/lib/clientScripts/move_to_backlog.py @@ -29,7 +29,7 @@ from bagit import make_bag import metsrw - +from django.core.exceptions import ValidationError logger = get_script_logger("archivematica.mcp.client.move_to_backlog") @@ -134,7 +134,7 @@ def _transfer_agents(transfer_id): var = UnitVariable.objects.get( unittype="Transfer", unituuid=transfer_id, variable="activeAgent" ) - except UnitVariable.DoesNotExist: + except (UnitVariable.DoesNotExist, ValidationError): pass else: query |= Q(id=var.variablevalue) diff --git a/src/MCPClient/lib/clientScripts/normalize.py b/src/MCPClient/lib/clientScripts/normalize.py index 3194af5eae..7b32373a19 100755 --- a/src/MCPClient/lib/clientScripts/normalize.py +++ b/src/MCPClient/lib/clientScripts/normalize.py @@ -25,7 +25,7 @@ from django.conf import settings as mcpclient_settings from .lib import setup_dicts - +from django.core.exceptions import ValidationError # Return codes SUCCESS = 0 @@ -341,7 +341,7 @@ def main(job, opts): # Find the file and it's FormatVersion (file identification) try: file_ = File.objects.get(uuid=opts.file_uuid) - except File.DoesNotExist: + except (File.DoesNotExist, ValidationError): job.print_error("File with uuid", opts.file_uuid, "does not exist in database.") return NO_RULE_FOUND job.print_output("File found:", file_.uuid, file_.currentlocation.decode()) @@ -417,7 +417,7 @@ def main(job, opts): do_fallback = False try: format_id = FileFormatVersion.objects.get(file_uuid=opts.file_uuid) - except FileFormatVersion.DoesNotExist: + except (FileFormatVersion.DoesNotExist, ValidationError): format_id = None # Look up the normalization command in the FPR diff --git a/src/MCPClient/lib/clientScripts/normalize_report.py b/src/MCPClient/lib/clientScripts/normalize_report.py index 1e8984954e..3eac0d1da1 100755 --- a/src/MCPClient/lib/clientScripts/normalize_report.py +++ b/src/MCPClient/lib/clientScripts/normalize_report.py @@ -8,6 +8,7 @@ from custom_handlers import get_script_logger from django.conf import settings as mcpclient_settings from django.contrib.auth.models import User +from django.core.exceptions import ValidationError from django.core.mail import send_mail from django.db import transaction from django.template import Context @@ -18,7 +19,6 @@ from main.models import SIP from main.models import Task - django.setup() logger = get_script_logger("archivematica.mcp.client.normalizeReport") @@ -113,7 +113,7 @@ def report(uuid): try: sip = SIP.objects.get(uuid=uuid) - except SIP.DoesNotExist: + except (SIP.DoesNotExist, ValidationError): logger.error("SIP with UUID %s not found.", uuid) return 1 @@ -142,7 +142,7 @@ def report(uuid): .currentlocation.decode() .replace("%SIPDirectory%", "") ) - except File.DoesNotExist: + except (File.DoesNotExist, ValidationError): pass if not len(failed_tasks): diff --git a/src/MCPClient/lib/clientScripts/parse_dataverse_mets.py b/src/MCPClient/lib/clientScripts/parse_dataverse_mets.py index 2c710f8534..32f1a35923 100644 --- a/src/MCPClient/lib/clientScripts/parse_dataverse_mets.py +++ b/src/MCPClient/lib/clientScripts/parse_dataverse_mets.py @@ -20,6 +20,7 @@ import databaseFunctions from main.models import Agent, File import metsrw +from django.core.exceptions import ValidationError logger = get_script_logger("archivematica.mcp.client.parse_dataverse_mets") transfer_objects_directory = "%transferDirectory%objects" @@ -74,7 +75,7 @@ def get_db_objects(job, mets, transfer_uuid): file_entry.originallocation, ) continue - except File.DoesNotExist: + except (File.DoesNotExist, ValidationError): logger.debug( "Could not find file type: '%s' in the database: %s with " "path: %s", entry.type, @@ -109,7 +110,7 @@ def get_db_objects(job, mets, transfer_uuid): file_entry.originallocation, ) continue - except File.DoesNotExist: + except (File.DoesNotExist, ValidationError): logger.error( "Could not find file type: '%s' in the database: %s with " "path: %s. Checksum: '%s'", diff --git a/src/MCPClient/lib/clientScripts/pid_declaration.py b/src/MCPClient/lib/clientScripts/pid_declaration.py index 4a3ac08c99..3fabcf3fa4 100644 --- a/src/MCPClient/lib/clientScripts/pid_declaration.py +++ b/src/MCPClient/lib/clientScripts/pid_declaration.py @@ -18,6 +18,7 @@ from main.models import Directory, File, SIP from change_names import change_name +from django.core.exceptions import ValidationError class DeclarePIDsException(Exception): @@ -163,7 +164,7 @@ def parse_and_attach_identifiers(self, unit_uuid, json_data): ) self._add_identifier_to_model(mdl, id_) continue - except File.DoesNotExist: + except (File.DoesNotExist, ValidationError): pass try: mdl = Directory.objects.get( @@ -171,7 +172,7 @@ def parse_and_attach_identifiers(self, unit_uuid, json_data): currentlocation=os.path.join(file_path, "").encode(), ) self._add_identifier_to_model(mdl, id_) - except Directory.DoesNotExist: + except (Directory.DoesNotExist, ValidationError): pass self.job.pyprint( "{} identifiers added for {} objects in the package".format( @@ -191,7 +192,7 @@ def _retrieve_identifiers_path(self, unit_uuid, sip_directory): .currentlocation.decode() .replace(self.SIP_DIRECTORY, sip_directory) ) - except File.DoesNotExist: + except (File.DoesNotExist, ValidationError): self.job.pyprint("No identifiers.json file found", file=sys.stderr) raise DeclarePIDsExceptionNonCritical() diff --git a/src/MCPClient/lib/clientScripts/policy_check.py b/src/MCPClient/lib/clientScripts/policy_check.py index e4fabde64e..f924694ff0 100755 --- a/src/MCPClient/lib/clientScripts/policy_check.py +++ b/src/MCPClient/lib/clientScripts/policy_check.py @@ -26,6 +26,7 @@ import databaseFunctions from dicts import replace_string_values from lib import setup_dicts +from django.core.exceptions import ValidationError # Note that linkTaskManagerFiles.py will take the highest exit code it has seen # from all tasks and will use that as the exit code of the job as a whole. @@ -80,7 +81,7 @@ def check(self): if not self.is_manually_normalized_access_derivative: try: self.file_model = File.objects.get(uuid=self.file_uuid) - except (File.DoesNotExist, django.core.exceptions.ValidationError): + except (File.DoesNotExist, ValidationError): self.job.pyprint( "Not performing a policy check because there is no file" " with UUID {}.".format(self.file_uuid) @@ -154,7 +155,7 @@ def _file_is_derivative(self, for_access=False): derived_file__uuid=self.file_uuid, event__event_type=event_type ) return True - except (Derivation.DoesNotExist, django.core.exceptions.ValidationError): + except (Derivation.DoesNotExist, ValidationError): return False def _get_policies_dir(self): @@ -205,7 +206,7 @@ def _get_manually_normalized_access_derivative_file_uuid(self): originallocation=manually_normalized_file_path.encode(), sip_id=self.sip_uuid, ).uuid - except (File.DoesNotExist, File.MultipleObjectsReturned): + except (File.DoesNotExist, File.MultipleObjectsReturned, ValidationError): return None def _get_rules(self): @@ -217,7 +218,7 @@ def _get_rules(self): file_uuid = self._get_manually_normalized_access_derivative_file_uuid() try: fmt = FormatVersion.active.get(fileformatversion__file_uuid=file_uuid) - except FormatVersion.DoesNotExist: + except (FormatVersion.DoesNotExist, ValidationError): rules = fmt = None if fmt: rules = FPRule.active.filter(format=fmt.uuid, purpose=self.purpose) @@ -373,7 +374,11 @@ def sip_logs_dir(self): unit_type = "Transfer" try: unit_model = model_cls.objects.get(uuid=self.sip_uuid) - except (model_cls.DoesNotExist, model_cls.MultipleObjectsReturned): + except ( + model_cls.DoesNotExist, + model_cls.MultipleObjectsReturned, + ValidationError, + ): self.job.print_error( "Warning: unable to retrieve {unit_type} model corresponding" " to {unit_type} UUID {sip_uuid}".format( @@ -411,7 +416,7 @@ def sip_subm_doc_dir(self): return self._sip_subm_doc_dir try: sip_model = SIP.objects.get(uuid=self.sip_uuid) - except (SIP.DoesNotExist, SIP.MultipleObjectsReturned): + except (SIP.DoesNotExist, SIP.MultipleObjectsReturned, ValidationError): self.job.print_error( "Warning: unable to retrieve SIP model corresponding to SIP" " UUID {} (when attempting to get the path to" diff --git a/src/MCPClient/lib/clientScripts/post_store_aip_hook.py b/src/MCPClient/lib/clientScripts/post_store_aip_hook.py index 216a13acbe..4002c601b2 100755 --- a/src/MCPClient/lib/clientScripts/post_store_aip_hook.py +++ b/src/MCPClient/lib/clientScripts/post_store_aip_hook.py @@ -20,6 +20,8 @@ import storageService as storage_service from archivematicaFunctions import find_transfer_path_from_ingest +from django.core.exceptions import ValidationError + logger = get_script_logger("archivematica.mcp.client.post_store_aip_hook") COMPLETED = 0 @@ -58,7 +60,7 @@ def dspace_handle_to_archivesspace(job, sip_uuid): # Get association to ArchivesSpace if it exists try: digital_object = models.ArchivesSpaceDigitalObject.objects.get(sip_id=sip_uuid) - except models.ArchivesSpaceDigitalObject.DoesNotExist: + except (models.ArchivesSpaceDigitalObject.DoesNotExist, ValidationError): job.pyprint("SIP", sip_uuid, "not associated with an ArchivesSpace component") return NO_ACTION job.pyprint( diff --git a/src/MCPClient/lib/clientScripts/restructure_for_compliance.py b/src/MCPClient/lib/clientScripts/restructure_for_compliance.py index ff79e9c8f7..bf3fc8769d 100755 --- a/src/MCPClient/lib/clientScripts/restructure_for_compliance.py +++ b/src/MCPClient/lib/clientScripts/restructure_for_compliance.py @@ -38,7 +38,7 @@ from custom_handlers import get_script_logger import bag - +from django.core.exceptions import ValidationError logger = get_script_logger("archivematica.mcp.client.restructureForCompliance") @@ -150,7 +150,7 @@ def call(jobs): sip = None try: transfer = Transfer.objects.get(uuid=sip_uuid) - except Transfer.DoesNotExist: + except (Transfer.DoesNotExist, ValidationError): sip = SIP.objects.get(uuid=sip_uuid) if transfer: diff --git a/src/MCPClient/lib/clientScripts/store_aip.py b/src/MCPClient/lib/clientScripts/store_aip.py index 4e864108fd..4ed2a80c8b 100755 --- a/src/MCPClient/lib/clientScripts/store_aip.py +++ b/src/MCPClient/lib/clientScripts/store_aip.py @@ -40,7 +40,7 @@ from archivematicaFunctions import escape import metrics - +from django.core.exceptions import ValidationError logger = get_script_logger("archivematica.mcp.client.storeAIP") @@ -198,7 +198,7 @@ def store_aip(job, aip_destination_uri, aip_path, sip_uuid, sip_name, sip_type): related_package = UnitVariable.objects.get( unituuid=sip_uuid, variable="relatedPackage" ) - except UnitVariable.DoesNotExist: + except (UnitVariable.DoesNotExist, ValidationError): pass else: related_package_uuid = related_package.variablevalue @@ -221,7 +221,7 @@ def store_aip(job, aip_destination_uri, aip_path, sip_uuid, sip_name, sip_type): dc = DublinCore.objects.get( metadataappliestotype_id=sip_metadata_uuid, metadataappliestoidentifier=uuid ) - except DublinCore.DoesNotExist: + except (DublinCore.DoesNotExist, ValidationError): aip_subtype = "Archival Information Package" else: aip_subtype = dc.type diff --git a/src/MCPClient/lib/clientScripts/transcribe_file.py b/src/MCPClient/lib/clientScripts/transcribe_file.py index 5543b78388..0f33d69013 100755 --- a/src/MCPClient/lib/clientScripts/transcribe_file.py +++ b/src/MCPClient/lib/clientScripts/transcribe_file.py @@ -20,6 +20,7 @@ from django.conf import settings as mcpclient_settings from lib import setup_dicts +from django.core.exceptions import ValidationError def concurrent_instances(): @@ -80,7 +81,7 @@ def fetch_rules_for(file_): return FPRule.objects.filter( format=format.format_version, purpose="transcription" ) - except FileFormatVersion.DoesNotExist: + except (FileFormatVersion.DoesNotExist, ValidationError): return [] diff --git a/src/MCPClient/lib/clientScripts/upload_archivesspace.py b/src/MCPClient/lib/clientScripts/upload_archivesspace.py index 4c86fcb4c8..5054c5ef72 100755 --- a/src/MCPClient/lib/clientScripts/upload_archivesspace.py +++ b/src/MCPClient/lib/clientScripts/upload_archivesspace.py @@ -17,6 +17,7 @@ django.setup() from django.db import transaction +from django.core.exceptions import ValidationError logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -167,13 +168,13 @@ def upload_to_archivesspace( fv = FormatVersion.objects.get(fileformatversion__file_uuid=uuid) format_version = fv.description format_name = fv.format.description - except FormatVersion.DoesNotExist: + except (FormatVersion.DoesNotExist, ValidationError): format_name = format_version = None # Client wants access copy info try: original_file = File.objects.get(filegrpuse="original", uuid=uuid) - except (File.DoesNotExist, File.MultipleObjectsReturned): + except (File.DoesNotExist, File.MultipleObjectsReturned, ValidationError): original_name = "" size = format_name = format_version = None else: @@ -185,7 +186,7 @@ def upload_to_archivesspace( access_file = File.objects.get( filegrpuse="access", original_file_set__source_file=uuid ) - except (File.DoesNotExist, File.MultipleObjectsReturned): + except (File.DoesNotExist, File.MultipleObjectsReturned, ValidationError): # Just use original file info pass else: diff --git a/src/MCPClient/lib/clientScripts/upload_qubit.py b/src/MCPClient/lib/clientScripts/upload_qubit.py index 7a97030ef4..ffb675c3d2 100755 --- a/src/MCPClient/lib/clientScripts/upload_qubit.py +++ b/src/MCPClient/lib/clientScripts/upload_qubit.py @@ -39,6 +39,8 @@ # dashboard import main.models as models +from django.core.exceptions import ValidationError + # moved after django.setup() logger = get_script_logger("archivematica.upload.qubit") @@ -102,7 +104,8 @@ def start(job, data): # This upload was called before, restore Access record access = models.Access.objects.get(sipuuid=data.uuid) except ( - models.Access.DoesNotExist + models.Access.DoesNotExist, + ValidationError, ): # First time this job is called, create new Access record access = models.Access(sipuuid=data.uuid) # Look for access system ID diff --git a/src/MCPClient/lib/clientScripts/validate_file.py b/src/MCPClient/lib/clientScripts/validate_file.py index 0c4a08af0e..13898fadea 100755 --- a/src/MCPClient/lib/clientScripts/validate_file.py +++ b/src/MCPClient/lib/clientScripts/validate_file.py @@ -20,6 +20,7 @@ from pprint import pformat import django +from django.core.exceptions import ValidationError from django.db import transaction django.setup() @@ -105,7 +106,7 @@ def _get_rules(self): """Return all FPR rules that apply to files of this type.""" try: fmt = FormatVersion.active.get(fileformatversion__file_uuid=self.file_uuid) - except FormatVersion.DoesNotExist: + except (FormatVersion.DoesNotExist, ValidationError): rules = fmt = None if fmt: rules = FPRule.active.filter(format=fmt.uuid, purpose=self.purpose) @@ -227,7 +228,7 @@ def _file_is_preservation_derivative(self): derived_file__uuid=self.file_uuid, event__event_type="normalization" ) return True - except Derivation.DoesNotExist: + except (Derivation.DoesNotExist, ValidationError): return False def _file_is_access_derivative(self): @@ -242,11 +243,11 @@ def _file_is_access_derivative(self): derived_file__uuid=self.file_uuid, event__isnull=True ) return True - except Derivation.DoesNotExist: + except (Derivation.DoesNotExist, ValidationError): return False else: return False - except File.DoesNotExist: + except (File.DoesNotExist, ValidationError): return False def _not_derivative_msg(self): @@ -264,7 +265,7 @@ def sip_logs_dir(self): return self._sip_logs_dir try: sip_model = SIP.objects.get(uuid=self.sip_uuid) - except (SIP.DoesNotExist, SIP.MultipleObjectsReturned): + except (SIP.DoesNotExist, SIP.MultipleObjectsReturned, ValidationError): self.job.print_error( "Warning: unable to retrieve SIP model corresponding to SIP" " UUID {}".format(self.sip_uuid) diff --git a/src/MCPClient/lib/metrics.py b/src/MCPClient/lib/metrics.py index 219f79c162..7f5eb60f7f 100644 --- a/src/MCPClient/lib/metrics.py +++ b/src/MCPClient/lib/metrics.py @@ -11,6 +11,7 @@ from common_metrics import PROCESSING_TIME_BUCKETS from common_metrics import TASK_DURATION_BUCKETS from django.conf import settings +from django.core.exceptions import ValidationError from django.db.models import Sum from django.utils import timezone from fpr.models import FormatVersion @@ -24,7 +25,6 @@ from prometheus_client import start_http_server from version import get_full_version - job_counter = Counter( "mcpclient_job_total", "Number of jobs processed, labeled by script", @@ -283,7 +283,7 @@ def aip_stored(sip_uuid, size): try: earliest_file = File.objects.filter(sip_id=sip_uuid).earliest("enteredsystem") - except File.DoesNotExist: + except (File.DoesNotExist, ValidationError): pass else: duration = (timezone.now() - earliest_file.enteredsystem).total_seconds() @@ -333,7 +333,7 @@ def dip_stored(sip_uuid, size): try: earliest_file = File.objects.filter(sip_id=sip_uuid).earliest("enteredsystem") - except File.DoesNotExist: + except (File.DoesNotExist, ValidationError): pass else: duration = (timezone.now() - earliest_file.enteredsystem).total_seconds() @@ -355,7 +355,7 @@ def transfer_started(transfer_type): def transfer_completed(transfer_uuid): try: transfer = Transfer.objects.get(uuid=transfer_uuid) - except Transfer.DoesNotExist: + except (Transfer.DoesNotExist, ValidationError): return transfer_type = transfer.type or "Unknown" diff --git a/src/MCPClient/tests/test_validate_file.py b/src/MCPClient/tests/test_validate_file.py new file mode 100644 index 0000000000..9c569c7644 --- /dev/null +++ b/src/MCPClient/tests/test_validate_file.py @@ -0,0 +1,110 @@ +import pytest +from fpr.models import Format +from fpr.models import FormatGroup +from fpr.models import FormatVersion +from fpr.models import FPCommand +from fpr.models import FPRule +from fpr.models import FPTool +from job import Job +from main.models import Event +from main.models import File +from main.models import FileFormatVersion +from main.models import SIP +from validate_file import main + + +@pytest.fixture +def sip(tmp_path): + sip_dir = tmp_path / "sip" + sip_dir.mkdir() + # Create logs directory in the SIP. + (sip_dir / "logs").mkdir() + + return SIP.objects.create(currentpath=str(sip_dir)) + + +@pytest.fixture +def file_obj(tmp_path, sip): + d = tmp_path / "dir" + d.mkdir() + txt_file = d / "file.txt" + txt_file.write_text("hello world") + + return File.objects.create( + sip=sip, originallocation=txt_file, currentlocation=txt_file + ) + + +@pytest.fixture +def format_version(): + format_group = FormatGroup.objects.create(description="a format group") + format = Format.objects.create(description="a format", group=format_group) + + return FormatVersion.objects.create(description="a format 1.0", format=format) + + +@pytest.fixture +def command(): + tool = FPTool.objects.create(description="a tool") + + return FPCommand.objects.create( + description="a command", + script_type="pythonScript", + command="script.py", + tool=tool, + ) + + +@pytest.fixture +def fprule(format_version, command): + return FPRule.objects.create( + purpose=FPRule.VALIDATION, format=format_version, command=command + ) + + +@pytest.fixture +def file_format_version(file_obj, format_version): + FileFormatVersion.objects.create(file_uuid=file_obj, format_version=format_version) + + +@pytest.mark.django_db +def test_main( + mocker, sip, file_obj, format_version, fprule, command, file_format_version +): + exit_status = 0 + stdout = '{"eventOutcomeInformation": "pass", "eventOutcomeDetailNote": "a note"}' + stderr = "" + execute_or_run = mocker.patch( + "validate_file.executeOrRun", return_value=(exit_status, stdout, stderr) + ) + job = mocker.Mock(spec=Job) + file_type = "original" + + main( + job=job, + file_path=file_obj.currentlocation, + file_uuid=file_obj.uuid, + sip_uuid=sip.uuid, + shared_path=sip.currentpath, + file_type=file_type, + ) + + # Check the executed script. + execute_or_run.assert_called_once_with( + type=command.script_type, + text=command.command, + printing=False, + arguments=[file_obj.currentlocation], + ) + + # Verify a PREMIS validation event was created with the output of the + # validation command. + assert ( + Event.objects.filter( + file_uuid=file_obj.uuid, + event_type="validation", + event_outcome="pass", + event_outcome_detail="a note", + ).count() + == 1 + ) diff --git a/src/MCPServer/lib/server/jobs/client.py b/src/MCPServer/lib/server/jobs/client.py index bb565e2cb1..9977580ede 100644 --- a/src/MCPServer/lib/server/jobs/client.py +++ b/src/MCPServer/lib/server/jobs/client.py @@ -6,6 +6,7 @@ import logging from django.conf import settings +from django.core.exceptions import ValidationError from main import models from server import metrics from server.db import auto_close_old_connections @@ -13,7 +14,6 @@ from server.tasks import get_task_backend from server.tasks import Task - logger = logging.getLogger("archivematica.mcp.server.jobs.client") @@ -198,6 +198,7 @@ def filter_subdir(self): except ( models.UnitVariable.DoesNotExist, models.UnitVariable.MultipleObjectsReturned, + ValidationError, ): var = None diff --git a/src/MCPServer/lib/server/jobs/local.py b/src/MCPServer/lib/server/jobs/local.py index 750001dc57..99c47a8719 100644 --- a/src/MCPServer/lib/server/jobs/local.py +++ b/src/MCPServer/lib/server/jobs/local.py @@ -4,11 +4,11 @@ import abc import logging +from django.core.exceptions import ValidationError from main import models from server.db import auto_close_old_connections from server.jobs.base import Job - logger = logging.getLogger("archivematica.mcp.server.jobs.local") @@ -40,7 +40,7 @@ def run(self, *args, **kwargs): unituuid=self.package.uuid, variable=self.link.config["variable"], ) - except models.UnitVariable.DoesNotExist: + except (models.UnitVariable.DoesNotExist, ValidationError): link_id = self.link.config["chain_id"] else: link_id = unitvar.microservicechainlink diff --git a/src/MCPServer/lib/server/packages.py b/src/MCPServer/lib/server/packages.py index 2b561c3dc8..13224cd4d0 100644 --- a/src/MCPServer/lib/server/packages.py +++ b/src/MCPServer/lib/server/packages.py @@ -11,6 +11,7 @@ import storageService as storage_service from django.conf import settings +from django.core.exceptions import ValidationError from django.utils import timezone from main import models from server.db import auto_close_old_connections @@ -18,7 +19,6 @@ from server.processing_config import processing_configuration_file_exists from server.utils import uuid_from_path - logger = logging.getLogger("archivematica.mcp.server.packages") @@ -335,7 +335,7 @@ def create_package( kwargs["transfermetadatasetrow"] = models.TransferMetadataSet.objects.get( id=metadata_set_id ) - except models.TransferMetadataSet.DoesNotExist: + except (models.TransferMetadataSet.DoesNotExist, ValidationError): pass transfer = models.Transfer.objects.create(**kwargs) if not processing_configuration_file_exists(processing_config): @@ -371,7 +371,7 @@ def wrap(*args, **kwargs): except Exception as err: # The main purpose of this decorator is to update the Transfer with # the new state (fail). If the Transfer does not exist we give up. - if isinstance(err, models.Transfer.DoesNotExist): + if isinstance(err, models.Transfer.DoesNotExist, ValidationError): raise else: logger.exception("Exception occurred during transfer processing") diff --git a/src/MCPServer/tests/test_package.py b/src/MCPServer/tests/test_package.py index 85e81b8512..b0d7fa4116 100644 --- a/src/MCPServer/tests/test_package.py +++ b/src/MCPServer/tests/test_package.py @@ -1,15 +1,20 @@ import uuid +from concurrent.futures import ThreadPoolExecutor from pathlib import Path import pytest +from django.core.exceptions import ValidationError from main import models from server.packages import _determine_transfer_paths from server.packages import _move_to_internal_shared_dir from server.packages import _pad_destination_filepath_if_it_already_exists +from server.packages import create_package from server.packages import DIP from server.packages import Package from server.packages import SIP from server.packages import Transfer +from server.queues import PackageQueue +from server.workflow import Workflow @pytest.mark.parametrize( @@ -60,7 +65,7 @@ def test_dip_get_or_create_from_db_path_without_uuid(tmp_path): assert dip.current_path == str(dip_path) try: models.SIP.objects.get(uuid=dip.uuid) - except models.SIP.DoesNotExist: + except (models.SIP.DoesNotExist, ValidationError): pytest.fail("DIP.get_or_create_from_db_by_path didn't create a SIP model") @@ -75,7 +80,7 @@ def test_dip_get_or_create_from_db_path_with_uuid(tmp_path): assert dip.current_path == str(dip_path) try: models.SIP.objects.get(uuid=dip_uuid) - except models.SIP.DoesNotExist: + except (models.SIP.DoesNotExist, ValidationError): pytest.fail("DIP.get_or_create_from_db_by_path didn't create a SIP model") @@ -109,7 +114,7 @@ def test_transfer_get_or_create_from_db_path_with_uuid(tmp_path): assert transfer.current_path == str(transfer_path) try: models.Transfer.objects.get(uuid=transfer_uuid) - except models.Transfer.DoesNotExist: + except (models.Transfer.DoesNotExist, ValidationError): pytest.fail( "Transfer.get_or_create_from_db_by_path didn't create a Transfer model" ) @@ -140,7 +145,7 @@ def test_package_get_or_create_from_db_by_path_updates_model( ) try: model.objects.get(**{"uuid": package_id, loc_attribute: path_dst}) - except models.Transfer.DoesNotExist: + except (models.Transfer.DoesNotExist, ValidationError): pytest.fail( "Method {}.get_or_create_from_db_by_path didn't update {} model".format( package_class.__name__, model.__name__ @@ -392,3 +397,36 @@ def test_package_statuses(tmp_path, package_class, model_class): Package.cleanup_old_db_entries() assert model_class.objects.get(pk=package_id).status == models.PACKAGE_STATUS_FAILED + + +@pytest.mark.django_db(transaction=True) +def test_create_package(mocker, tmp_path, admin_user, settings): + package_queue = mocker.Mock(spec=PackageQueue) + executor = mocker.Mock(spec=ThreadPoolExecutor) + workflow = mocker.Mock(spec=Workflow) + + d = tmp_path / "sub" + d.mkdir() + (d / "tmp").mkdir() + settings.SHARED_DIRECTORY = str(d) + + # Verify there are no existing transfers. + assert models.Transfer.objects.count() == 0 + + create_package( + package_queue, + executor, + "foobar", + "standard", + "", + "", + d.as_posix(), + "", + admin_user.pk, + workflow, + auto_approve=True, + processing_config="automated", + ) + + # Verify a transfer was added. + assert models.Transfer.objects.count() == 1 diff --git a/src/dashboard/src/components/api/views.py b/src/dashboard/src/components/api/views.py index 4f388244e5..d03ff14ab1 100644 --- a/src/dashboard/src/components/api/views.py +++ b/src/dashboard/src/components/api/views.py @@ -29,6 +29,7 @@ from components.unit import views as unit_views from contrib.mcp.client import MCPClient from django.conf import settings as django_settings +from django.core.exceptions import ValidationError from django.db.models import Q from django.views.decorators.csrf import csrf_exempt from main import models @@ -247,13 +248,13 @@ def status(request, unit_uuid, unit_type): if unit_type == "unitTransfer": try: unit = models.Transfer.objects.get(uuid=unit_uuid) - except models.Transfer.DoesNotExist: + except (models.Transfer.DoesNotExist, ValidationError): unit = None response["type"] = "transfer" elif unit_type == "unitSIP": try: unit = models.SIP.objects.get(uuid=unit_uuid) - except models.SIP.DoesNotExist: + except (models.SIP.DoesNotExist, ValidationError): unit = None response["type"] = "SIP" @@ -739,7 +740,7 @@ def path_metadata(request): file_lod.level_of_description = models.LevelOfDescription.objects.get( pk=request.POST["level_of_description"] ).name - except (KeyError, models.LevelOfDescription.DoesNotExist): + except (KeyError, models.LevelOfDescription.DoesNotExist, ValidationError): file_lod.level_of_description = "" file_lod.save() body = {"success": True} @@ -952,6 +953,6 @@ def task(request, task_uuid): """Return details of a task""" try: task = models.Task.objects.get(taskuuid=task_uuid) - except models.Task.DoesNotExist: + except (models.Task.DoesNotExist, ValidationError): return _error_response(f"Task with UUID {task_uuid} does not exist") return helpers.json_response(format_task(task, detailed_output=True)) diff --git a/src/dashboard/src/components/ingest/views.py b/src/dashboard/src/components/ingest/views.py index a0c842c1d4..296b8b3823 100644 --- a/src/dashboard/src/components/ingest/views.py +++ b/src/dashboard/src/components/ingest/views.py @@ -36,6 +36,7 @@ from contrib.mcp.client import MCPClient from django.conf import settings as django_settings from django.contrib import messages +from django.core.exceptions import ValidationError from django.forms.models import modelformset_factory from django.http import Http404 from django.http import HttpResponse @@ -147,7 +148,7 @@ def ingest_metadata_edit(request, uuid, id=None): metadataappliestotype=sip_type_id, metadataappliestoidentifier=uuid ) id = dc.id - except models.DublinCore.DoesNotExist: + except (models.DublinCore.DoesNotExist, ValidationError): dc = models.DublinCore( metadataappliestotype=sip_type_id, metadataappliestoidentifier=uuid ) @@ -204,7 +205,7 @@ def aic_metadata_add(request, uuid): metadataappliestotype=sip_type_id, metadataappliestoidentifier=uuid ) id = dc.id - except models.DublinCore.DoesNotExist: + except (models.DublinCore.DoesNotExist, ValidationError): dc = models.DublinCore( metadataappliestotype=sip_type_id, metadataappliestoidentifier=uuid ) diff --git a/src/dashboard/src/components/transfer/views.py b/src/dashboard/src/components/transfer/views.py index d5fb8cbd0f..abef710e54 100644 --- a/src/dashboard/src/components/transfer/views.py +++ b/src/dashboard/src/components/transfer/views.py @@ -24,6 +24,7 @@ from components.ingest.forms import DublinCoreMetadataForm from contrib.mcp.client import MCPClient from django.conf import settings as django_settings +from django.core.exceptions import ValidationError from django.http import HttpResponse from django.shortcuts import redirect from django.shortcuts import render @@ -157,7 +158,7 @@ def transfer_metadata_edit(request, uuid, id=None): metadataappliestoidentifier__exact=uuid, ) return redirect("transfer:transfer_metadata_edit", uuid, dc.id) - except models.DublinCore.DoesNotExist: + except (models.DublinCore.DoesNotExist, ValidationError): dc = models.DublinCore( metadataappliestotype=transfer_metadata_type_id(), metadataappliestoidentifier=uuid, diff --git a/src/dashboard/src/fpr/forms.py b/src/dashboard/src/fpr/forms.py index 09f31fc2d9..983b64edab 100644 --- a/src/dashboard/src/fpr/forms.py +++ b/src/dashboard/src/fpr/forms.py @@ -2,7 +2,6 @@ from django.utils.translation import gettext_lazy as _ from fpr import models as fprmodels - # ########## FORMATS ############ diff --git a/src/dashboard/src/fpr/utils.py b/src/dashboard/src/fpr/utils.py index 0c604f2457..b441a06af3 100644 --- a/src/dashboard/src/fpr/utils.py +++ b/src/dashboard/src/fpr/utils.py @@ -2,7 +2,6 @@ from django.contrib import messages from django.utils.translation import gettext as _ - # ########## DEPENDENCIES ############ diff --git a/src/dashboard/src/fpr/views.py b/src/dashboard/src/fpr/views.py index c06453e853..9143a888e6 100644 --- a/src/dashboard/src/fpr/views.py +++ b/src/dashboard/src/fpr/views.py @@ -1,5 +1,6 @@ from django.contrib import messages from django.contrib.auth.decorators import user_passes_test +from django.core.exceptions import ValidationError from django.http import Http404 from django.shortcuts import get_object_or_404 from django.shortcuts import redirect @@ -10,7 +11,6 @@ from fpr import models as fprmodels from fpr import utils - CLASS_CATEGORY_MAP = { "format": fprmodels.Format, "formatgroup": fprmodels.FormatGroup, @@ -412,7 +412,7 @@ def idcommand_edit(request, uuid=None): initial["tool"] = fprmodels.IDTool.objects.get( uuid=request.GET["parent"], enabled=True ) - except (KeyError, fprmodels.IDTool.DoesNotExist): + except (KeyError, fprmodels.IDTool.DoesNotExist, ValidationError): initial["tool"] = None form = fprforms.IDCommandForm( @@ -659,7 +659,7 @@ def fpcommand_edit(request, uuid=None): initial["tool"] = fprmodels.FPTool.objects.get( uuid=request.GET["parent"], enabled=True ) - except (KeyError, fprmodels.FPTool.DoesNotExist): + except (KeyError, fprmodels.FPTool.DoesNotExist, ValidationError): initial["tool"] = None form = fprforms.FPCommandForm(instance=fpcommand, initial=initial) utils.warn_if_replacing_with_old_revision(request, fpcommand) diff --git a/src/dashboard/src/main/management/commands/rebuild_transfer_backlog.py b/src/dashboard/src/main/management/commands/rebuild_transfer_backlog.py index 1725f01a50..e2d6aa48d5 100644 --- a/src/dashboard/src/main/management/commands/rebuild_transfer_backlog.py +++ b/src/dashboard/src/main/management/commands/rebuild_transfer_backlog.py @@ -53,6 +53,7 @@ import storageService from components.rights.load import load_rights from django.conf import settings as django_settings +from django.core.exceptions import ValidationError from django.core.management.base import CommandError from fileOperations import addFileToTransfer from fileOperations import extract_package @@ -578,7 +579,7 @@ def _import_pipeline_dependant_transfer( """ try: Transfer.objects.get(uuid=transfer_uuid) - except Transfer.DoesNotExist: + except (Transfer.DoesNotExist, ValidationError): cmd.warning(f"Skipping transfer {transfer_uuid} - not found in the database!") return es.index_transfer_and_files( diff --git a/src/dashboard/src/main/models.py b/src/dashboard/src/main/models.py index 13043ba361..66c2c589a8 100644 --- a/src/dashboard/src/main/models.py +++ b/src/dashboard/src/main/models.py @@ -25,6 +25,7 @@ import uuid from django.contrib.auth.models import User +from django.core.exceptions import ValidationError from django.db import IntegrityError from django.db import models from django.db import transaction @@ -503,7 +504,7 @@ def agents(self): unit_variable = UnitVariable.objects.get( unittype="SIP", unituuid=self.uuid, variable="activeAgent" ) - except UnitVariable.DoesNotExist: + except (UnitVariable.DoesNotExist, ValidationError): pass else: agent_lookups = agent_lookups | models.Q(id=unit_variable.variablevalue) @@ -571,7 +572,7 @@ def agents(self): unit_variable = UnitVariable.objects.get( unittype="Transfer", unituuid=self.uuid, variable="activeAgent" ) - except UnitVariable.DoesNotExist: + except (UnitVariable.DoesNotExist, ValidationError): pass else: agent_lookups = agent_lookups | models.Q(id=unit_variable.variablevalue) @@ -591,7 +592,7 @@ def processing_configuration(self): unituuid=self.uuid, variable="processingConfiguration", ) - except UnitVariable.DoesNotExist: + except (UnitVariable.DoesNotExist, ValidationError): result = None else: result = unit_variable.variablevalue