Skip to content

Commit

Permalink
Remove django-extensions package
Browse files Browse the repository at this point in the history
* Add custom UUIDField and use it for uuid model attributes
* Remove django_extensions from existing migrations
* Add migrations for current changes
* Update requirements

Co-authored-by: Douglas Cerna (Soy Douglas) <[email protected]>
  • Loading branch information
Dhwaniartefact and replaceafill authored Aug 11, 2023
1 parent 279c190 commit 2555b50
Show file tree
Hide file tree
Showing 53 changed files with 632 additions and 281 deletions.
2 changes: 1 addition & 1 deletion hack/submodules/archivematica-storage-service
Submodule archivematica-storage-service updated 59 files
+6 −0 integration/run.sh
+12 −16 integration/test_integration.py
+0 −1 requirements/base.in
+0 −3 requirements/base.txt
+0 −3 requirements/local.txt
+0 −3 requirements/production.txt
+0 −3 requirements/test.txt
+22 −21 storage_service/administration/forms.py
+1 −4 storage_service/common/decorators.py
+32 −0 storage_service/common/fields.py
+1 −7 storage_service/common/gpgutils.py
+2 −2 storage_service/common/management/commands/import_aip.py
+5 −5 storage_service/common/premis.py
+1 −1 storage_service/common/startup.py
+0 −184 storage_service/common/tests/test_shutil.py
+1 −1 storage_service/common/utils.py
+0 −66 storage_service/common/which.py
+27 −20 storage_service/locations/api/resources.py
+0 −4 storage_service/locations/api/sword/helpers.py
+0 −7 storage_service/locations/api/sword/views.py
+1 −1 storage_service/locations/fixtures/dspace.json
+1 −1 storage_service/locations/fixtures/gpg.json
+13 −13 storage_service/locations/fixtures/package.json
+1 −1 storage_service/locations/fixtures/replica_staging.json
+1 −1 storage_service/locations/fixtures/s3.json
+3 −3 storage_service/locations/forms.py
+11 −14 storage_service/locations/migrations/0001_initial.py
+11 −7 storage_service/locations/migrations/0003_v0_5.py
+3 −4 storage_service/locations/migrations/0004_v0_7.py
+2 −12 storage_service/locations/migrations/0014_verbose_field_names.py
+13 −20 storage_service/locations/migrations/0029_python3.py
+99 −0 storage_service/locations/migrations/0034_use_uuidfield.py
+0 −6 storage_service/locations/models/arkivum.py
+1 −1 storage_service/locations/models/async_manager.py
+0 −3 storage_service/locations/models/asynchronous.py
+3 −9 storage_service/locations/models/dataverse.py
+3 −7 storage_service/locations/models/dspace.py
+0 −5 storage_service/locations/models/dspace_rest.py
+0 −6 storage_service/locations/models/duracloud.py
+10 −12 storage_service/locations/models/event.py
+12 −10 storage_service/locations/models/fedora.py
+0 −8 storage_service/locations/models/fixity_log.py
+0 −6 storage_service/locations/models/gpg.py
+0 −6 storage_service/locations/models/local_filesystem.py
+14 −14 storage_service/locations/models/location.py
+2 −7 storage_service/locations/models/lockssomatic.py
+0 −6 storage_service/locations/models/nfs.py
+22 −23 storage_service/locations/models/package.py
+6 −23 storage_service/locations/models/pipeline.py
+0 −6 storage_service/locations/models/pipeline_local.py
+1 −7 storage_service/locations/models/s3.py
+8 −10 storage_service/locations/models/space.py
+0 −6 storage_service/locations/models/swift.py
+4 −4 storage_service/locations/tests/test_api.py
+8 −4 storage_service/locations/tests/test_callbacks.py
+31 −32 storage_service/locations/tests/test_datatable.py
+2 −2 storage_service/locations/tests/test_dspace.py
+7 −6 storage_service/locations/tests/test_package.py
+0 −2 storage_service/storage_service/settings/base.py
3 changes: 0 additions & 3 deletions requirements-dev.txt
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,6 @@ django-cas-ng==3.6.0
# via -r requirements.txt
django-csp==3.7
# via -r requirements.txt
django-extensions==1.7.9
# via -r requirements.txt
django-forms-bootstrap==3.1.0
# via -r requirements.txt
django-prometheus==2.0.0
Expand Down Expand Up @@ -251,7 +249,6 @@ six==1.16.0
# via
# -r requirements.txt
# amclient
# django-extensions
# jsonschema
# mozilla-django-oidc
# opf-fido
Expand Down
1 change: 0 additions & 1 deletion requirements.in
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ brotli
clamd
django-autoslug==1.9.8
django-csp
django-extensions==1.7.9
django-forms-bootstrap>=3.0.0,<4.0.0
django-prometheus>=2.0,<2.1
django-tastypie==0.14.3
Expand Down
3 changes: 0 additions & 3 deletions requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,6 @@ django-cas-ng==3.6.0
# via -r requirements.in
django-csp==3.7
# via -r requirements.in
django-extensions==1.7.9
# via -r requirements.in
django-forms-bootstrap==3.1.0
# via -r requirements.in
django-prometheus==2.0.0
Expand Down Expand Up @@ -152,7 +150,6 @@ requests==2.27.1
six==1.16.0
# via
# amclient
# django-extensions
# jsonschema
# mozilla-django-oidc
# opf-fido
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ def process_xml_metadata(mets, sip_dir, sip_uuid, sip_type, xml_validation):
continue
valid, errors = _validate_xml(tree, schema_uri)
_add_validation_event(
mets, metadata_file.uuid, schema_uri, valid, errors
mets, str(metadata_file.uuid), schema_uri, valid, errors
)
if not valid:
xml_metadata_errors += errors
Expand Down Expand Up @@ -251,5 +251,5 @@ def _add_validation_event(mets, file_uuid, schema_uri, valid, errors):
"eventOutcomeDetailNote": "\n".join([str(err) for err in errors]),
}
event_object = insertIntoEvents(file_uuid, **event_data)
metadata_fsentry = mets.get_file(file_uuid=file_uuid)
metadata_fsentry = mets.get_file(file_uuid=str(file_uuid))
metadata_fsentry.add_premis_event(createmets2.createEvent(event_object))
34 changes: 19 additions & 15 deletions src/MCPClient/lib/clientScripts/archivematicaCreateMETSReingest.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,11 @@ def update_object(job, mets):
# Iterate through original files
for fsentry in mets.all_files():
# Only update original files
if fsentry.use != "original" or fsentry.type != "Item" or not fsentry.file_uuid:
if (
fsentry.use != "original"
or fsentry.type != "Item"
or not str(fsentry.file_uuid)
):
continue

# Copy techMD
Expand Down Expand Up @@ -101,7 +105,7 @@ def update_object(job, mets):
fixity.find("premis:messageDigest", namespaces=ns.NSMAP).text = f.checksum

# If FileID exists, update file ID
if models.FileID.objects.filter(file_id=fsentry.file_uuid):
if models.FileID.objects.filter(file_id=str(fsentry.file_uuid)):
job.pyprint("Updating format for", fsentry.file_uuid)
modified = True
# Delete old formats
Expand All @@ -119,7 +123,7 @@ def update_object(job, mets):

# If FPCommand output exists, update objectCharacteristicsExtension
if models.FPCommandOutput.objects.filter(
file_id=fsentry.file_uuid,
file_id=str(fsentry.file_uuid),
rule__purpose__in=["characterization", "default_characterization"],
).exists():
job.pyprint(
Expand All @@ -143,7 +147,7 @@ def update_object(job, mets):

# If Derivation exists, update relationships
if models.Derivation.objects.filter(
source_file_id=fsentry.file_uuid, event__isnull=False
source_file_id=str(fsentry.file_uuid), event__isnull=False
):
job.pyprint("Updating relationships for", fsentry.file_uuid)
modified = True
Expand Down Expand Up @@ -375,7 +379,7 @@ def add_events(job, mets, sip_uuid):
for event in models.Event.objects.filter(file_uuid__sip__uuid=sip_uuid).iterator():
job.pyprint("Adding", event.event_type, "event to file", event.file_uuid_id)
try:
fsentry = fsentries[event.file_uuid_id]
fsentry = fsentries[str(event.file_uuid_id)]
except KeyError:
job.pyprint(
"File with UUID",
Expand Down Expand Up @@ -433,7 +437,7 @@ def add_new_files(job, mets, sip_uuid, sip_dir):
if rel_path != old_mets_rel_path:
job.pyprint(rel_path, "not found in METS, must be new file")
f = models.File.objects.get(
currentlocation=current_loc, sip_id=sip_uuid
currentlocation=current_loc, sip_id=str(sip_uuid)
)
new_files.append(f)
if rel_path == "objects/metadata/metadata.csv":
Expand All @@ -455,20 +459,20 @@ def add_new_files(job, mets, sip_uuid, sip_dir):

for f in new_files:
# Create amdSecs
job.pyprint("Adding amdSec for", f.currentlocation, "(", f.uuid, ")")
job.pyprint("Adding amdSec for", f.currentlocation, "(", str(f.uuid), ")")
amdsec, amdid = createmets2.getAMDSec(
job,
fileUUID=f.uuid,
fileUUID=str(f.uuid),
filePath=None, # Only needed if use=original
use=f.filegrpuse,
sip_uuid=sip_uuid,
sip_uuid=str(sip_uuid),
transferUUID=None, # Only needed if use=original
itemdirectoryPath=None, # Only needed if use=original
typeOfTransfer=None, # Only needed if use=original
baseDirectoryPath=sip_dir,
state=state,
)
job.pyprint(f.uuid, "has amdSec with ID", amdid)
job.pyprint(str(f.uuid), "has amdSec with ID", amdid)

# Create parent directories if needed
dirs = os.path.dirname(
Expand All @@ -485,12 +489,12 @@ def add_new_files(job, mets, sip_uuid, sip_dir):
derived_from = None
if f.original_file_set.exists():
original_f = f.original_file_set.get().source_file
derived_from = mets.get_file(file_uuid=original_f.uuid)
derived_from = mets.get_file(file_uuid=str(original_f.uuid))
entry = metsrw.FSEntry(
path=f.currentlocation.replace("%SIPDirectory%", "", 1),
use=f.filegrpuse,
type="Item",
file_uuid=f.uuid,
file_uuid=str(f.uuid),
derived_from=derived_from,
)
metsrw_amdsec = metsrw.AMDSec.parse(amdsec)
Expand All @@ -514,7 +518,7 @@ def delete_files(mets, sip_uuid):
sip_id=sip_uuid, event__event_type="deletion"
).values_list("uuid", flat=True)
for file_uuid in deleted_files:
df = mets.get_file(file_uuid=file_uuid)
df = mets.get_file(file_uuid=str(file_uuid))
df.use = "deleted"
df.path = None
df.label = None
Expand Down Expand Up @@ -554,7 +558,7 @@ def update_metadata_csv(job, mets, metadata_csv, sip_uuid, sip_dir, state):
file_obj = None
try:
file_obj = models.File.objects.get(
sip_id=sip_uuid, originallocation__endswith="%" + f
sip_id=str(sip_uuid), originallocation__endswith="%" + f
)
except models.File.DoesNotExist:
try:
Expand All @@ -564,7 +568,7 @@ def update_metadata_csv(job, mets, metadata_csv, sip_uuid, sip_dir, state):
except models.File.DoesNotExist:
pass
if file_obj is not None:
fsentry = mets.get_file(file_uuid=file_obj.uuid)
fsentry = mets.get_file(file_uuid=str(file_obj.uuid))
else:
fsentry = _get_directory_fsentry(mets, f)
if fsentry is None:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -269,7 +269,7 @@ def createRightsStatement(job, statement, fileUUID, state):
).text = "UUID"
etree.SubElement(
linkingObjectIdentifier, ns.premisBNS + "linkingObjectIdentifierValue"
).text = fileUUID
).text = str(fileUUID)
return rightsStatement


Expand Down
10 changes: 6 additions & 4 deletions src/MCPClient/lib/clientScripts/archivematica_clamscan.py
Original file line number Diff line number Diff line change
Expand Up @@ -219,9 +219,12 @@ def version_attrs(self):


def file_already_scanned(file_uuid):
return Event.objects.filter(
file_uuid_id=file_uuid, event_type="virus check"
).exists()
return (
file_uuid != "None"
and Event.objects.filter(
file_uuid_id=file_uuid, event_type="virus check"
).exists()
)


def queue_event(file_uuid, date, scanner, passed, queue):
Expand Down Expand Up @@ -274,7 +277,6 @@ def get_scanner():
"""
choice = str(mcpclient_settings.CLAMAV_CLIENT_BACKEND).lower()
if choice not in SCANNERS_NAMES:

logger.warning(
"Unexpected antivirus scanner (CLAMAV_CLIENT_BACKEND):" ' "%s"; using %s.',
choice,
Expand Down
41 changes: 21 additions & 20 deletions src/MCPClient/lib/clientScripts/create_mets_v2.py
Original file line number Diff line number Diff line change
Expand Up @@ -230,7 +230,9 @@ def _add_identifier(object_elem, identifier, bns=ns.premisBNS):
idfr_type, idfr_val = identifier
objectIdentifier = etree.SubElement(object_elem, bns + "objectIdentifier")
etree.SubElement(objectIdentifier, bns + "objectIdentifierType").text = idfr_type
etree.SubElement(objectIdentifier, bns + "objectIdentifierValue").text = idfr_val
etree.SubElement(objectIdentifier, bns + "objectIdentifierValue").text = str(
idfr_val
)
return object_elem


Expand Down Expand Up @@ -527,7 +529,7 @@ def create_premis_object(fileUUID):
:param str fileUUID: UUID of the File to create an object for
:return: premis:object Element, suitable for inserting into mets:xmlData
"""
f = File.objects.get(uuid=fileUUID)
f = File.objects.get(uuid=str(fileUUID))
# PREMIS:OBJECT
object_elem = etree.Element(ns.premisBNS + "object", nsmap={"premis": ns.premisNS})
object_elem.set(ns.xsiBNS + "type", "premis:file")
Expand Down Expand Up @@ -657,7 +659,7 @@ def create_premis_object_derivations(fileUUID):
).text = "UUID"
etree.SubElement(
relatedObjectIdentifier, ns.premisBNS + "relatedObjectIdentifierValue"
).text = derivation.derived_file_id
).text = str(derivation.derived_file_id)

relatedEventIdentifier = etree.SubElement(
relationship, ns.premisBNS + "relatedEventIdentifier"
Expand All @@ -667,7 +669,7 @@ def create_premis_object_derivations(fileUUID):
).text = "UUID"
etree.SubElement(
relatedEventIdentifier, ns.premisBNS + "relatedEventIdentifierValue"
).text = derivation.event_id
).text = str(derivation.event_id)

elements.append(relationship)

Expand All @@ -691,7 +693,7 @@ def create_premis_object_derivations(fileUUID):
).text = "UUID"
etree.SubElement(
relatedObjectIdentifier, ns.premisBNS + "relatedObjectIdentifierValue"
).text = derivation.source_file_id
).text = str(derivation.source_file_id)

relatedEventIdentifier = etree.SubElement(
relationship, ns.premisBNS + "relatedEventIdentifier"
Expand All @@ -701,7 +703,7 @@ def create_premis_object_derivations(fileUUID):
).text = "UUID"
etree.SubElement(
relatedEventIdentifier, ns.premisBNS + "relatedEventIdentifierValue"
).text = derivation.event_id
).text = str(derivation.event_id)

elements.append(relationship)

Expand All @@ -714,7 +716,7 @@ def createDigiprovMD(fileUUID, state):
"""
ret = []

events = Event.objects.filter(file_uuid_id=fileUUID)
events = Event.objects.filter(file_uuid_id=str(fileUUID))
for event_record in events:
state.globalDigiprovMDCounter += 1
digiprovMD = etree.Element(
Expand All @@ -730,7 +732,7 @@ def createDigiprovMD(fileUUID, state):
xmlData.append(createEvent(event_record))

for agent in Agent.objects.extend_queryset_with_preservation_system(
Agent.objects.filter(event__file_uuid_id=fileUUID).distinct()
Agent.objects.filter(event__file_uuid_id=str(fileUUID)).distinct()
):
state.globalDigiprovMDCounter += 1
digiprovMD = etree.Element(
Expand Down Expand Up @@ -761,10 +763,9 @@ def createEvent(event_record):
etree.SubElement(
eventIdentifier, ns.premisBNS + "eventIdentifierType"
).text = "UUID"
etree.SubElement(
eventIdentifier, ns.premisBNS + "eventIdentifierValue"
).text = event_record.event_id

etree.SubElement(eventIdentifier, ns.premisBNS + "eventIdentifierValue").text = str(
event_record.event_id
)
etree.SubElement(event, ns.premisBNS + "eventType").text = event_record.event_type
etree.SubElement(
event, ns.premisBNS + "eventDateTime"
Expand Down Expand Up @@ -895,7 +896,7 @@ def getAMDSec(
)
AMD.append(digiprovMD)

for a in createDigiprovMD(fileUUID, state):
for a in createDigiprovMD(str(fileUUID), state):
AMD.append(a)

return ret
Expand Down Expand Up @@ -1136,7 +1137,7 @@ def createFileSec(
GROUPID = ""
if f.filegrpuuid:
# GROUPID was determined elsewhere
GROUPID = "Group-%s" % (f.filegrpuuid)
GROUPID = f"Group-{f.filegrpuuid}"
if use == "TRIM file metadata":
use = "metadata"

Expand All @@ -1147,7 +1148,7 @@ def createFileSec(
"maildirFile",
):
# These files are in a group defined by themselves
GROUPID = "Group-%s" % (f.uuid)
GROUPID = f"Group-{f.uuid}"
if use == "maildirFile":
use = "original"
# Check for CSV-based Dublincore dmdSec
Expand Down Expand Up @@ -1193,7 +1194,7 @@ def createFileSec(
}
original_file = File.objects.filter(**kwargs).first()
if original_file is not None:
GROUPID = "Group-" + original_file.uuid
GROUPID = f"Group-{original_file.uuid}"

elif use in ("preservation", "text/ocr", "derivative"):
# Derived files should be in the original file's group
Expand All @@ -1205,7 +1206,7 @@ def createFileSec(
" where the derived file is {}".format(f.uuid)
)
raise
GROUPID = "Group-" + d.source_file_id
GROUPID = f"Group-{d.source_file_id}"

elif use == "service":
# Service files are in the original file's group
Expand All @@ -1223,10 +1224,10 @@ def createFileSec(
"currentlocation__startswith": fileFileIDPath,
}
original_file = File.objects.get(**kwargs)
GROUPID = "Group-" + original_file.uuid
GROUPID = f"Group-{original_file.uuid}"

elif use == "TRIM container metadata":
GROUPID = "Group-%s" % (f.uuid)
GROUPID = f"Group-{f.uuid}"
use = "metadata"

# Special DSPACEMETS processing
Expand Down Expand Up @@ -1765,7 +1766,7 @@ def main(

# Get the <dmdSec> for the entire AIP; it is associated to the root
# <mets:div> in the physical structMap.
sip_mdl = SIP.objects.filter(uuid=sipUUID).first()
sip_mdl = SIP.objects.filter(uuid=str(sipUUID)).first()
if sip_mdl:
aipDmdSec = getDirDmdSec(sip_mdl, sip_dir_name)
state.globalDmdSecCounter += 1
Expand Down
Loading

0 comments on commit 2555b50

Please sign in to comment.