Add DB models for package tag and audit

Note the removal of the package-schema relationship. Packages and tags enable us to provide a unified interface (API + UI) for metadata submission, with standards-independent vocabularies/ontologies (which are themselves based on known standards). The existing ODP record model continues to hold a standards-based JSON record (representing the archival information package), which can be generated from package + tag + resource metadata, as applicable.

migrate/versions/2024_04_17_e9ac68f05e6b_archive_integration.py → migrate/versions/2024_05_23_44c77b7c26ed_archive_integration.py

@@ -1,41 +1,66 @@
 """Archive integration
 
-Revision ID: e9ac68f05e6b
+Revision ID: 44c77b7c26ed
 Revises: df57d06e1ee5
-Create Date: 2024-04-17 13:25:02.261646
+Create Date: 2024-05-23 09:18:43.012998
 
 """
 from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.dialects import postgresql
 
 # revision identifiers, used by Alembic.
-revision = 'e9ac68f05e6b'
+revision = '44c77b7c26ed'
 down_revision = 'df57d06e1ee5'
 branch_labels = None
 depends_on = None
 
 
 def upgrade():
+    # new enum values must be committed before they can be used
+    op.execute("alter type tagtype add value if not exists 'package' before 'collection'")
+    op.execute('commit')
+
     # ### commands auto generated by Alembic - adjusted ###
     op.create_table('archive',
                     sa.Column('id', sa.String(), nullable=False),
                     sa.Column('url', sa.String(), nullable=False),
                     sa.PrimaryKeyConstraint('id')
                     )
+    op.create_table('package_audit',
+                    sa.Column('id', sa.Integer(), sa.Identity(always=False), nullable=False),
+                    sa.Column('client_id', sa.String(), nullable=False),
+                    sa.Column('user_id', sa.String(), nullable=True),
+                    sa.Column('command', postgresql.ENUM(name='auditcommand', create_type=False), nullable=False),
+                    sa.Column('timestamp', sa.TIMESTAMP(timezone=True), nullable=False),
+                    sa.Column('_id', sa.String(), nullable=False),
+                    sa.Column('_title', sa.String(), nullable=False),
+                    sa.Column('_status', sa.String(), nullable=False),
+                    sa.Column('_notes', sa.String(), nullable=True),
+                    sa.Column('_provider_id', sa.String(), nullable=False),
+                    sa.PrimaryKeyConstraint('id')
+                    )
+    op.create_table('package_tag_audit',
+                    sa.Column('id', sa.Integer(), sa.Identity(always=False), nullable=False),
+                    sa.Column('client_id', sa.String(), nullable=False),
+                    sa.Column('user_id', sa.String(), nullable=True),
+                    sa.Column('command', postgresql.ENUM(name='auditcommand', create_type=False), nullable=False),
+                    sa.Column('timestamp', sa.TIMESTAMP(timezone=True), nullable=False),
+                    sa.Column('_id', sa.String(), nullable=False),
+                    sa.Column('_package_id', sa.String(), nullable=False),
+                    sa.Column('_tag_id', sa.String(), nullable=False),
+                    sa.Column('_user_id', sa.String(), nullable=True),
+                    sa.Column('_data', postgresql.JSONB(astext_type=sa.Text()), nullable=False),
+                    sa.PrimaryKeyConstraint('id')
+                    )
     op.create_table('package',
                     sa.Column('id', sa.String(), nullable=False),
-                    sa.Column('metadata_', postgresql.JSONB(astext_type=sa.Text()), nullable=False),
-                    sa.Column('validity', postgresql.JSONB(astext_type=sa.Text()), nullable=False),
+                    sa.Column('title', sa.String(), nullable=False),
+                    sa.Column('status', sa.Enum('pending', 'submitted', 'archived', name='packagestatus'), nullable=False),
                     sa.Column('notes', sa.String(), nullable=True),
                     sa.Column('timestamp', sa.TIMESTAMP(timezone=True), nullable=False),
                     sa.Column('provider_id', sa.String(), nullable=False),
-                    sa.Column('schema_id', sa.String(), nullable=False),
-                    sa.Column('schema_type', postgresql.ENUM(name='schematype', create_type=False), nullable=False),
-                    sa.CheckConstraint("schema_type = 'metadata'", name='package_schema_type_check'),
                     sa.ForeignKeyConstraint(['provider_id'], ['provider.id'], ondelete='RESTRICT'),
-                    sa.ForeignKeyConstraint(['schema_id', 'schema_type'], ['schema.id', 'schema.type'], name='package_schema_fkey',
-                                            ondelete='RESTRICT'),
                     sa.PrimaryKeyConstraint('id')
                     )
     op.create_table('provider_user',
@@ -75,6 +100,20 @@ def upgrade():
                     sa.ForeignKeyConstraint(['resource_id'], ['resource.id'], ondelete='RESTRICT'),
                     sa.PrimaryKeyConstraint('package_id', 'resource_id')
                     )
+    op.create_table('package_tag',
+                    sa.Column('id', sa.String(), nullable=False),
+                    sa.Column('package_id', sa.String(), nullable=False),
+                    sa.Column('tag_id', sa.String(), nullable=False),
+                    sa.Column('tag_type', postgresql.ENUM(name='tagtype', create_type=False), nullable=False),
+                    sa.Column('user_id', sa.String(), nullable=True),
+                    sa.Column('data', postgresql.JSONB(astext_type=sa.Text()), nullable=False),
+                    sa.Column('timestamp', sa.TIMESTAMP(timezone=True), nullable=False),
+                    sa.CheckConstraint("tag_type = 'package'", name='package_tag_tag_type_check'),
+                    sa.ForeignKeyConstraint(['package_id'], ['package.id'], ondelete='CASCADE'),
+                    sa.ForeignKeyConstraint(['tag_id', 'tag_type'], ['tag.id', 'tag.type'], name='package_tag_tag_fkey', ondelete='CASCADE'),
+                    sa.ForeignKeyConstraint(['user_id'], ['user.id'], ondelete='RESTRICT'),
+                    sa.PrimaryKeyConstraint('id')
+                    )
     op.create_table('record_package',
                     sa.Column('record_id', sa.String(), nullable=False),
                     sa.Column('package_id', sa.String(), nullable=False),
@@ -107,10 +146,13 @@ def downgrade():
                     sa.PrimaryKeyConstraint('client_id', 'collection_id', name='client_collection_pkey')
                     )
     op.drop_table('record_package')
+    op.drop_table('package_tag')
     op.drop_table('package_resource')
     op.drop_table('archive_resource')
     op.drop_table('resource')
     op.drop_table('provider_user')
     op.drop_table('package')
+    op.drop_table('package_tag_audit')
+    op.drop_table('package_audit')
     op.drop_table('archive')
     # ### end Alembic commands ###

odp/db/models/package.py

@@ -1,56 +1,61 @@
 import uuid
 
-from sqlalchemy import CheckConstraint, Column, Enum, ForeignKey, ForeignKeyConstraint, String, TIMESTAMP
+from sqlalchemy import CheckConstraint, Column, Enum, ForeignKey, ForeignKeyConstraint, Identity, Integer, String, TIMESTAMP
 from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.ext.associationproxy import association_proxy
 from sqlalchemy.orm import relationship
 
-from odp.const.db import SchemaType
+from odp.const.db import AuditCommand, PackageStatus, TagType
 from odp.db import Base
 
 
 class Package(Base):
-    """A package represents a set of resources that constitute a
-    digital object, and includes original metadata from the provider."""
+    """A submission information package originating from a data provider."""
 
     __tablename__ = 'package'
 
-    __table_args__ = (
-        ForeignKeyConstraint(
-            ('schema_id', 'schema_type'), ('schema.id', 'schema.type'),
-            name='package_schema_fkey', ondelete='RESTRICT',
-        ),
-        CheckConstraint(
-            f"schema_type = '{SchemaType.metadata}'",
-            name='package_schema_type_check',
-        ),
-    )
-
     id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
-    metadata_ = Column(JSONB, nullable=False)
-    validity = Column(JSONB, nullable=False)
+    title = Column(String, nullable=False)
+    status = Column(Enum(PackageStatus), nullable=False)
     notes = Column(String)
     timestamp = Column(TIMESTAMP(timezone=True), nullable=False)
 
     provider_id = Column(String, ForeignKey('provider.id', ondelete='RESTRICT'), nullable=False)
     provider = relationship('Provider')
 
-    schema_id = Column(String, nullable=False)
-    schema_type = Column(Enum(SchemaType), nullable=False)
-    schema = relationship('Schema')
-
     # many-to-many package_resource entities are persisted by
     # assigning/removing Resource instances to/from resources
     package_resources = relationship('PackageResource', cascade='all, delete-orphan', passive_deletes=True)
     resources = association_proxy('package_resources', 'resource', creator=lambda r: PackageResource(resource=r))
 
+    # view of associated tags (one-to-many)
+    tags = relationship('PackageTag', viewonly=True)
+
     # view of associated record via one-to-many record_package relation
     # the plural 'records' is used because these attributes are collections,
     # although there can be only zero or one related record
     package_records = relationship('RecordPackage', viewonly=True)
     records = association_proxy('package_records', 'record')
 
-    _repr_ = 'id', 'provider_id', 'schema_id'
+    _repr_ = 'id', 'title', 'status', 'provider_id'
+
+
+class PackageAudit(Base):
+    """Package audit log."""
+
+    __tablename__ = 'package_audit'
+
+    id = Column(Integer, Identity(), primary_key=True)
+    client_id = Column(String, nullable=False)
+    user_id = Column(String)
+    command = Column(Enum(AuditCommand), nullable=False)
+    timestamp = Column(TIMESTAMP(timezone=True), nullable=False)
+
+    _id = Column(String, nullable=False)
+    _title = Column(String, nullable=False)
+    _status = Column(String, nullable=False)
+    _notes = Column(String)
+    _provider_id = Column(String, nullable=False)
 
 
 class PackageResource(Base):
@@ -67,3 +72,51 @@ class PackageResource(Base):
 
     package = relationship('Package', viewonly=True)
     resource = relationship('Resource')
+
+
+class PackageTag(Base):
+    """Tag instance model, representing a tag attached to a package."""
+
+    __tablename__ = 'package_tag'
+
+    __table_args__ = (
+        ForeignKeyConstraint(
+            ('tag_id', 'tag_type'), ('tag.id', 'tag.type'),
+            name='package_tag_tag_fkey', ondelete='CASCADE',
+        ),
+        CheckConstraint(
+            f"tag_type = '{TagType.package}'",
+            name='package_tag_tag_type_check',
+        ),
+    )
+
+    id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
+    package_id = Column(String, ForeignKey('package.id', ondelete='CASCADE'), nullable=False)
+    tag_id = Column(String, nullable=False)
+    tag_type = Column(Enum(TagType), nullable=False)
+    user_id = Column(String, ForeignKey('user.id', ondelete='RESTRICT'))
+
+    data = Column(JSONB, nullable=False)
+    timestamp = Column(TIMESTAMP(timezone=True), nullable=False)
+
+    package = relationship('Package')
+    tag = relationship('Tag')
+    user = relationship('User')
+
+
+class PackageTagAudit(Base):
+    """Package tag audit log."""
+
+    __tablename__ = 'package_tag_audit'
+
+    id = Column(Integer, Identity(), primary_key=True)
+    client_id = Column(String, nullable=False)
+    user_id = Column(String)
+    command = Column(Enum(AuditCommand), nullable=False)
+    timestamp = Column(TIMESTAMP(timezone=True), nullable=False)
+
+    _id = Column(String, nullable=False)
+    _package_id = Column(String, nullable=False)
+    _tag_id = Column(String, nullable=False)
+    _user_id = Column(String)
+    _data = Column(JSONB, nullable=False)