Generate work for document

.github/workflows/tests.yml

@@ -34,16 +34,25 @@ jobs:
       - name: Install dependencies
         run: |
           pip install uv
-          uv pip install --system -r <(uv pip compile --all-extras pyproject.toml)
+          uv sync --all-extras
+          pip install .[ci]
 
       - name: Start Docker stack
         run: |
           cp .mockenv .env
           docker compose -f .docker/postgres-compose.yaml up -d
 
-      - name: Run tests
-        run: pytest tests
+      - name: Run tests with coverage
+        run: |
+          coverage run -m pytest tests
+          coverage report
+          coverage xml
 
       - name: Stop Docker stack
         if: always()
-        run: docker compose -f compose.yaml -f compose.development.override.yaml down
+        run: docker compose -f .docker/postgres-compose.yaml down
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          file: coverage.xml

.gitignore

@@ -12,4 +12,5 @@ _version.py
 pdfs/*
 test-pdf/*.pdf
 dsst_etl.egg-info/*
-uv.lock
+uv.lock
+.coverage

.mockenv

@@ -4,9 +4,12 @@ POSTGRES_USER=postgres
 POSTGRES_PASSWORD=postgres
 POSTGRES_HOST=localhost
 POSTGRES_PORT=5432
+POSTGRES_DB=pdx
 
 # requests will be rejected without user-agent
 USER_AGENT=
 
 # metapub; get key from https://pubmed.ncbi.nlm.nih.gov for faster requests
 NCBI_API_KEY=
+
+S3_BUCKET_NAME=osm-pdf-uploads

README.md

@@ -124,7 +124,7 @@ docker compose -f .docker/postgres-compose.yaml down -v
 pre-commit install
 
 # run the pre-commit hooks on all files
-pre-commit run -all
+   pre-commit run --all-files
 
 # run the tests
 pytest

alembic/versions/4a908d10b459_update_non_null_of_document_in_works_.py

@@ -0,0 +1,40 @@
+"""update non-null of document in works table
+
+Revision ID: 4a908d10b459
+Revises: 360c65a62392
+Create Date: 2024-11-21 16:40:56.966690
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '4a908d10b459'
+down_revision: Union[str, None] = '360c65a62392'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('works', 'initial_document_id',
+               existing_type=sa.INTEGER(),
+               nullable=True)
+    op.alter_column('works', 'primary_document_id',
+               existing_type=sa.INTEGER(),
+               nullable=True)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('works', 'primary_document_id',
+               existing_type=sa.INTEGER(),
+               nullable=False)
+    op.alter_column('works', 'initial_document_id',
+               existing_type=sa.INTEGER(),
+               nullable=False)
+    # ### end Alembic commands ###

dsst_etl/__init__.py

@@ -2,6 +2,7 @@
 DSST ETL Package
 """
 
+import logging
 import os
 
 from dotenv import load_dotenv
@@ -10,6 +11,9 @@
 
 load_dotenv()
 
+logger = logging.getLogger(__name__)
+
+
 def get_db_url():
     database_url = (
         "postgresql://"
@@ -20,9 +24,11 @@ def get_db_url():
     )
     return database_url
 
+
 def get_db_engine():
     return create_engine(get_db_url())
 
+
 engine = get_db_engine()
 SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
 

dsst_etl/_utils.py

@@ -6,7 +6,7 @@ def get_compute_context_id():
 
 
 def get_bucket_name():
-    bucket_name = os.getenv('S3_BUCKET_NAME')
+    bucket_name = os.getenv("S3_BUCKET_NAME")
     if not bucket_name:
         raise ValueError("S3_BUCKET_NAME environment variable is not set")
-    return bucket_name
+    return bucket_name

dsst_etl/db.py

@@ -1,13 +1,11 @@
-import os
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy_utils import create_database, database_exists
 
 from dsst_etl import get_db_engine
-from sqlalchemy import create_engine
-from sqlalchemy.orm import sessionmaker
 
 from .models import Base
 
 
-
 def get_db_session():
     engine = get_db_engine()
     Session = sessionmaker(bind=engine)
@@ -16,4 +14,6 @@ def get_db_session():
 
 def init_db():
     engine = get_db_engine()
+    if not database_exists(engine.url):
+        create_database(engine.url)
     Base.metadata.create_all(engine)

dsst_etl/models.py

@@ -1,8 +1,5 @@
-from datetime import datetime
-
-from sqlalchemy import Column, DateTime, ForeignKey, Integer, LargeBinary, String, Text
+from sqlalchemy import Column, DateTime, ForeignKey, Integer, String, Text
 from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.orm import relationship
 from sqlalchemy.sql import func
 
 Base = declarative_base()
@@ -14,12 +11,8 @@ class Works(Base):
     id = Column(Integer, primary_key=True)
     created_at = Column(DateTime, default=func.now())
     modified_at = Column(DateTime, default=func.now(), onupdate=func.now())
-    initial_document_id = Column(
-        Integer, ForeignKey("documents.id"), nullable=False
-    )
-    primary_document_id = Column(
-        Integer, ForeignKey("documents.id"), nullable=False
-    )
+    initial_document_id = Column(Integer, ForeignKey("documents.id"), nullable=True)
+    primary_document_id = Column(Integer, ForeignKey("documents.id"), nullable=True)
     provenance_id = Column(Integer, ForeignKey("provenance.id"))
 
     # Relationships
@@ -30,7 +23,7 @@ class Works(Base):
 
 class Documents(Base):
     __tablename__ = "documents"
-    
+
     id = Column(Integer, primary_key=True)
     hash_data = Column(String, nullable=False, unique=True)
     created_at = Column(DateTime, default=func.now())
@@ -42,7 +35,6 @@ class Documents(Base):
     # provenance = relationship("Provenance")
 
 
-
 class Provenance(Base):
     __tablename__ = "provenance"
 
@@ -52,4 +44,3 @@ class Provenance(Base):
     compute = Column(Text)
     personnel = Column(Text)
     comment = Column(Text)
-