feat: File management for fetch-binary and fetch-science functions

.devcontainer/devcontainer.json

@@ -34,6 +34,7 @@
  "WEBPODA_AUTH_CODE": "${localEnv:WEBPODA_AUTH_CODE}",
  "SDC_AUTH_CODE": "${localEnv:SDC_AUTH_CODE}",
  "IMAP_DATA_ACCESS_URL": "${localEnv:IMAP_DATA_ACCESS_URL}",
+ "SQLALCHEMY_URL": "${localEnv:SQLALCHEMY_URL}",
  // Define WireMock variables to connect Docker outside of Docker.
  "WIREMOCK_DIND": "1",
  "TESTCONTAINERS_HOST_OVERRIDE": "host.docker.internal"

.github/workflows/ci.yml

@@ -16,11 +16,14 @@ on:
  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:
 
+# write to checks/pull-request extra permission needed by 5monkeys/cobertura-action to post coverage stats
+# write packages needed by docker image step
 permissions:
  id-token: write
  contents: write
  checks: write
  packages: write
+ pull-requests: write
 
 env:
  PREFERED_PYTHON_VERSION: '3.12'
@@ -150,6 +153,13 @@ jobs:
  path: 'test-results.xml'
  reporter: java-junit
 
+ - name: Coverage Report
+ uses: 5monkeys/cobertura-action@v14
+ with:
+ report_name: Coverage Report (${{ matrix.python-versions }})
+ path: "coverage.xml"
+ minimum_coverage: 80
+
  - name: Create Release ${{github.ref_name}} & upload artifacts
  uses: softprops/action-gh-release@v2
  if: ${{ startsWith(github.ref, 'refs/tags/') }}
@@ -160,6 +170,44 @@ jobs:
  files: |
  dist/${{ env.PACKAGE_NAME }}_python${{matrix.python-versions}}_${{ env.PACKAGE_VERSION }}.zip
 
+ test_on_windows:
+ strategy:
+ matrix:
+ python-versions: ['3.10', '3.11', '3.12']
+ os: [windows-latest]
+ runs-on: ${{ matrix.os }}
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-python@v5
+ with:
+ python-version: ${{ matrix.python-versions }}
+
+ - name: Install dependencies
+ run: |
+ python -m pip install --upgrade pip
+ pip install poetry
+ poetry install
+
+ - name: Run tests
+ run: poetry run pytest -s --cov-config=.coveragerc --cov=src --cov-append --cov-report=xml --cov-report term-missing --cov-report=html --junitxml=test-results.xml tests
+
+ - name: Upload Coverage report
+ uses: actions/upload-artifact@v4
+ if: matrix.python-versions == env.PREFERED_PYTHON_VERSION
+ with:
+ name: CoverageReport_${{ matrix.os }}_python${{matrix.python-versions}}_${{ env.PACKAGE_VERSION }}
+ path: htmlcov
+ if-no-files-found: error
+
+ - name: Test Report
+ uses: dorny/test-reporter@v1
+ if: success() || failure()
+ with:
+ name: Test Results (${{ matrix.os }}) (${{ matrix.python-versions }})
+ path: 'test-results.xml'
+ reporter: java-junit
+
+
  build_single_file_binary:
  strategy:
  matrix:

.gitignore

@@ -135,3 +135,4 @@ site/
 .work
 /output
 dev.env
+debug

deploy/Dockerfile

@@ -11,7 +11,7 @@ ENV PYTHONDONTWRITEBYTECODE=1
 ENV PYTHONUNBUFFERED=1
 
 # Install the postgres client and any other compile time dependencies needed to build our app
-RUN apt-get update && apt-get install -y libpq-dev gcc
+RUN apt-get update && apt-get install -y libpq-dev gcc git
 
 # Creates a non-root user with an explicit UID and adds permission to access the /app folder
 # For more info, please refer to https://aka.ms/vscode-docker-python-configure-containers

pack.sh

@@ -18,7 +18,7 @@ poetry build
 
 # output a requierments.txt file used by docker during the build
 poetry self add poetry-plugin-export
-poetry export --format=requirements.txt > dist/requirements.txt
+poetry export --without-hashes --format=requirements.txt > dist/requirements.txt
 
 # move the files into a folder with the python version
 mkdir -p dist/python$PYTHON_VERSION

pyproject.toml

@@ -1,6 +1,7 @@
 [project]
 requires-python = ">=3.10"
 name = "imap-mag"
+version = "0.1.0"
 
 [tool.poetry]
 name = "imap-mag"
@@ -28,19 +29,20 @@ alembic = "^1.13.2"
 sqlalchemy-utils = "^0.41.2"
 requests = "^2.32.3"
 pandas = "^2.2.2"
-imap-data-access = "^0.7.0"
+imap-data-access = "^0.9.0"
 cdflib = "^1.3.1"
 psycopg = {extras = ["binary"], version = "^3.2.1"}
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^8.3.1"
 pytest-cov = "^5.0.0"
+pytest-mock = "^3.14.0"
 pyinstaller = "^6.5.0"
 pre-commit = "^3.8.0"
 ruff = "^0.5.4"
-wiremock = "^2.6.1"
 docker = "^7.1.0"
 testcontainers = "^4.7.2"
+wiremock = {git = "https://github.com/ImperialCollegeLondon/python-wiremock.git", rev = "fix-test-containers-on-windows"}
 
 [tool.poetry.scripts]
 # can execute via poetry, e.g. `poetry run imap-mag hello world`

run-docker.sh

@@ -20,9 +20,9 @@ if [ "$1" == "debug" ] || [ "$1" == "DEBUG" ] || [ "$1" == "-i" ]; then
  $IMAGE_NAME
 elif [ -z "$1" ]; then # no args passed
  docker run --rm -it \
-  --env-file dev.env \
-  -v /mnt/imap-data:/data \
-  $IMAGE_NAME
+ --env-file dev.env \
+ -v /mnt/imap-data:/data \
+ $IMAGE_NAME
 else
  echo "Extra arguments: $@"
  docker run --rm -it \

src/imap_db/migrations/versions/2024_08_09-669111c45c37_added_version_hash_date_and_software_.py

@@ -0,0 +1,52 @@
+"""Added version, hash, date and software version columns
+
+Revision ID: 669111c45c37
+Revises: d0457f3e98c8
+Create Date: 2024-08-09 13:35:21.578940
+
+"""
+
+from datetime import datetime
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "669111c45c37"
+down_revision = "d0457f3e98c8"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.add_column(
+ "files", sa.Column("version", sa.Integer(), nullable=False, default=0)
+ )
+ op.add_column(
+ "files", sa.Column("hash", sa.String(length=64), nullable=False, default="")
+ )
+ op.add_column(
+ "files",
+ sa.Column(
+ "date", sa.DateTime(), nullable=False, default=datetime.fromtimestamp(0)
+ ),
+ )
+ op.add_column(
+ "files",
+ sa.Column(
+ "software_version", sa.String(length=16), nullable=False, default="0.0.0"
+ ),
+ )
+ op.create_unique_constraint(None, "files", ["path"])
+ # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.drop_constraint(None, "files", type_="unique")
+ op.drop_column("files", "software_version")
+ op.drop_column("files", "date")
+ op.drop_column("files", "hash")
+ op.drop_column("files", "version")
+ # ### end Alembic commands ###

src/imap_db/model.py

@@ -1,4 +1,6 @@
-from sqlalchemy import String
+from datetime import datetime
+
+from sqlalchemy import DateTime, Integer, String
 from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
 
 
@@ -8,9 +10,14 @@ class Base(DeclarativeBase):
 
 class File(Base):
  __tablename__ = "files"
+
  id: Mapped[int] = mapped_column(primary_key=True)
  name: Mapped[str] = mapped_column(String(128))
- path: Mapped[str] = mapped_column(String(256))
+ path: Mapped[str] = mapped_column(String(256), unique=True)
+ version: Mapped[int] = mapped_column(Integer())
+ hash: Mapped[str] = mapped_column(String(64))
+ date: Mapped[datetime] = mapped_column(DateTime())
+ software_version: Mapped[str] = mapped_column(String(16))
 
  def __repr__(self) -> str:
  return f"<File {self.id} (name={self.name}, path={self.path})>"

src/imap_mag/DB.py

@@ -1,11 +1,34 @@
+import abc
+import logging
 import os
+from pathlib import Path
 
+import typer
 from imap_db.model import File
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker
 
+from imap_mag import __version__
+from imap_mag.outputManager import IFileMetadataProvider, IOutputManager, generate_hash
+
+
+class IDatabase(abc.ABC):
+ """Interface for database manager."""
+
+ def insert_file(self, file: File) -> None:
+ """Insert a file into the database."""
+ self.insert_files([file])
+ pass
+
+ @abc.abstractmethod
+ def insert_files(self, files: list[File]) -> None:
+ """Insert a list of files into the database."""
+ pass
+
+
+class Database(IDatabase):
+ """Database manager."""
 
-class DB:
  def __init__(self, db_url=None):
  env_url = os.getenv("SQLALCHEMY_URL")
  if db_url is None and env_url is not None:
@@ -16,10 +39,12 @@ def __init__(self, db_url=None):
  "No database URL provided. Consider setting SQLALCHEMY_URL environment variable."
  )
 
+ # TODO: Check database is available
+
  self.engine = create_engine(db_url)
  self.Session = sessionmaker(bind=self.engine)
 
- def insert_files(self, files: list[File]):
+ def insert_files(self, files: list[File]) -> None:
  session = self.Session()
  try:
  for file in files:
@@ -40,3 +65,60 @@ def insert_files(self, files: list[File]):
  raise e
  finally:
  session.close()
+
+
+class DatabaseOutputManager(IOutputManager):
+ """Decorator for adding files to database as well as output."""
+
+ __output_manager: IOutputManager
+ __database: IDatabase
+
+ def __init__(
+ self, output_manager: IOutputManager, database: Database | None = None
+ ):
+ """Initialize database and output manager."""
+
+ self.__output_manager = output_manager
+
+ if database is None:
+ self.__database = Database()
+ else:
+ self.__database = database
+
+ def add_file(
+ self, original_file: Path, metadata_provider: IFileMetadataProvider
+ ) -> tuple[Path, IFileMetadataProvider]:
+ (destination_file, metadata_provider) = self.__output_manager.add_file(
+ original_file, metadata_provider
+ )
+
+ file_hash: str = generate_hash(original_file)
+
+ if not (
+ destination_file.exists() and (generate_hash(destination_file) == file_hash)
+ ):
+ logging.error(
+ f"File {destination_file} does not exist or is not the same as original {original_file}."
+ )
+ destination_file.unlink(missing_ok=True)
+ raise typer.Abort()
+
+ logging.info(f"Inserting {destination_file} into database.")
+
+ try:
+ self.__database.insert_file(
+ File(
+ name=destination_file.name,
+ path=destination_file.absolute().as_posix(),
+ version=metadata_provider.version,
+ hash=file_hash,
+ date=metadata_provider.date,
+ software_version=__version__,
+ )
+ )
+ except Exception as e:
+ logging.error(f"Error inserting {destination_file} into database: {e}")
+ destination_file.unlink()
+ raise e
+
+ return (destination_file, metadata_provider)

src/imap_mag/__init__.py

@@ -0,0 +1,13 @@
+"""The main module for project."""
+
+from importlib.metadata import PackageNotFoundError, version
+
+
+def get_version() -> str:
+ try:
+ return version("imap-mag")
+ except PackageNotFoundError:
+ print("IMAP MAG CLI Version unknown, not installed via pip.")
+
+
+__version__ = get_version()