diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 124a466..902f158 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -16,6 +16,19 @@ jobs: env: AWS_DEFAULT_REGION: us-west-2 + services: + pgstac: + image: ghcr.io/stac-utils/pgstac:v0.7.10 + env: + POSTGRES_USER: username + POSTGRES_PASSWORD: password + POSTGRES_DB: postgis + PGUSER: username + PGPASSWORD: password + PGDATABASE: postgis + ports: + - 5432:5432 + steps: - uses: actions/checkout@v3 diff --git a/README.md b/README.md index 372e758..edc1373 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,20 @@ This script is also available at `scripts/sync_env.sh`, which can be invoked wit . scripts/sync_env.sh stac-ingestor-env-secret- ``` +## Testing + +```shell +pytest +``` + +Some tests require a locally-running **pgstac** database, and will be skipped if there isn't one at `postgresql://username:password@localhost:5432/postgis`. +To run the **pgstac** tests: + +```shell +docker compose up -d +pytest +docker compose down +``` ## License diff --git a/api/requirements.txt b/api/requirements.txt index c6828dc..832cbbd 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -10,6 +10,7 @@ psycopg[binary,pool]>=3.0.15 pydantic_ssm_settings>=0.2.0 pydantic>=1.9.0,<2 pypgstac==0.7.10 +pystac[jsonschema]>=1.8.4 python-multipart==0.0.5 requests>=2.27.1 s3fs==2023.3.0 diff --git a/api/src/collection.py b/api/src/collection.py index 1e24e67..611a137 100644 --- a/api/src/collection.py +++ b/api/src/collection.py @@ -1,10 +1,11 @@ import os -from typing import Union +from typing import Optional, Union import fsspec import xarray as xr import xstac from pypgstac.db import PgstacDB + from src.schemas import ( COGDataset, DashboardCollection, @@ -13,6 +14,7 @@ ZarrDataset, ) from src.utils import ( + DbCreds, IngestionType, convert_decimals_to_float, get_db_credentials, @@ -40,8 +42,10 @@ class Publisher: "type": "Collection", "stac_version": "1.0.0", } + db_creds: Optional[DbCreds] - def __init__(self) -> None: + def __init__(self, db_creds: Optional[DbCreds] = None) -> None: + self.db_creds = db_creds self.func_map = { DataType.zarr: self.create_zarr_collection, DataType.cog: self.create_cog_collection, @@ -147,9 +151,9 @@ def ingest(self, collection: DashboardCollection): does necessary preprocessing, and loads into the PgSTAC collection table """ - creds = get_db_credentials(os.environ["DB_SECRET_ARN"]) + db_creds = self._get_db_credentials() collection = [convert_decimals_to_float(collection.dict(by_alias=True))] - with PgstacDB(dsn=creds.dsn_string, debug=True) as db: + with PgstacDB(dsn=db_creds.dsn_string, debug=True) as db: load_into_pgstac( db=db, ingestions=collection, table=IngestionType.collections ) @@ -158,7 +162,13 @@ def delete(self, collection_id: str): """ Deletes the collection from the database """ - creds = get_db_credentials(os.environ["DB_SECRET_ARN"]) - with PgstacDB(dsn=creds.dsn_string, debug=True) as db: + db_creds = self._get_db_credentials() + with PgstacDB(dsn=db_creds.dsn_string, debug=True) as db: loader = VEDALoader(db=db) loader.delete_collection(collection_id) + + def _get_db_credentials(self) -> DbCreds: + if self.db_creds: + return self.db_creds + else: + return get_db_credentials(os.environ["DB_SECRET_ARN"]) diff --git a/api/tests/conftest.py b/api/tests/conftest.py index 7cbe965..64e8441 100644 --- a/api/tests/conftest.py +++ b/api/tests/conftest.py @@ -1,9 +1,15 @@ +import datetime import os +from typing import Generator import boto3 +import psycopg import pytest from fastapi.testclient import TestClient from moto import mock_dynamodb, mock_ssm +from pypgstac.db import PgstacDB +from pystac import Collection, Extent, SpatialExtent, TemporalExtent +from src.schemas import DashboardCollection from stac_pydantic import Item @@ -145,6 +151,21 @@ def example_stac_item(): } +@pytest.fixture +def dashboard_collection() -> DashboardCollection: + collection = Collection( + "test-collection", + "A test collection", + Extent( + SpatialExtent( + [[-180, -90, 180, 90]], + ), + TemporalExtent([[datetime.datetime.utcnow(), None]]), + ), + ) + return DashboardCollection.parse_obj(collection.to_dict()) + + @pytest.fixture def example_ingestion(example_stac_item): from src import schemas @@ -155,3 +176,14 @@ def example_ingestion(example_stac_item): status=schemas.Status.queued, item=Item.parse_obj(example_stac_item), ) + + +@pytest.fixture +def pgstac() -> Generator[PgstacDB, None, None]: + dsn = "postgresql://username:password@localhost:5432/postgis" + try: + psycopg.connect(dsn) + except Exception: + pytest.skip(f"could not connect to pgstac database: {dsn}") + with PgstacDB(dsn, commit_on_exit=False) as db: + yield db diff --git a/api/tests/test_collection.py b/api/tests/test_collection.py new file mode 100644 index 0000000..ed02e2a --- /dev/null +++ b/api/tests/test_collection.py @@ -0,0 +1,32 @@ +import pytest +from pypgstac.db import PgstacDB +from pystac import Collection +from src.collection import Publisher +from src.schemas import DashboardCollection +from src.utils import DbCreds + + +@pytest.fixture +def publisher() -> Publisher: + return Publisher( + DbCreds( + username="username", + password="password", + host="localhost", + port=5432, + dbname="postgis", + engine="postgresql", + ) + ) + + +def test_ingest( + pgstac: PgstacDB, publisher: Publisher, dashboard_collection: DashboardCollection +) -> None: + publisher.ingest(dashboard_collection) + collection = Collection.from_dict( + pgstac.query_one( + r"SELECT * FROM pgstac.get_collection(%s)", [dashboard_collection.id] + ) + ) + collection.validate() diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..e1537c0 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,15 @@ +version: '3' +services: + database: + container_name: pgstac + image: ghcr.io/stac-utils/pgstac:v0.7.10 + environment: + - POSTGRES_USER=username + - POSTGRES_PASSWORD=password + - POSTGRES_DB=postgis + - PGUSER=username + - PGPASSWORD=password + - PGDATABASE=postgis + ports: + - "5432:5432" + command: postgres -N 500