feat: begin pydantic v2 migration

.example.env

@@ -4,16 +4,14 @@ CDK_DEFAULT_REGION=[REQUIRED IF DEPLOYING TO EXISTING VPC]
 
 STAGE=[FILL ME IN]
 
-VEDA_DB_PGSTAC_VERSION=0.7.10
+VEDA_DB_PGSTAC_VERSION=0.9.1
 VEDA_DB_SCHEMA_VERSION=0.1.0
 VEDA_DB_SNAPSHOT_ID=[OPTIONAL BUT **REQUIRED** FOR ALL DEPLOYMENTS AFTER BASING DEPLOYMENT ON SNAPSHOT]
 VEDA_DB_PUBLICLY_ACCESSIBLE=TRUE
 VEDA_DB_USE_RDS_PROXY=[OPTIONAL]
 VEDA_DB_RDS_INSTANCE_CLASS=[OPTIONAL]
 VEDA_DB_RDS_INSTANCE_SIZE=[OPTIONAL]
 
-VEDA_DOMAIN_HOSTED_ZONE_ID=[OPTIONAL]
-VEDA_DOMAIN_HOSTED_ZONE_NAME=[OPTIONAL]
 VEDA_DOMAIN_API_PREFIX=[OPTIONAL CUSTOM PREFIX DECOUPLES API URL FROM A/B VERSION OF STACK DEPLOYED]
 
 VEDA_DOMAIN_ALT_HOSTED_ZONE_ID=[OPTIONAL SECOND HOSTED ZONE]

common/auth/setup.py

@@ -3,13 +3,13 @@
 
 from setuptools import find_packages, setup
 
-inst_reqs = ["cryptography>=42.0.5", "pyjwt>=2.8.0", "fastapi", "pydantic<2"]
+inst_reqs = ["cryptography>=42.0.5", "pyjwt>=2.8.0", "fastapi", "pydantic>=2.8.2"]
 
 setup(
     name="veda_auth",
     version="0.0.1",
     description="",
-    python_requires=">=3.7",
+    python_requires=">=3.9",
     packages=find_packages(),
     zip_safe=False,
     install_requires=inst_reqs,

config.py

@@ -2,7 +2,9 @@
 from getpass import getuser
 from typing import List, Optional
 
-from pydantic import BaseSettings, Field, constr
+from pydantic.v1.env_settings import BaseSettings
+from pydantic.v1.fields import Field
+from pydantic.v1.types import constr
 
 AwsSubnetId = constr(regex=r"^subnet-[a-z0-9]{17}$")
 

database/infrastructure/config.py

@@ -2,7 +2,9 @@
 from typing import Optional
 
 from aws_cdk import aws_ec2, aws_rds
-from pydantic import BaseSettings, Field, validator
+from pydantic.v1.class_validators import validator
+from pydantic.v1.env_settings import BaseSettings
+from pydantic.v1.fields import Field
 
 
 class vedaDBSettings(BaseSettings):

docker-compose.yml

@@ -97,7 +97,7 @@ services:
   database:
     container_name: veda.db
     platform: linux/amd64
-    image: ghcr.io/stac-utils/pgstac:v0.7.10
+    image: ghcr.io/stac-utils/pgstac:v0.9.1
     environment:
       - POSTGRES_USER=username
       - POSTGRES_PASSWORD=password
@@ -128,7 +128,7 @@ services:
       - PGPASSWORD=password
       - PGDATABASE=postgis
       - DYNAMODB_ENDPOINT=http://localhost:8085
-      - VEDA_DB_PGSTAC_VERSION=0.7.10
+      - VEDA_DB_PGSTAC_VERSION=0.9.1
     ports:
       - "8083:8083"
     command: bash -c "bash /tmp/scripts/wait-for-it.sh -t 120 -h database -p 5432 && python /asset/local.py"

ingest_api/infrastructure/config.py

@@ -2,12 +2,17 @@
 from typing import List, Optional
 
 import aws_cdk
-from pydantic import AnyHttpUrl, BaseSettings, Field, constr
+from pydantic import AnyHttpUrl, Field, constr
+from pydantic_settings import BaseSettings, SettingsConfigDict
 
-AwsArn = constr(regex=r"^arn:aws:iam::\d{12}:role/.+")
+AwsArn = constr(pattern=r"^arn:aws:iam::\d{12}:role/.+")
 
 
 class IngestorConfig(BaseSettings):
+    model_config = SettingsConfigDict(
+        case_sensitive=False, env_file=".env", env_prefix="VEDA_", extra="ignore"
+    )
+
     # S3 bucket names where TiTiler could do HEAD and GET Requests
     # specific private and public buckets MUST be added if you want to use s3:// urls
     # You can whitelist all bucket by setting `*`.
@@ -88,11 +93,6 @@ class IngestorConfig(BaseSettings):
         description="Raster API root path. Used to infer url of raster-api before app synthesis.",
     )
 
-    class Config:
-        case_sensitive = False
-        env_file = ".env"
-        env_prefix = "VEDA_"
-
     @property
     def stack_name(self) -> str:
         return f"veda-stac-ingestion-{self.stage}"

ingest_api/infrastructure/construct.py

@@ -52,7 +52,7 @@ def __init__(
             "RASTER_URL": config.veda_raster_api_cf_url,
             "ROOT_PATH": config.ingest_root_path,
             "STAGE": config.stage,
-            "COGNITO_DOMAIN": config.cognito_domain,
+            "COGNITO_DOMAIN": str(config.cognito_domain),
         }
 
         build_api_lambda_params = {

ingest_api/runtime/requirements.txt

@@ -1,17 +1,17 @@
-# Waiting for https://github.com/stac-utils/stac-pydantic/pull/116 and 117
 cryptography>=42.0.5
 ddbcereal==2.1.1
-fastapi>=0.109.1
+fastapi>=0.114.1
 fsspec==2023.3.0
 mangum>=0.15.0
 orjson>=3.6.8
 psycopg[binary,pool]>=3.0.15
-pydantic_ssm_settings>=0.2.0
-pydantic>=1.10.12
+# pydantic_ssm_settings>=0.2.0
+pydantic>=2.8.2
+pydantic-settings>=2.5.2
 python-multipart==0.0.7
 requests>=2.27.1
 s3fs==2023.3.0
-stac-pydantic @ git+https://github.com/ividito/stac-pydantic.git@3f4cb381c85749bb4b15d1181179057ec0f51a94
+stac-pydantic>=3.1.2
 boto3==1.24.59
 aws_xray_sdk>=2.6.0,<3
 aws-lambda-powertools>=1.18.0

ingest_api/runtime/src/collection_publisher.py

@@ -4,7 +4,7 @@
 from src.schemas import DashboardCollection
 from src.utils import IngestionType, get_db_credentials, load_into_pgstac
 from src.vedaloader import VEDALoader
-from stac_pydantic import Item
+from stac_pydantic.item import Item
 
 
 class CollectionPublisher:

ingest_api/runtime/src/config.py

@@ -2,11 +2,11 @@
 from getpass import getuser
 from typing import Optional
 
-from pydantic import AnyHttpUrl, BaseSettings, Field, constr
-from pydantic_ssm_settings import AwsSsmSourceConfig
+from pydantic import AnyHttpUrl, Field, constr
+from pydantic_settings import BaseSettings, SettingsConfigDict
 from veda_auth import VedaAuth
 
-AwsArn = constr(regex=r"^arn:aws:iam::\d{12}:role/.+")
+AwsArn = constr(pattern=r"^arn:aws:iam::\d{12}:role/.+")
 
 
 class Settings(BaseSettings):
@@ -47,8 +47,7 @@ def cognito_token_url(self) -> AnyHttpUrl:
         """Cognito user pool token and refresh url"""
         return f"{self.cognito_domain}/oauth2/token"
 
-    class Config(AwsSsmSourceConfig):
-        env_file = ".env"
+    model_config = SettingsConfigDict(env_file=".env")
 
     @classmethod
     def from_ssm(cls, stack: str):

ingest_api/runtime/src/schema_helpers.py

@@ -1,7 +1,7 @@
 from datetime import datetime
 from typing import List, Union
 
-from pydantic import BaseModel, root_validator
+from pydantic import BaseModel, model_validator
 from stac_pydantic.collection import Extent, TimeInterval
 
 # Smaller utility models to support the larger models in schemas.py
@@ -24,7 +24,7 @@ class BboxExtent(BaseModel):
     xmax: float
     ymax: float
 
-    @root_validator
+    @model_validator(mode="before")
     def check_extent(cls, v):
         # mins must be below maxes
         if v["xmin"] >= v["xmax"] or v["ymin"] >= v["ymax"]:
@@ -43,7 +43,7 @@ class TemporalExtent(BaseModel):
     startdate: datetime
     enddate: datetime
 
-    @root_validator
+    @model_validator(mode="before")
     def check_dates(cls, v):
         if v["startdate"] >= v["enddate"]:
             raise ValueError("Invalid extent - startdate must be before enddate")

ingest_api/runtime/src/schemas.py

@@ -7,17 +7,13 @@
 from urllib.parse import urlparse
 
 import src.validators as validators
-from pydantic import (
-    BaseModel,
-    ConfigDict,
-    Field,
-    Json,
-    PositiveInt,
-    error_wrappers,
-    validator,
-)
+from pydantic import BaseModel, Json, PositiveInt
+from pydantic import field_validator, ConfigDict, error_wrappers, Field
+
 from src.schema_helpers import SpatioTemporalExtent
-from stac_pydantic import Collection, Item, shared
+from stac_pydantic import shared
+from stac_pydantic.collection import Collection
+from stac_pydantic.item import Item
 from stac_pydantic.links import Link
 
 from fastapi.encoders import jsonable_encoder
@@ -32,7 +28,7 @@ class LinkWithExtraFields(Link):
 
 
 class AccessibleAsset(shared.Asset):
-    @validator("href")
+    @field_validator("href")
     def is_accessible(cls, href):
         url = urlparse(href)
 
@@ -51,7 +47,7 @@ def is_accessible(cls, href):
 class AccessibleItem(Item):
     assets: Dict[str, AccessibleAsset]
 
-    @validator("collection")
+    @field_validator("collection")
     def exists(cls, collection):
         validators.collection_exists(collection_id=collection)
         return collection
@@ -65,8 +61,7 @@ class DashboardCollection(Collection):
     assets: Optional[Dict]
     extent: SpatioTemporalExtent
 
-    class Config:
-        allow_population_by_field_name = True
+    model_config: ConfigDict = ConfigDict(populate_by_name = True)
 
 
 class Status(str, enum.Enum):
@@ -103,17 +98,14 @@ class AuthResponse(BaseModel):
 class Ingestion(BaseModel):
     id: str = Field(..., description="ID of the STAC item")
     status: Status = Field(..., description="Status of the ingestion")
-    message: Optional[str] = Field(
-        None, description="Message returned from the step function."
-    )
+    message: Optional[str] = None # Message returned from the step function.
     created_by: str = Field(..., description="User who created the ingestion")
     created_at: datetime = Field(None, description="Timestamp of ingestion creation")
     updated_at: datetime = Field(None, description="Timestamp of ingestion update")
 
     item: Union[Item, Json[Item]] = Field(..., description="STAC item to ingest")
 
-    @validator("created_at", pre=True, always=True, allow_reuse=True)
-    @validator("updated_at", pre=True, always=True, allow_reuse=True)
+    @field_validator("created_at","updated_at", mode="before")
     def set_ts_now(cls, v):
         return v or datetime.now()
 
@@ -133,19 +125,21 @@ def save(self, db: "services.Database"):
     def dynamodb_dict(self, by_alias=True):
         """DynamoDB-friendly serialization"""
         # convert to dictionary
-        output = self.dict(exclude={"item"})
+        output = self.model_dump(exclude={"item"}, by_alias=by_alias)
 
         # add STAC item as string
         output["item"] = self.item.json()
 
         # make JSON-friendly (will be able to do with Pydantic V2, https://github.com/pydantic/pydantic/issues/1409#issuecomment-1423995424)
         return jsonable_encoder(output)
 
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
 
 class ListIngestionRequest(BaseModel):
     status: Status = Field(Status.queued, description="Status of the ingestion")
-    limit: PositiveInt = Field(None, description="Limit number of results")
-    next: Optional[str] = Field(None, description="Next token (json) to load")
+    limit: Optional[PositiveInt] = None
+    next: Optional[str] = None # Next token (json) to load
 
     def __post_init_post_parse__(self) -> None:
         # https://github.com/tiangolo/fastapi/issues/1474#issuecomment-1049987786
@@ -165,15 +159,16 @@ def __post_init_post_parse__(self) -> None:
                     )
                 ]
             )
+    model_config = ConfigDict(arbitrary_types_allowed=True)
 
 
 class ListIngestionResponse(BaseModel):
     items: List[Ingestion] = Field(
         ..., description="List of STAC items from ingestion."
     )
-    next: Optional[str] = Field(None, description="Next token (json) to load")
+    next: Optional[str] = None # Next token (json) to load
 
-    @validator("next", pre=True)
+    @field_validator("next", mode="before")
     def b64_encode_next(cls, next):
         """
         Base64 encode next parameter for easier transportability

ingest_api/runtime/src/services.py

@@ -4,7 +4,8 @@
 import src.schemas as schemas
 from boto3.dynamodb import conditions
 from boto3.dynamodb.types import DYNAMODB_CONTEXT
-from pydantic import parse_obj_as
+from pydantic import TypeAdapter
+
 
 if TYPE_CHECKING:
     from mypy_boto3_dynamodb.service_resource import Table
@@ -25,24 +26,25 @@ def fetch_one(self, username: str, ingestion_id: str):
             Key={"created_by": username, "id": ingestion_id},
         )
         try:
-            return schemas.Ingestion.parse_obj(response["Item"])
+            return schemas.Ingestion.model_validate(response["Item"])
         except KeyError:
             raise NotInDb("Record not found")
 
     def fetch_many(
         self, status: str, next: Optional[dict] = None, limit: Optional[int] = None
     ) -> schemas.ListIngestionResponse:
+        adapter = TypeAdapter(List[schemas.Ingestion])
         response = self.table.query(
             IndexName="status",
             KeyConditionExpression=conditions.Key("status").eq(status),
             **{"Limit": limit} if limit else {},
             **{"ExclusiveStartKey": next} if next else {},
         )
+        items = adapter.validate_python(response["Items"])
         return {
-            "items": parse_obj_as(List[schemas.Ingestion], response["Items"]),
+            "items": items,
             "next": response.get("LastEvaluatedKey"),
         }
 
-
 class NotInDb(Exception):
     ...

ingest_api/runtime/src/validators.py

@@ -90,7 +90,7 @@ def collection_exists(collection_id: str) -> bool:
     from src.main import settings
 
     url = "/".join(
-        f'{url.strip("/")}' for url in [settings.stac_url, "collections", collection_id]
+        f'{str(url).strip("/")}' for url in [settings.stac_url, "collections", collection_id]
     )
 
     if (response := requests.get(url)).ok:

network/infrastructure/config.py

@@ -1,7 +1,7 @@
 """Configuration options for the VPC."""
 from typing import Dict
 
-from pydantic import BaseSettings
+from pydantic.v1.env_settings import BaseSettings
 
 
 # https://medium.com/aws-activate-startup-blog/practical-vpc-design-8412e1a18dcc#.bmeh8m3si

raster_api/infrastructure/config.py

@@ -4,7 +4,8 @@
 
 from typing import Dict, List, Optional
 
-from pydantic import BaseSettings, Field
+from pydantic.v1.env_settings import BaseSettings
+from pydantic.v1.fields import Field
 
 
 class vedaRasterSettings(BaseSettings):

s3_website/infrastructure/config.py

@@ -3,7 +3,8 @@
 """
 from typing import Optional
 
-from pydantic import BaseSettings, Field
+from pydantic.v1.env_settings import BaseSettings
+from pydantic.v1.fields import Field
 
 
 class vedaS3WebsiteSettings(BaseSettings):