diff --git a/.example.env b/.example.env index c3fb4b62..05ca264c 100644 --- a/.example.env +++ b/.example.env @@ -4,7 +4,7 @@ CDK_DEFAULT_REGION=[REQUIRED IF DEPLOYING TO EXISTING VPC] STAGE=[FILL ME IN] -VEDA_DB_PGSTAC_VERSION=0.7.10 +VEDA_DB_PGSTAC_VERSION=0.9.1 VEDA_DB_SCHEMA_VERSION=0.1.0 VEDA_DB_SNAPSHOT_ID=[OPTIONAL BUT **REQUIRED** FOR ALL DEPLOYMENTS AFTER BASING DEPLOYMENT ON SNAPSHOT] VEDA_DB_PUBLICLY_ACCESSIBLE=TRUE @@ -12,8 +12,6 @@ VEDA_DB_USE_RDS_PROXY=[OPTIONAL] VEDA_DB_RDS_INSTANCE_CLASS=[OPTIONAL] VEDA_DB_RDS_INSTANCE_SIZE=[OPTIONAL] -VEDA_DOMAIN_HOSTED_ZONE_ID=[OPTIONAL] -VEDA_DOMAIN_HOSTED_ZONE_NAME=[OPTIONAL] VEDA_DOMAIN_API_PREFIX=[OPTIONAL CUSTOM PREFIX DECOUPLES API URL FROM A/B VERSION OF STACK DEPLOYED] VEDA_DOMAIN_ALT_HOSTED_ZONE_ID=[OPTIONAL SECOND HOSTED ZONE] diff --git a/common/auth/setup.py b/common/auth/setup.py index a076c143..1b08a04b 100644 --- a/common/auth/setup.py +++ b/common/auth/setup.py @@ -3,13 +3,13 @@ from setuptools import find_packages, setup -inst_reqs = ["cryptography>=42.0.5", "pyjwt>=2.8.0", "fastapi", "pydantic<2"] +inst_reqs = ["cryptography>=42.0.5", "pyjwt>=2.8.0", "fastapi", "pydantic>=2.8.2"] setup( name="veda_auth", version="0.0.1", description="", - python_requires=">=3.7", + python_requires=">=3.9", packages=find_packages(), zip_safe=False, install_requires=inst_reqs, diff --git a/config.py b/config.py index c792b1b7..f44d4b85 100644 --- a/config.py +++ b/config.py @@ -2,7 +2,9 @@ from getpass import getuser from typing import List, Optional -from pydantic import BaseSettings, Field, constr +from pydantic.v1.env_settings import BaseSettings +from pydantic.v1.fields import Field +from pydantic.v1.types import constr AwsSubnetId = constr(regex=r"^subnet-[a-z0-9]{17}$") diff --git a/database/infrastructure/config.py b/database/infrastructure/config.py index 4e3f8e56..16e31a3a 100644 --- a/database/infrastructure/config.py +++ b/database/infrastructure/config.py @@ -2,7 +2,9 @@ from typing import Optional from aws_cdk import aws_ec2, aws_rds -from pydantic import BaseSettings, Field, validator +from pydantic.v1.class_validators import validator +from pydantic.v1.env_settings import BaseSettings +from pydantic.v1.fields import Field class vedaDBSettings(BaseSettings): diff --git a/docker-compose.yml b/docker-compose.yml index 5c513a03..3ce23a25 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -97,7 +97,7 @@ services: database: container_name: veda.db platform: linux/amd64 - image: ghcr.io/stac-utils/pgstac:v0.7.10 + image: ghcr.io/stac-utils/pgstac:v0.9.1 environment: - POSTGRES_USER=username - POSTGRES_PASSWORD=password @@ -128,7 +128,7 @@ services: - PGPASSWORD=password - PGDATABASE=postgis - DYNAMODB_ENDPOINT=http://localhost:8085 - - VEDA_DB_PGSTAC_VERSION=0.7.10 + - VEDA_DB_PGSTAC_VERSION=0.9.1 ports: - "8083:8083" command: bash -c "bash /tmp/scripts/wait-for-it.sh -t 120 -h database -p 5432 && python /asset/local.py" diff --git a/ingest_api/infrastructure/config.py b/ingest_api/infrastructure/config.py index d8277013..05df9ba0 100644 --- a/ingest_api/infrastructure/config.py +++ b/ingest_api/infrastructure/config.py @@ -2,12 +2,17 @@ from typing import List, Optional import aws_cdk -from pydantic import AnyHttpUrl, BaseSettings, Field, constr +from pydantic import AnyHttpUrl, Field, constr +from pydantic_settings import BaseSettings, SettingsConfigDict -AwsArn = constr(regex=r"^arn:aws:iam::\d{12}:role/.+") +AwsArn = constr(pattern=r"^arn:aws:iam::\d{12}:role/.+") class IngestorConfig(BaseSettings): + model_config = SettingsConfigDict( + case_sensitive=False, env_file=".env", env_prefix="VEDA_", extra="ignore" + ) + # S3 bucket names where TiTiler could do HEAD and GET Requests # specific private and public buckets MUST be added if you want to use s3:// urls # You can whitelist all bucket by setting `*`. @@ -88,11 +93,6 @@ class IngestorConfig(BaseSettings): description="Raster API root path. Used to infer url of raster-api before app synthesis.", ) - class Config: - case_sensitive = False - env_file = ".env" - env_prefix = "VEDA_" - @property def stack_name(self) -> str: return f"veda-stac-ingestion-{self.stage}" diff --git a/ingest_api/infrastructure/construct.py b/ingest_api/infrastructure/construct.py index d7dcd05f..adbefc29 100644 --- a/ingest_api/infrastructure/construct.py +++ b/ingest_api/infrastructure/construct.py @@ -52,7 +52,7 @@ def __init__( "RASTER_URL": config.veda_raster_api_cf_url, "ROOT_PATH": config.ingest_root_path, "STAGE": config.stage, - "COGNITO_DOMAIN": config.cognito_domain, + "COGNITO_DOMAIN": str(config.cognito_domain), } build_api_lambda_params = { diff --git a/ingest_api/runtime/requirements.txt b/ingest_api/runtime/requirements.txt index c773522e..a72c7f18 100644 --- a/ingest_api/runtime/requirements.txt +++ b/ingest_api/runtime/requirements.txt @@ -1,17 +1,17 @@ -# Waiting for https://github.com/stac-utils/stac-pydantic/pull/116 and 117 cryptography>=42.0.5 ddbcereal==2.1.1 -fastapi>=0.109.1 +fastapi>=0.114.1 fsspec==2023.3.0 mangum>=0.15.0 orjson>=3.6.8 psycopg[binary,pool]>=3.0.15 -pydantic_ssm_settings>=0.2.0 -pydantic>=1.10.12 +# pydantic_ssm_settings>=0.2.0 +pydantic>=2.8.2 +pydantic-settings>=2.5.2 python-multipart==0.0.7 requests>=2.27.1 s3fs==2023.3.0 -stac-pydantic @ git+https://github.com/ividito/stac-pydantic.git@3f4cb381c85749bb4b15d1181179057ec0f51a94 +stac-pydantic>=3.1.2 boto3==1.24.59 aws_xray_sdk>=2.6.0,<3 aws-lambda-powertools>=1.18.0 diff --git a/ingest_api/runtime/src/collection_publisher.py b/ingest_api/runtime/src/collection_publisher.py index ecbcb3a7..29b61683 100644 --- a/ingest_api/runtime/src/collection_publisher.py +++ b/ingest_api/runtime/src/collection_publisher.py @@ -4,7 +4,7 @@ from src.schemas import DashboardCollection from src.utils import IngestionType, get_db_credentials, load_into_pgstac from src.vedaloader import VEDALoader -from stac_pydantic import Item +from stac_pydantic.item import Item class CollectionPublisher: diff --git a/ingest_api/runtime/src/config.py b/ingest_api/runtime/src/config.py index 0b9e1c0b..3cf765ad 100644 --- a/ingest_api/runtime/src/config.py +++ b/ingest_api/runtime/src/config.py @@ -2,11 +2,11 @@ from getpass import getuser from typing import Optional -from pydantic import AnyHttpUrl, BaseSettings, Field, constr -from pydantic_ssm_settings import AwsSsmSourceConfig +from pydantic import AnyHttpUrl, Field, constr +from pydantic_settings import BaseSettings, SettingsConfigDict from veda_auth import VedaAuth -AwsArn = constr(regex=r"^arn:aws:iam::\d{12}:role/.+") +AwsArn = constr(pattern=r"^arn:aws:iam::\d{12}:role/.+") class Settings(BaseSettings): @@ -47,8 +47,7 @@ def cognito_token_url(self) -> AnyHttpUrl: """Cognito user pool token and refresh url""" return f"{self.cognito_domain}/oauth2/token" - class Config(AwsSsmSourceConfig): - env_file = ".env" + model_config = SettingsConfigDict(env_file=".env") @classmethod def from_ssm(cls, stack: str): diff --git a/ingest_api/runtime/src/schema_helpers.py b/ingest_api/runtime/src/schema_helpers.py index 35544ff4..831fa709 100644 --- a/ingest_api/runtime/src/schema_helpers.py +++ b/ingest_api/runtime/src/schema_helpers.py @@ -1,7 +1,7 @@ from datetime import datetime from typing import List, Union -from pydantic import BaseModel, root_validator +from pydantic import BaseModel, model_validator from stac_pydantic.collection import Extent, TimeInterval # Smaller utility models to support the larger models in schemas.py @@ -24,7 +24,7 @@ class BboxExtent(BaseModel): xmax: float ymax: float - @root_validator + @model_validator(mode="before") def check_extent(cls, v): # mins must be below maxes if v["xmin"] >= v["xmax"] or v["ymin"] >= v["ymax"]: @@ -43,7 +43,7 @@ class TemporalExtent(BaseModel): startdate: datetime enddate: datetime - @root_validator + @model_validator(mode="before") def check_dates(cls, v): if v["startdate"] >= v["enddate"]: raise ValueError("Invalid extent - startdate must be before enddate") diff --git a/ingest_api/runtime/src/schemas.py b/ingest_api/runtime/src/schemas.py index 96d30bd2..cd36866b 100644 --- a/ingest_api/runtime/src/schemas.py +++ b/ingest_api/runtime/src/schemas.py @@ -7,17 +7,13 @@ from urllib.parse import urlparse import src.validators as validators -from pydantic import ( - BaseModel, - ConfigDict, - Field, - Json, - PositiveInt, - error_wrappers, - validator, -) +from pydantic import BaseModel, Json, PositiveInt +from pydantic import field_validator, ConfigDict, error_wrappers, Field + from src.schema_helpers import SpatioTemporalExtent -from stac_pydantic import Collection, Item, shared +from stac_pydantic import shared +from stac_pydantic.collection import Collection +from stac_pydantic.item import Item from stac_pydantic.links import Link from fastapi.encoders import jsonable_encoder @@ -32,7 +28,7 @@ class LinkWithExtraFields(Link): class AccessibleAsset(shared.Asset): - @validator("href") + @field_validator("href") def is_accessible(cls, href): url = urlparse(href) @@ -51,7 +47,7 @@ def is_accessible(cls, href): class AccessibleItem(Item): assets: Dict[str, AccessibleAsset] - @validator("collection") + @field_validator("collection") def exists(cls, collection): validators.collection_exists(collection_id=collection) return collection @@ -65,8 +61,7 @@ class DashboardCollection(Collection): assets: Optional[Dict] extent: SpatioTemporalExtent - class Config: - allow_population_by_field_name = True + model_config: ConfigDict = ConfigDict(populate_by_name = True) class Status(str, enum.Enum): @@ -103,17 +98,14 @@ class AuthResponse(BaseModel): class Ingestion(BaseModel): id: str = Field(..., description="ID of the STAC item") status: Status = Field(..., description="Status of the ingestion") - message: Optional[str] = Field( - None, description="Message returned from the step function." - ) + message: Optional[str] = None # Message returned from the step function. created_by: str = Field(..., description="User who created the ingestion") created_at: datetime = Field(None, description="Timestamp of ingestion creation") updated_at: datetime = Field(None, description="Timestamp of ingestion update") item: Union[Item, Json[Item]] = Field(..., description="STAC item to ingest") - @validator("created_at", pre=True, always=True, allow_reuse=True) - @validator("updated_at", pre=True, always=True, allow_reuse=True) + @field_validator("created_at","updated_at", mode="before") def set_ts_now(cls, v): return v or datetime.now() @@ -133,7 +125,7 @@ def save(self, db: "services.Database"): def dynamodb_dict(self, by_alias=True): """DynamoDB-friendly serialization""" # convert to dictionary - output = self.dict(exclude={"item"}) + output = self.model_dump(exclude={"item"}, by_alias=by_alias) # add STAC item as string output["item"] = self.item.json() @@ -141,11 +133,13 @@ def dynamodb_dict(self, by_alias=True): # make JSON-friendly (will be able to do with Pydantic V2, https://github.com/pydantic/pydantic/issues/1409#issuecomment-1423995424) return jsonable_encoder(output) + model_config = ConfigDict(arbitrary_types_allowed=True) + class ListIngestionRequest(BaseModel): status: Status = Field(Status.queued, description="Status of the ingestion") - limit: PositiveInt = Field(None, description="Limit number of results") - next: Optional[str] = Field(None, description="Next token (json) to load") + limit: Optional[PositiveInt] = None + next: Optional[str] = None # Next token (json) to load def __post_init_post_parse__(self) -> None: # https://github.com/tiangolo/fastapi/issues/1474#issuecomment-1049987786 @@ -165,15 +159,16 @@ def __post_init_post_parse__(self) -> None: ) ] ) + model_config = ConfigDict(arbitrary_types_allowed=True) class ListIngestionResponse(BaseModel): items: List[Ingestion] = Field( ..., description="List of STAC items from ingestion." ) - next: Optional[str] = Field(None, description="Next token (json) to load") + next: Optional[str] = None # Next token (json) to load - @validator("next", pre=True) + @field_validator("next", mode="before") def b64_encode_next(cls, next): """ Base64 encode next parameter for easier transportability diff --git a/ingest_api/runtime/src/services.py b/ingest_api/runtime/src/services.py index f4cc2b05..5f8ad315 100644 --- a/ingest_api/runtime/src/services.py +++ b/ingest_api/runtime/src/services.py @@ -4,7 +4,8 @@ import src.schemas as schemas from boto3.dynamodb import conditions from boto3.dynamodb.types import DYNAMODB_CONTEXT -from pydantic import parse_obj_as +from pydantic import TypeAdapter + if TYPE_CHECKING: from mypy_boto3_dynamodb.service_resource import Table @@ -25,24 +26,25 @@ def fetch_one(self, username: str, ingestion_id: str): Key={"created_by": username, "id": ingestion_id}, ) try: - return schemas.Ingestion.parse_obj(response["Item"]) + return schemas.Ingestion.model_validate(response["Item"]) except KeyError: raise NotInDb("Record not found") def fetch_many( self, status: str, next: Optional[dict] = None, limit: Optional[int] = None ) -> schemas.ListIngestionResponse: + adapter = TypeAdapter(List[schemas.Ingestion]) response = self.table.query( IndexName="status", KeyConditionExpression=conditions.Key("status").eq(status), **{"Limit": limit} if limit else {}, **{"ExclusiveStartKey": next} if next else {}, ) + items = adapter.validate_python(response["Items"]) return { - "items": parse_obj_as(List[schemas.Ingestion], response["Items"]), + "items": items, "next": response.get("LastEvaluatedKey"), } - class NotInDb(Exception): ... diff --git a/ingest_api/runtime/src/validators.py b/ingest_api/runtime/src/validators.py index 706f6f55..59014e28 100644 --- a/ingest_api/runtime/src/validators.py +++ b/ingest_api/runtime/src/validators.py @@ -90,7 +90,7 @@ def collection_exists(collection_id: str) -> bool: from src.main import settings url = "/".join( - f'{url.strip("/")}' for url in [settings.stac_url, "collections", collection_id] + f'{str(url).strip("/")}' for url in [settings.stac_url, "collections", collection_id] ) if (response := requests.get(url)).ok: diff --git a/network/infrastructure/config.py b/network/infrastructure/config.py index ebde45f3..0913c29a 100644 --- a/network/infrastructure/config.py +++ b/network/infrastructure/config.py @@ -1,7 +1,7 @@ """Configuration options for the VPC.""" from typing import Dict -from pydantic import BaseSettings +from pydantic.v1.env_settings import BaseSettings # https://medium.com/aws-activate-startup-blog/practical-vpc-design-8412e1a18dcc#.bmeh8m3si diff --git a/raster_api/infrastructure/config.py b/raster_api/infrastructure/config.py index f3474226..ddffc7a2 100644 --- a/raster_api/infrastructure/config.py +++ b/raster_api/infrastructure/config.py @@ -4,7 +4,8 @@ from typing import Dict, List, Optional -from pydantic import BaseSettings, Field +from pydantic.v1.env_settings import BaseSettings +from pydantic.v1.fields import Field class vedaRasterSettings(BaseSettings): diff --git a/s3_website/infrastructure/config.py b/s3_website/infrastructure/config.py index daa62332..9415cb61 100644 --- a/s3_website/infrastructure/config.py +++ b/s3_website/infrastructure/config.py @@ -3,7 +3,8 @@ """ from typing import Optional -from pydantic import BaseSettings, Field +from pydantic.v1.env_settings import BaseSettings +from pydantic.v1.fields import Field class vedaS3WebsiteSettings(BaseSettings): diff --git a/setup.py b/setup.py index 906c5bc7..30ed7d43 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,9 @@ "constructs>=10.0.0,<11.0.0", "aws-cdk.aws_apigatewayv2_alpha~=2.47.0.a0", "aws_cdk.aws_apigatewayv2_integrations_alpha~=2.47.0.a0", - "pydantic~=1.0", + "pydantic~=2.8.2", + "pydantic-settings~=2.5.2", + "pypgstac==0.9.1", "eoapi-cdk==5.4.0", ], "test": [ @@ -20,7 +22,7 @@ "pytest-asyncio", "pytest-cov", "httpx==0.23.3", - "pypgstac==0.7.4", + "pypgstac==0.9.1", "psycopg[binary, pool]", "fastapi", "openapi-schema-validator", diff --git a/stac_api/infrastructure/config.py b/stac_api/infrastructure/config.py index 2196a430..66f59ed0 100644 --- a/stac_api/infrastructure/config.py +++ b/stac_api/infrastructure/config.py @@ -2,7 +2,10 @@ from typing import Dict, Optional -from pydantic import AnyHttpUrl, BaseSettings, Field, root_validator +from pydantic.v1 import AnyHttpUrl +from pydantic.v1.class_validators import root_validator +from pydantic.v1.env_settings import BaseSettings +from pydantic.v1.fields import Field class vedaSTACSettings(BaseSettings): diff --git a/stac_api/runtime/src/config.py b/stac_api/runtime/src/config.py index 78eccf94..1387ce97 100644 --- a/stac_api/runtime/src/config.py +++ b/stac_api/runtime/src/config.py @@ -7,7 +7,10 @@ from typing import Optional import boto3 -from pydantic import AnyHttpUrl, BaseSettings, Field, root_validator, validator +from pydantic import AnyHttpUrl +from pydantic.v1.class_validators import root_validator, validator +from pydantic.v1.env_settings import BaseSettings +from pydantic.v1.fields import Field from fastapi.responses import ORJSONResponse from stac_fastapi.api.models import create_get_request_model, create_post_request_model diff --git a/stac_api/runtime/src/core.py b/stac_api/runtime/src/core.py index 54cd3a0f..7fc1299c 100644 --- a/stac_api/runtime/src/core.py +++ b/stac_api/runtime/src/core.py @@ -5,7 +5,7 @@ import orjson from asyncpg.exceptions import InvalidDatetimeFormatError from buildpg import render -from pydantic import ValidationError +from pydantic.v1.error_wrappers import ValidationError from pygeofilter.backends.cql2_json import to_cql2 from pygeofilter.parsers.cql2_text import parse as parse_cql2_text diff --git a/stac_api/runtime/src/render.py b/stac_api/runtime/src/render.py index 23221f01..91d51a86 100644 --- a/stac_api/runtime/src/render.py +++ b/stac_api/runtime/src/render.py @@ -4,7 +4,7 @@ from urllib.parse import urlencode import orjson -from pydantic import BaseModel +from pydantic.v1 import BaseModel def orjson_dumps(v: Dict[str, Any], *args: Any, default: Any) -> str: diff --git a/stac_api/runtime/src/search.py b/stac_api/runtime/src/search.py index cd0a5352..8c73f2a6 100644 --- a/stac_api/runtime/src/search.py +++ b/stac_api/runtime/src/search.py @@ -13,7 +13,8 @@ Polygon, _GeometryBase, ) -from pydantic import BaseModel, validator +from pydantic.v1 import BaseModel +from pydantic.v1.class_validators import validator from stac_pydantic.shared import BBox from stac_fastapi.types.rfc3339 import rfc3339_str_to_datetime, str_to_interval diff --git a/stac_api/runtime/src/validation.py b/stac_api/runtime/src/validation.py index 9f429e3c..d22845af 100644 --- a/stac_api/runtime/src/validation.py +++ b/stac_api/runtime/src/validation.py @@ -4,7 +4,8 @@ import re from typing import Dict -from pydantic import BaseModel, Field +from pydantic.v1 import BaseModel +from pydantic.v1.fields import Field from pystac import STACObjectType from pystac.errors import STACValidationError from pystac.validation import validate_dict diff --git a/standalone_base_infrastructure/standalone_config.py b/standalone_base_infrastructure/standalone_config.py index ab60da1c..5b4b66c1 100644 --- a/standalone_base_infrastructure/standalone_config.py +++ b/standalone_base_infrastructure/standalone_config.py @@ -1,7 +1,8 @@ """Configuration options for optional stand-alone VPC Stack""" from typing import Optional -from pydantic import BaseSettings, Field +from pydantic.v1.env_settings import BaseSettings +from pydantic.v1.fields import Field class baseSettings(BaseSettings):