diff --git a/.github/actions/cdk-deploy/action.yml b/.github/actions/cdk-deploy/action.yml index 14fab9c3..893f4949 100644 --- a/.github/actions/cdk-deploy/action.yml +++ b/.github/actions/cdk-deploy/action.yml @@ -1,4 +1,4 @@ -name: Pre-Production - Test, and Deploy Workflow from Veda-Deploy +name: Test and Deploy Workflow inputs: env_aws_secret_name: diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml new file mode 100644 index 00000000..aee2878b --- /dev/null +++ b/.github/workflows/cicd.yml @@ -0,0 +1,55 @@ +name: CDK Deploy Dev Workflow 🚀 + +permissions: + id-token: write + contents: read + +on: + push: + branches: + - develop + +jobs: + define-environment: + name: Set ✨ environment ✨ + runs-on: ubuntu-latest + steps: + - name: Set the environment based on the branch + id: define_environment + run: | + if [ "${{ github.ref }}" = "refs/heads/develop" ]; then + echo "env_name=dev" >> $GITHUB_OUTPUT + fi + - name: Print the environment + run: echo "The environment is ${{ steps.define_environment.outputs.env_name }}" + + outputs: + env_name: ${{ steps.define_environment.outputs.env_name }} + + deploy: + + name: Deploy to ${{ needs.define-environment.outputs.env_name }} 🚀 + runs-on: ubuntu-latest + if: ${{ needs.define-environment.outputs.env_name }} + needs: [define-environment] + environment: ${{ needs.define-environment.outputs.env_name }} + concurrency: ${{ needs.define-environment.outputs.env_name }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + lfs: "true" + submodules: "false" + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} + role-session-name: "veda-backend-github-${{ needs.define-environment.outputs.env_name }}-deployment" + aws-region: us-west-2 + + - name: Run veda-backend deployment + uses: "./.github/actions/cdk-deploy" + with: + env_aws_secret_name: ${{ vars.ENV_AWS_SECRET_NAME }} \ No newline at end of file diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml deleted file mode 100644 index f3c12243..00000000 --- a/.github/workflows/develop.yml +++ /dev/null @@ -1,143 +0,0 @@ -name: Develop - Lint, Test, and Deploy Workflow - -on: - push: - branches: - - develop - -jobs: - lint-dev: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.9' - - - uses: actions/cache@v4 - with: - path: ${{ env.pythonLocation }} - key: ${{ env.pythonLocation }}-${{ hashFiles('setup.py') }} - - - name: Install python dependencies - run: | - python -m pip install --upgrade pip - python -m pip install -e .[dev,deploy,test] - - - name: Run pre-commit - run: pre-commit run --all-files - - test-dev: - needs: [lint-dev] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.9' - - - uses: actions/cache@v4 - with: - path: ${{ env.pythonLocation }} - key: ${{ env.pythonLocation }}-${{ hashFiles('setup.py') }} - - - name: Install python dependencies - run: | - python -m pip install --upgrade pip - python -m pip install -e .[dev,deploy,test] - - - name: Launch services - run: AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY=${{secrets.AWS_SECRET_ACCESS_KEY}} docker compose up --build -d - - - name: Ingest Stac Items/Collection - run: | - ./scripts/load-data-container.sh - - - name: Sleep for 10 seconds - run: sleep 10s - shell: bash - - - name: Integrations tests - run: python -m pytest .github/workflows/tests/ -vv -s - - - name: Install reqs for ingest api - run: python -m pip install -r ingest_api/runtime/requirements_dev.txt - - - name: Install veda auth for ingest api - run: python -m pip install common/auth - - - name: Ingest unit tests - run: NO_PYDANTIC_SSM_SETTINGS=1 python -m pytest ingest_api/runtime/tests/ -vv -s - - - name: Stop services - run: docker compose stop - - pre-release: - needs: [test-dev] - runs-on: ubuntu-latest - concurrency: release - permissions: - id-token: write - contents: write - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Python Semantic Release - uses: python-semantic-release/python-semantic-release@master - with: - changelog: "false" - github_token: ${{ secrets.GITHUB_TOKEN }} - - deploy-dev: - needs: [pre-release] - - runs-on: ubuntu-latest - steps: - - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.9' - - - name: Setup Node - uses: actions/setup-node@v4 - with: - node-version: 20 - - - name: Configure awscli - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-west-2 - - - uses: actions/cache@v4 - with: - path: ~/.npm - key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }} - - - name: Install CDK - run: npm install -g aws-cdk@2 - - - uses: actions/cache@v4 - with: - path: ${{ env.pythonLocation }} - key: ${{ env.pythonLocation }}-${{ hashFiles('setup.py') }} - - - name: Install python dependencies - run: | - python -m pip install --upgrade pip - python -m pip install -e .[dev,deploy,test] - - - name: Get dev environment configuration for develop branch - run: ./scripts/get-env.sh "veda-backend-uah-dev-env" - - - name: Deploy - run: | - echo $STAGE - cdk deploy --require-approval never --outputs-file ${HOME}/cdk-outputs.json \ No newline at end of file diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index 7541f92b..00000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,144 +0,0 @@ -name: Main - Lint, Test, and Deploy Workflow - -on: - push: - branches: - - main - -jobs: - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.9' - - - uses: actions/cache@v4 - with: - path: ${{ env.pythonLocation }} - key: ${{ env.pythonLocation }}-${{ hashFiles('setup.py') }} - - - name: Install python dependencies - run: | - python -m pip install --upgrade pip - python -m pip install -e .[dev,deploy,test] - - - name: Run pre-commit - run: pre-commit run --all-files - - test: - needs: [lint] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.9' - - - uses: actions/cache@v4 - with: - path: ${{ env.pythonLocation }} - key: ${{ env.pythonLocation }}-${{ hashFiles('setup.py') }} - - - name: Install python dependencies - run: | - python -m pip install --upgrade pip - python -m pip install -e .[dev,deploy,test] - - - name: Launch services - run: AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY=${{secrets.AWS_SECRET_ACCESS_KEY}} docker compose up --build -d - - - name: Ingest Stac Items/Collection - run: | - ./scripts/load-data-container.sh - - - name: Sleep for 10 seconds - run: sleep 10s - shell: bash - - - name: Integrations tests - run: python -m pytest .github/workflows/tests/ -vv -s - - - name: Install reqs for ingest api - run: python -m pip install -r ingest_api/runtime/requirements_dev.txt - - - name: Install veda auth for ingest api - run: python -m pip install common/auth - - - name: Ingest unit tests - run: NO_PYDANTIC_SSM_SETTINGS=1 python -m pytest ingest_api/runtime/tests/ -vv -s - - - name: Stop services - run: docker compose stop - - release: - needs: [test] - runs-on: ubuntu-latest - concurrency: release - permissions: - id-token: write - contents: write - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Python Semantic Release - uses: python-semantic-release/python-semantic-release@master - with: - changelog: "false" - github_token: ${{ secrets.GITHUB_TOKEN }} - - deploy: - needs: [release] - - runs-on: ubuntu-latest - steps: - - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.9' - - - name: Setup Node - uses: actions/setup-node@v4 - with: - node-version: 20 - - - name: Configure awscli - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-west-2 - - - uses: actions/cache@v4 - with: - path: ~/.npm - key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }} - - - name: Install CDK - run: npm install -g aws-cdk@2 - - - uses: actions/cache@v4 - with: - path: ${{ env.pythonLocation }} - key: ${{ env.pythonLocation }}-${{ hashFiles('setup.py') }} - - - name: Install python dependencies - run: | - python -m pip install --upgrade pip - python -m pip install -e .[dev,deploy,test] - - - name: Get dev environment configuration for staging branch - run: ./scripts/get-env.sh "veda-backend-uah-staging-env" - - - name: Deploy - run: | - echo $STAGE - cdk deploy --require-approval never --outputs-file ${HOME}/cdk-outputs.json - \ No newline at end of file diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 03b7fbad..7222a93b 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -77,14 +77,17 @@ jobs: - name: Install reqs for ingest api run: python -m pip install -r ingest_api/runtime/requirements_dev.txt + - name: Install reqs for stac api + run: python -m pip install stac_api/runtime/ + - name: Install veda auth for ingest api run: python -m pip install common/auth - name: Ingest unit tests run: NO_PYDANTIC_SSM_SETTINGS=1 python -m pytest ingest_api/runtime/tests/ -vv -s - # - name: Stac-api transactions unit tests - # run: python -m pytest stac_api/runtime/tests/ -vv -s + - name: Stac-api transactions unit tests + run: python -m pytest stac_api/runtime/tests/ --asyncio-mode=auto -vv -s - name: Stop services run: docker compose stop @@ -133,9 +136,7 @@ jobs: - name: Get environment configuration for target branch run: | - if [ "${{ github.base_ref }}" == "main" ]; then - ./scripts/get-env.sh "veda-backend-uah-staging-env" - elif [ "${{ github.base_ref }}" == "develop" ]; then + if [ "${{ github.base_ref }}" == "develop" ]; then ./scripts/get-env.sh "veda-backend-uah-dev-env" else echo "No environment associated with ${GITHUB_REF##*/} branch. Test changes against dev stack" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..d8c01813 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,29 @@ +name: Release Workflow + +permissions: + id-token: write + contents: read + +on: + push: + branches: + - develop + - main + +jobs: + release: + runs-on: ubuntu-latest + concurrency: release + permissions: + id-token: write + contents: write + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Python Semantic Release + uses: python-semantic-release/python-semantic-release@master + with: + changelog: "false" + github_token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/README.md b/README.md index 3407ff31..82cf4bae 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,9 @@ # veda-backend + This project deploys a complete backend for a [SpatioTemporal Asset Catalog](https://stacspec.org/) including a postgres database, a metadata API, and raster tiling API. Veda-backend is a non-forked version of the [eoAPI](https://github.com/developmentseed/eoAPI) demo project. Veda-backend is decoupled from the demo project to selectively incorporate new stable functionality from the fast moving development in eoAPI while providing a continuous baseline for veda-backend users and to support project specific business and deployment logic. The primary tools employed in the [eoAPI demo](https://github.com/developmentseed/eoAPI) and this project are: + - [stac-spec](https://github.com/radiantearth/stac-spec) - [stac-api-spec](https://github.com/radiantearth/stac-api-spec) - [stac-fastapi](https://github.com/stac-utils/stac-fastapi) @@ -11,6 +13,7 @@ The primary tools employed in the [eoAPI demo](https://github.com/developmentsee - [eoapi-cdk](https://github.com/developmentseed/eoapi-cdk/tree/main#eoapi-cdk-constructs) + [radiantearth/stac-browser](https://github.com/radiantearth/stac-browser) ## VEDA backend context + ![architecture diagram](.readme/veda-overview-bw.svg) _Edit this diagram in VS Code using the [Draw.io Integration Extension](https://marketplace.visualstudio.com/items?itemName=hediet.vscode-drawio) and export a new SVG_ @@ -33,12 +36,14 @@ An [.example.env](.example.env) template is supplied for local deployments. If u ### Fetch environment variables using AWS CLI -To retrieve the variables for a stage that has been previously deployed, the secrets manager can be used to quickly populate an .env file with [scripts/sync-env-local.sh](scripts/sync-env-local.sh). +To retrieve the variables for a stage that has been previously deployed, the secrets manager can be used to quickly populate an .env file with [scripts/sync-env-local.sh](scripts/sync-env-local.sh). -``` +```bash ./scripts/sync-env-local.sh ``` + ### Basic environment variables + | Name | Explanation | | --- | --- | | `APP_NAME` | Optional app name used to name stack and resources, defaults to `veda-backend` | @@ -48,6 +53,7 @@ To retrieve the variables for a stage that has been previously deployed, the sec | `VEDA_DB_SNAPSHOT_ID` | **Once used always REQUIRED** Optional RDS snapshot identifier to initialize RDS from a snapshot | ### Advanced configuration + The constructs and applications in this project are configured using pydantic. The settings are defined in config.py files stored alongside the associated construct or application--for example the settings for the RDS PostgreSQL construct are defined in database/infrastructure/config.py. For custom configuration, use environment variables to override the pydantic defaults. | Construct | Env Prefix | Configuration | @@ -64,24 +70,28 @@ The constructs and applications in this project are configured using pydantic. T ### Deploying to the cloud #### Install deployment pre-requisites + - [Node](https://nodejs.org/) - [NVM](https://github.com/nvm-sh/nvm#node-version-manager---) - [jq](https://jqlang.github.io/jq/) (used for exporting environment variable secrets to `.env` in [scripts/sync-env-local.sh](/scripts/sync-env-local.sh)) These can be installed with [homebrew](https://brew.sh/) on MacOS -``` + +```bash brew install node brew install nvm brew install jq ``` #### Virtual environment example -``` + +```bash python3 -m venv .venv source .venv/bin/activate ``` #### Install requirements + ```bash nvm use --lts npm install --location=global aws-cdk @@ -99,7 +109,7 @@ cdk diff # Execute deployment and standby--security changes will require approval for deployment cdk deploy ``` - + ## Deleting the CloudFormation stack If this is a development stack that is safe to delete, you can delete the stack in CloudFormation console or via `cdk destroy`, however, the additional manual steps were required to completely delete the stack resources: @@ -112,24 +122,29 @@ If this is a development stack that is safe to delete, you can delete the stack ## Custom deployments The default settings for this project generate a complete AWS environment including a VPC and gateways for the stack. See this guidance for adjusting the veda-backend stack for existing managed and/or shared AWS environments. + - [Deploy to an existing managed AWS environment](docs/deploying_to_existing_environments.md) - [Creating a shared base VPC and AWS environment](docs/deploying_to_existing_environments.md#optional-deploy-standalone-base-infrastructure) ## Local Docker deployment Start up a local stack -``` + +```bash docker compose up ``` + Clean up after running locally -``` + +```bash docker compose down ``` ## Running tests locally To run tests implicated in CI, a script is included that requires as little setup as possible -``` + +```bash ./scripts/run-local-tests.sh ``` @@ -137,7 +152,7 @@ In case of failure, all container logs will be written out to `container_logs.lo # Operations -## Adding new data to veda-backend +## Adding new data to veda-backend > **Warning** PgSTAC records should be loaded in the database using [pypgstac](https://github.com/stac-utils/pgstac#pypgstac) for proper indexing and partitioning. @@ -145,11 +160,13 @@ The VEDA ecosystem includes tools specifially created for loading PgSTAC records ## Support scripts Support scripts are provided for manual system operations. + - [Rotate pgstac password](support_scripts/README.md#rotate-pgstac-password) # VEDA ecosystem ## Projects + | Name | Explanation | | --- | --- | | **veda-backend** | Central index (database) and APIs for recording, discovering, viewing, and using VEDA assets | @@ -159,6 +176,7 @@ Support scripts are provided for manual system operations. | [**veda-data**](https://github.com/NASA-IMPACT/veda-data) | Collection and asset discovery configuration | | [**veda-data-airflow**](https://github.com/NASA-IMPACT/veda-data-airflow) | Cloud optimize data assets and submit records for publication to veda-stac-ingestor | | [**veda-docs**](https://github.com/NASA-IMPACT/veda-docs) | Documentation repository for end users of VEDA ecosystem data and tools | +| [**veda-routes**](https://github.com/NASA-IMPACT/veda-routes)| Configuration for VEDA's Content Delivery Network | ## VEDA usage examples @@ -169,7 +187,9 @@ Support scripts are provided for manual system operations. # STAC community resources ## STAC browser + Radiant Earth's [stac-browser](https://github.com/radiantearth/stac-browser) is a browser for STAC catalogs. The demo version of this browser [radiantearth.github.io/stac-browser](https://radiantearth.github.io/stac-browser/#/) can be used to browse the contents of the veda-backend STAC catalog, paste the veda-backend stac-api URL deployed by this project in the demo and click load. Read more about the recent developments and usage of stac-browser [here](https://medium.com/radiant-earth-insights/the-exciting-future-of-the-stac-browser-2351143aa24b). # License + This project is licensed under **Apache 2**, see the [LICENSE](LICENSE) file for more details. diff --git a/app.py b/app.py index c2bf290a..8c28038b 100644 --- a/app.py +++ b/app.py @@ -8,14 +8,12 @@ from config import veda_app_settings from database.infrastructure.construct import RdsConstruct -from domain.infrastructure.construct import DomainConstruct from ingest_api.infrastructure.config import IngestorConfig as ingest_config from ingest_api.infrastructure.construct import ApiConstruct as ingest_api_construct from ingest_api.infrastructure.construct import IngestorConstruct as ingestor_construct from network.infrastructure.construct import VpcConstruct from permissions_boundary.infrastructure.construct import PermissionsBoundaryAspect from raster_api.infrastructure.construct import RasterApiLambdaConstruct -from routes.infrastructure.construct import CloudfrontDistributionConstruct from s3_website.infrastructure.construct import VedaWebsite from stac_api.infrastructure.construct import StacApiLambdaConstruct @@ -71,15 +69,12 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: stage=veda_app_settings.stage_name(), ) -domain = DomainConstruct(veda_stack, "domain", stage=veda_app_settings.stage_name()) - raster_api = RasterApiLambdaConstruct( veda_stack, "raster-api", stage=veda_app_settings.stage_name(), vpc=vpc.vpc, database=database, - domain=domain, ) stac_api = StacApiLambdaConstruct( @@ -89,23 +84,12 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: vpc=vpc.vpc, database=database, raster_api=raster_api, - domain=domain, ) website = VedaWebsite( veda_stack, "stac-browser-bucket", stage=veda_app_settings.stage_name() ) -veda_routes = CloudfrontDistributionConstruct( - veda_stack, - "routes", - stage=veda_app_settings.stage_name(), - raster_api_id=raster_api.raster_api.api_id, - stac_api_id=stac_api.stac_api.api_id, - origin_bucket=website.bucket, - region=veda_app_settings.cdk_default_region, -) - # Only create a stac browser if we can infer the catalog url from configuration before synthesis (API Gateway URL not yet available) stac_catalog_url = veda_app_settings.get_stac_catalog_url() if stac_catalog_url: @@ -128,7 +112,6 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: raster_api_url=raster_api.raster_api.url, ) - ingest_api = ingest_api_construct( veda_stack, "ingest-api", @@ -136,7 +119,6 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: db_secret=database.pgstac.secret, db_vpc=vpc.vpc, db_vpc_subnets=database.vpc_subnets, - domain=domain, ) ingestor = ingestor_construct( @@ -149,51 +131,6 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: db_vpc_subnets=database.vpc_subnets, ) -veda_routes.add_ingest_behavior( - ingest_api=ingest_api.api, stage=veda_app_settings.stage_name() -) - -# Must be done after all CF behaviors exist -veda_routes.create_route_records(stage=veda_app_settings.stage_name()) - - -# TODO this conditional supports deploying a second set of APIs to a separate custom domain and should be removed if no longer necessary -if veda_app_settings.alt_domain(): - alt_domain = DomainConstruct( - veda_stack, - "alt-domain", - stage=veda_app_settings.stage_name(), - alt_domain=True, - ) - - alt_raster_api = RasterApiLambdaConstruct( - veda_stack, - "alt-raster-api", - stage=veda_app_settings.stage_name(), - vpc=vpc.vpc, - database=database, - domain_name=alt_domain.raster_domain_name, - ) - - alt_stac_api = StacApiLambdaConstruct( - veda_stack, - "alt-stac-api", - stage=veda_app_settings.stage_name(), - vpc=vpc.vpc, - database=database, - raster_api=raster_api, - domain_name=alt_domain.stac_domain_name, - ) - - alt_ingest_api = ingest_api_construct( - veda_stack, - "alt-ingest-api", - config=ingestor_config, - db_secret=database.pgstac.secret, - db_vpc=vpc.vpc, - domain_name=alt_domain.ingest_domain_name, - ) - git_sha = subprocess.check_output(["git", "rev-parse", "HEAD"]).decode().strip() try: git_tag = subprocess.check_output(["git", "describe", "--tags"]).decode().strip() diff --git a/config.py b/config.py index 091d6a38..c792b1b7 100644 --- a/config.py +++ b/config.py @@ -90,7 +90,12 @@ class vedaAppSettings(BaseSettings): veda_stac_root_path: str = Field( "", - description="Optional path prefix to add to all api endpoints. Used to infer url of stac-api before app synthesis.", + description="STAC API root path. Used to infer url of stac-api before app synthesis.", + ) + + veda_raster_root_path: str = Field( + "", + description="Raster API root path", ) veda_domain_create_custom_subdomains: bool = Field( @@ -115,15 +120,6 @@ def cdk_env(self) -> dict: else: return {} - def alt_domain(self) -> bool: - """True if alternative domain and host parameters provided""" - return all( - [ - self.veda_domain_alt_hosted_zone_id, - self.veda_domain_alt_hosted_zone_name, - ] - ) - def stage_name(self) -> str: """Force lowercase stage name""" return self.stage.lower() diff --git a/domain/infrastructure/config.py b/domain/infrastructure/config.py deleted file mode 100644 index b74c4e88..00000000 --- a/domain/infrastructure/config.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Configuration options for a custom API domain.""" - -from typing import Optional - -from pydantic import BaseSettings, Field - - -class vedaDomainSettings(BaseSettings): - """Application settings""" - - hosted_zone_id: Optional[str] = Field( - None, description="Route53 hosted zone identifier if using a custom domain name" - ) - hosted_zone_name: Optional[str] = Field( - None, description="Custom domain name, i.e. veda-backend.xyz" - ) - create_custom_subdomains: bool = Field( - False, - description=( - "When true and hosted zone config is provided, create a unique subdomain for stac and raster apis. " - "For example -stac. and -raster." - ), - ) - api_prefix: Optional[str] = Field( - None, - description=( - "Domain prefix override supports using a custom prefix instead of the " - "STAGE variabe (an alternate version of the stack can be deployed with a " - "unique STAGE=altprod and after testing prod API traffic can be cut over " - "to the alternate version of the stack by setting the prefix to prod)" - ), - ) - - # Temporary support for deploying APIs to a second custom domain - alt_hosted_zone_id: Optional[str] = Field( - None, description="Second Route53 zone identifier if using a custom domain name" - ) - alt_hosted_zone_name: Optional[str] = Field( - None, description="Second custom domain name, i.e. alt-veda-backend.xyz" - ) - - class Config: - """model config""" - - env_file = ".env" - env_prefix = "VEDA_DOMAIN_" - - -veda_domain_settings = vedaDomainSettings() diff --git a/domain/infrastructure/construct.py b/domain/infrastructure/construct.py deleted file mode 100644 index 61b133fa..00000000 --- a/domain/infrastructure/construct.py +++ /dev/null @@ -1,148 +0,0 @@ -"""CDK Construct for a custom API domain.""" -from typing import Optional - -from aws_cdk import ( - CfnOutput, - aws_apigatewayv2_alpha, - aws_certificatemanager, - aws_route53, - aws_route53_targets, -) -from constructs import Construct - -from .config import veda_domain_settings - - -class DomainConstruct(Construct): - """CDK Construct for a custom API domain.""" - - def __init__( - self, - scope: Construct, - construct_id: str, - stage: str, - alt_domain: Optional[bool] = False, - **kwargs, - ) -> None: - """.""" - super().__init__(scope, construct_id, **kwargs) - - self.stac_domain_name = None - self.raster_domain_name = None - self.ingest_domain_name = None - - if veda_domain_settings.create_custom_subdomains: - # If alternative custom domain provided, use it instead of the default - if alt_domain is True: - hosted_zone_name = veda_domain_settings.alt_hosted_zone_name - hosted_zone_id = veda_domain_settings.alt_hosted_zone_id - else: - hosted_zone_name = veda_domain_settings.hosted_zone_name - hosted_zone_id = veda_domain_settings.hosted_zone_id - - hosted_zone = aws_route53.HostedZone.from_hosted_zone_attributes( - self, - "hosted-zone", - hosted_zone_id=hosted_zone_id, - zone_name=hosted_zone_name, - ) - certificate = aws_certificatemanager.Certificate( - self, - "certificate", - domain_name=f"*.{hosted_zone_name}", - validation=aws_certificatemanager.CertificateValidation.from_dns( - hosted_zone=hosted_zone - ), - ) - - # Use custom api prefix if provided or deployment stage if not - if veda_domain_settings.api_prefix: - raster_url_prefix = f"{veda_domain_settings.api_prefix.lower()}-raster" - stac_url_prefix = f"{veda_domain_settings.api_prefix.lower()}-stac" - ingest_url_prefix = f"{veda_domain_settings.api_prefix.lower()}-ingest" - else: - raster_url_prefix = f"{stage.lower()}-raster" - stac_url_prefix = f"{stage.lower()}-stac" - ingest_url_prefix = f"{stage.lower()}-ingest" - raster_domain_name = f"{raster_url_prefix}.{hosted_zone_name}" - stac_domain_name = f"{stac_url_prefix}.{hosted_zone_name}" - ingest_domain_name = f"{ingest_url_prefix}.{hosted_zone_name}" - - self.raster_domain_name = aws_apigatewayv2_alpha.DomainName( - self, - "rasterApiCustomDomain", - domain_name=raster_domain_name, - certificate=certificate, - ) - - aws_route53.ARecord( - self, - "raster-api-dns-record", - zone=hosted_zone, - target=aws_route53.RecordTarget.from_alias( - aws_route53_targets.ApiGatewayv2DomainProperties( - regional_domain_name=self.raster_domain_name.regional_domain_name, - regional_hosted_zone_id=self.raster_domain_name.regional_hosted_zone_id, - ) - ), - # Note: CDK will append the hosted zone name (eg: `veda-backend.xyz` to this record name) - record_name=raster_url_prefix, - ) - - self.stac_domain_name = aws_apigatewayv2_alpha.DomainName( - self, - "stacApiCustomDomain", - domain_name=stac_domain_name, - certificate=certificate, - ) - - aws_route53.ARecord( - self, - "stac-api-dns-record", - zone=hosted_zone, - target=aws_route53.RecordTarget.from_alias( - aws_route53_targets.ApiGatewayv2DomainProperties( - regional_domain_name=self.stac_domain_name.regional_domain_name, - regional_hosted_zone_id=self.stac_domain_name.regional_hosted_zone_id, - ) - ), - # Note: CDK will append the hosted zone name (eg: `veda-backend.xyz` to this record name) - record_name=stac_url_prefix, - ) - - self.ingest_domain_name = aws_apigatewayv2_alpha.DomainName( - self, - "ingestApiCustomDomain", - domain_name=ingest_domain_name, - certificate=certificate, - ) - - aws_route53.ARecord( - self, - "ingest-api-dns-record", - zone=hosted_zone, - target=aws_route53.RecordTarget.from_alias( - aws_route53_targets.ApiGatewayv2DomainProperties( - regional_domain_name=self.ingest_domain_name.regional_domain_name, - regional_hosted_zone_id=self.ingest_domain_name.regional_hosted_zone_id, - ) - ), - # Note: CDK will append the hosted zone name (eg: `veda-backend.xyz` to this record name) - record_name=ingest_url_prefix, - ) - - CfnOutput( - self, - "raster-api", - value=f"https://{raster_url_prefix}.{hosted_zone_name}/docs", - ) - CfnOutput( - self, - "stac-api", - value=f"https://{stac_url_prefix}.{hosted_zone_name}/", - ) - CfnOutput( - self, - "ingest-api", - value=f"https://{ingest_url_prefix}.{hosted_zone_name}/", - ) diff --git a/ingest_api/infrastructure/config.py b/ingest_api/infrastructure/config.py index 771d346d..31a894bb 100644 --- a/ingest_api/infrastructure/config.py +++ b/ingest_api/infrastructure/config.py @@ -59,19 +59,35 @@ class IngestorConfig(BaseSettings): description="Set optional global parameter to 'requester' if the requester agrees to pay S3 transfer costs", ) - stac_api_url: str = Field(description="URL of STAC API used to serve STAC Items") - - raster_api_url: str = Field( - description="URL of Raster API used to serve asset tiles" - ) - ingest_root_path: str = Field("", description="Root path for ingest API") - custom_host: Optional[str] = Field(description="Custom host name") db_pgstac_version: str = Field( ..., description="Version of PgStac database, i.e. 0.5", ) + stac_api_url: str = Field( + description="URL of STAC API Gateway endpoint used to serve STAC Items" + ) + + raster_api_url: str = Field( + description="URL of Raster API Gateway endpoing used to serve asset tiles" + ) + + custom_host: Optional[str] = Field( + None, + description="Complete url of custom host including subdomain. Used to infer url of apis before app synthesis.", + ) + + stac_root_path: Optional[str] = Field( + "", + description="STAC API root path. Used to infer url of stac-api before app synthesis.", + ) + + raster_root_path: Optional[str] = Field( + "", + description="Raster API root path. Used to infer url of raster-api before app synthesis.", + ) + class Config: case_sensitive = False env_file = ".env" @@ -87,3 +103,17 @@ def env(self) -> aws_cdk.Environment: account=self.aws_account, region=self.aws_region, ) + + @property + def veda_stac_api_cf_url(self) -> str: + """inferred cloudfront url of the stac api if app is configured with a custom host and root path""" + if self.custom_host and self.stac_root_path: + return f"https://{self.custom_host}{self.stac_root_path}" + return self.stac_api_url + + @property + def veda_raster_api_cf_url(self) -> str: + """inferred cloudfront url of the raster api if app is configured with a custom host and root path""" + if self.custom_host and self.raster_root_path: + return f"https://{self.custom_host}{self.raster_root_path}" + return self.raster_api_url diff --git a/ingest_api/infrastructure/construct.py b/ingest_api/infrastructure/construct.py index c4322647..d7dcd05f 100644 --- a/ingest_api/infrastructure/construct.py +++ b/ingest_api/infrastructure/construct.py @@ -1,5 +1,4 @@ import os -import typing from typing import Dict, Optional, Union from aws_cdk import CfnOutput, Duration, RemovalPolicy, Stack @@ -17,9 +16,6 @@ from .config import IngestorConfig -if typing.TYPE_CHECKING: - from domain.infrastructure.construct import DomainConstruct - class ApiConstruct(Construct): def __init__( @@ -30,7 +26,6 @@ def __init__( db_secret: secretsmanager.ISecret, db_vpc: ec2.IVpc, db_vpc_subnets=ec2.SubnetSelection, - domain: Optional["DomainConstruct"] = None, **kwargs, ) -> None: super().__init__(scope, construct_id, **kwargs) @@ -50,11 +45,11 @@ def __init__( "DYNAMODB_TABLE": self.table.table_name, "JWKS_URL": self.jwks_url, "NO_PYDANTIC_SSM_SETTINGS": "1", - "STAC_URL": config.stac_api_url, + "STAC_URL": config.veda_stac_api_cf_url, "USERPOOL_ID": config.userpool_id, "CLIENT_ID": config.client_id, "CLIENT_SECRET": config.client_secret, - "RASTER_URL": config.raster_api_url, + "RASTER_URL": config.veda_raster_api_cf_url, "ROOT_PATH": config.ingest_root_path, "STAGE": config.stage, "COGNITO_DOMAIN": config.cognito_domain, @@ -95,7 +90,6 @@ def __init__( self.api: aws_apigatewayv2_alpha.HttpApi = self.build_api( construct_id=construct_id, handler=self.api_lambda, - domain=domain, custom_host=config.custom_host, ) @@ -193,7 +187,6 @@ def build_api( *, construct_id: str, handler: aws_lambda.IFunction, - domain, custom_host: Optional[str], ) -> aws_apigatewayv2_alpha.HttpApi: integration_kwargs = dict(handler=handler) @@ -212,20 +205,12 @@ def build_api( ) ) - domain_mapping = None - # Legacy method to use a custom subdomain for this api (i.e. -ingest..com) - # If using a custom root path and/or a proxy server, do not use a custom subdomain - if domain and domain.ingest_domain_name: - domain_mapping = aws_apigatewayv2_alpha.DomainMappingOptions( - domain_name=domain.ingest_domain_name - ) stack_name = Stack.of(self).stack_name return aws_apigatewayv2_alpha.HttpApi( self, f"{stack_name}-{construct_id}", default_integration=ingest_api_integration, - default_domain_mapping=domain_mapping, ) def build_jwks_url(self, userpool_id: str) -> str: @@ -271,11 +256,11 @@ def __init__( lambda_env = { "DYNAMODB_TABLE": table.table_name, "NO_PYDANTIC_SSM_SETTINGS": "1", - "STAC_URL": config.stac_api_url, + "STAC_URL": config.veda_stac_api_cf_url, "USERPOOL_ID": config.userpool_id, "CLIENT_ID": config.client_id, "CLIENT_SECRET": config.client_secret, - "RASTER_URL": config.raster_api_url, + "RASTER_URL": config.veda_raster_api_cf_url, } if config.raster_data_access_role_arn: diff --git a/ingest_api/runtime/src/schema_helpers.py b/ingest_api/runtime/src/schema_helpers.py index 35544ff4..3db5bf6a 100644 --- a/ingest_api/runtime/src/schema_helpers.py +++ b/ingest_api/runtime/src/schema_helpers.py @@ -40,11 +40,11 @@ def check_extent(cls, v): class TemporalExtent(BaseModel): - startdate: datetime - enddate: datetime + startdate: Union[datetime, None] + enddate: Union[datetime, None] @root_validator def check_dates(cls, v): - if v["startdate"] >= v["enddate"]: + if (v["enddate"] is not None) and (v["startdate"] >= v["enddate"]): raise ValueError("Invalid extent - startdate must be before enddate") return v diff --git a/network/infrastructure/construct.py b/network/infrastructure/construct.py index bfd87db8..21d0ceb2 100644 --- a/network/infrastructure/construct.py +++ b/network/infrastructure/construct.py @@ -67,6 +67,9 @@ def __init__( "cloudwatch-logs": aws_ec2.InterfaceVpcEndpointAwsService.CLOUDWATCH_LOGS, "s3": aws_ec2.GatewayVpcEndpointAwsService.S3, "dynamodb": aws_ec2.GatewayVpcEndpointAwsService.DYNAMODB, + "ecr": aws_ec2.InterfaceVpcEndpointAwsService.ECR, + "ecr-docker": aws_ec2.InterfaceVpcEndpointAwsService.ECR_DOCKER, + "sts": aws_ec2.InterfaceVpcEndpointAwsService.STS, } for id, service in vpc_endpoints.items(): diff --git a/raster_api/infrastructure/construct.py b/raster_api/infrastructure/construct.py index c7362148..b75a9f21 100644 --- a/raster_api/infrastructure/construct.py +++ b/raster_api/infrastructure/construct.py @@ -1,8 +1,6 @@ """CDK Constrcut for a Lambda based TiTiler API with pgstac extension.""" import os -import typing -from typing import Optional from aws_cdk import ( CfnOutput, @@ -19,9 +17,6 @@ from .config import veda_raster_settings -if typing.TYPE_CHECKING: - from domain.infrastructure.construct import DomainConstruct - class RasterApiLambdaConstruct(Construct): """CDK Construct for a Lambda based TiTiler API with pgstac extension.""" @@ -34,8 +29,6 @@ def __init__( vpc, database, code_dir: str = "./", - # domain_name: aws_apigatewayv2_alpha.DomainName = None, - domain: Optional["DomainConstruct"] = None, **kwargs, ) -> None: """.""" @@ -102,19 +95,10 @@ def __init__( ) ) - domain_mapping = None - # Legacy method to use a custom subdomain for this api (i.e. -raster..com) - # If using a custom root path and/or a proxy server, do not use a custom subdomain - if domain and domain.raster_domain_name: - domain_mapping = aws_apigatewayv2_alpha.DomainMappingOptions( - domain_name=domain.raster_domain_name - ) - self.raster_api = aws_apigatewayv2_alpha.HttpApi( self, f"{stack_name}-{construct_id}", default_integration=raster_api_integration, - default_domain_mapping=domain_mapping, ) CfnOutput( diff --git a/raster_api/runtime/src/cmap_data/README.md b/raster_api/runtime/src/cmap_data/README.md index 821038d0..91c39ecc 100644 --- a/raster_api/runtime/src/cmap_data/README.md +++ b/raster_api/runtime/src/cmap_data/README.md @@ -28,3 +28,55 @@ cmap_vals = my_cmap(x)[:, :] cmap_uint8 = (cmap_vals * 255).astype('uint8') np.save("epa-ghgi-ch4.npy", cmap_uint8) ``` + +##### NLCD colormap + +refs: + +- https://www.mrlc.gov/data/legends/national-land-cover-database-class-legend-and-description +- https://github.com/NASA-IMPACT/veda-backend/issues/429 + +```python +import rasterio +from rio_tiler.colormap import parse_color +import numpy as np + +# The COGs in the nlcd-annual-conus collection store an internal colormap +nlcd_filename = "/vsis3/veda-data-store/nlcd-annual-conus/nlcd_2001_cog_v2.tif" + +# These categories are only used to set transparency and document categories defined in colormap +# https://www.mrlc.gov/data/legends/national-land-cover-database-class-legend-and-description +nlcd_categories = { + "11": "Open Water", + "12": "Perennial Ice/Snow", + "21": "Developed, Open Space", + "22": "Developed, Low Intensity", + "23": "Developed, Medium Intensity", + "24": "Developed, High Intensity", + "31": "Barren Land (Rock/Sand/Clay)", + "41": "Deciduous Forest", + "42": "Evergreen Forest", + "43": "Mixed Forest", + "51": "Dwarf Scrub", + "52": "Shrub/Scrub", + "71": "Grassland/Herbaceous", + "72": "Sedge/Herbaceous", + "73": "Lichens", + "74": "Moss", + "81": "Pasture/Hay", + "82": "Cultivated Crops", + "90": "Woody Wetlands", + "95": "Emergent Herbaceous Wetlands" +} + +with rasterio.open(nlcd_filename) as r: + internal_colormap = r.colormap(1) + +cmap = np.zeros((256, 4), dtype=np.uint8) +cmap[:] = np.array([0, 0, 0, 255]) +for c, v in internal_colormap.items(): + if str(c) in nlcd_categories.keys(): + cmap[c] = np.array(parse_color(v)) + +np.save("nlcd.npy", cmap) +``` diff --git a/raster_api/runtime/src/cmap_data/nlcd.npy b/raster_api/runtime/src/cmap_data/nlcd.npy new file mode 100644 index 00000000..ba5c60b8 Binary files /dev/null and b/raster_api/runtime/src/cmap_data/nlcd.npy differ diff --git a/routes/infrastructure/config.py b/routes/infrastructure/config.py deleted file mode 100755 index 0d2a878a..00000000 --- a/routes/infrastructure/config.py +++ /dev/null @@ -1,64 +0,0 @@ -"""Settings for Cloudfront distribution - any environment variables starting with -`VEDA_` will overwrite the values of variables in this file -""" -from typing import Optional - -from pydantic import BaseSettings, Field - - -class vedaRouteSettings(BaseSettings): - """Veda Route settings""" - - cloudfront: Optional[bool] = Field( - False, - description="Boolean if Cloudfront Distribution should be deployed", - ) - - cloudfront_oac: Optional[bool] = Field( - True, - description="Boolean that configures Cloufront STAC Browser Origin with Origin Access Control", - ) - - # STAC S3 browser bucket name - stac_browser_bucket: Optional[str] = Field( - None, description="STAC browser S3 bucket name" - ) - - # API Gateway URLs - ingest_url: Optional[str] = Field( - "", - description="URL of ingest API", - ) - - domain_hosted_zone_name: Optional[str] = Field( - None, - description="Domain name for the cloudfront distribution", - ) - - domain_hosted_zone_id: Optional[str] = Field( - None, description="Domain ID for the cloudfront distribution" - ) - - cert_arn: Optional[str] = Field( - None, - description="Certificate’s ARN", - ) - - shared_web_acl_id: Optional[str] = Field( - None, description="Shared Web ACL ID ARN for CloudFront Distribution" - ) - - custom_host: str = Field( - None, - description="Complete url of custom host including subdomain. Used to infer url of stac-api before app synthesis.", - ) - - class Config: - """model config""" - - env_prefix = "VEDA_" - case_sentive = False - env_file = ".env" - - -veda_route_settings = vedaRouteSettings() diff --git a/routes/infrastructure/construct.py b/routes/infrastructure/construct.py deleted file mode 100755 index 3ef67622..00000000 --- a/routes/infrastructure/construct.py +++ /dev/null @@ -1,206 +0,0 @@ -"""CDK Construct for a Cloudfront Distribution.""" -from typing import Optional - -from aws_cdk import CfnOutput, Stack -from aws_cdk import aws_certificatemanager as certificatemanager -from aws_cdk import aws_cloudfront as cf -from aws_cdk import aws_cloudfront_origins as origins -from aws_cdk import aws_iam as iam -from aws_cdk import aws_route53, aws_route53_targets -from aws_cdk import aws_s3 as s3 -from constructs import Construct - -from .config import veda_route_settings - - -class CloudfrontDistributionConstruct(Construct): - """CDK Construct for a Cloudfront Distribution.""" - - def __init__( - self, - scope: Construct, - construct_id: str, - stage: str, - raster_api_id: str, - stac_api_id: str, - origin_bucket: s3.Bucket, - region: Optional[str], - **kwargs, - ) -> None: - """.""" - super().__init__(scope, construct_id) - - stack_name = Stack.of(self).stack_name - - if veda_route_settings.cloudfront: - # Certificate must be in zone us-east-1 - domain_cert = ( - certificatemanager.Certificate.from_certificate_arn( - self, "domainCert", veda_route_settings.cert_arn - ) - if veda_route_settings.cert_arn - else None - ) - - if veda_route_settings.cloudfront_oac: - # create the origin access control resource - cfn_origin_access_control = cf.CfnOriginAccessControl( - self, - "VedaCfnOriginAccessControl", - origin_access_control_config=cf.CfnOriginAccessControl.OriginAccessControlConfigProperty( - name=f"veda-{stage}-oac", - origin_access_control_origin_type="s3", - signing_behavior="always", - signing_protocol="sigv4", - description="Origin Access Control for STAC Browser", - ), - ) - if ( - veda_route_settings.domain_hosted_zone_name - == veda_route_settings.custom_host - ): - self.cf_domain_names = [ - f"{stage}.{veda_route_settings.domain_hosted_zone_name}", - f"{veda_route_settings.domain_hosted_zone_name}", - ] - else: - self.cf_domain_names = [ - f"{stage}.{veda_route_settings.domain_hosted_zone_name}" - ] - - self.distribution = cf.Distribution( - self, - stack_name, - comment=stack_name, - default_behavior=cf.BehaviorOptions( - origin=origins.S3Origin( - origin_bucket, origin_id="stac-browser" - ), - cache_policy=cf.CachePolicy.CACHING_DISABLED, - origin_request_policy=cf.OriginRequestPolicy.CORS_S3_ORIGIN, - response_headers_policy=cf.ResponseHeadersPolicy.CORS_ALLOW_ALL_ORIGINS, - viewer_protocol_policy=cf.ViewerProtocolPolicy.REDIRECT_TO_HTTPS, - ), - certificate=domain_cert, - default_root_object="index.html", - enable_logging=True, - web_acl_id=veda_route_settings.shared_web_acl_id, - domain_names=self.cf_domain_names - if veda_route_settings.domain_hosted_zone_name - else None, - ) - # associate the created OAC with the distribution - distribution_props = self.distribution.node.default_child - if distribution_props is not None: - distribution_props.add_override( - "Properties.DistributionConfig.Origins.0.S3OriginConfig.OriginAccessIdentity", - "", - ) - distribution_props.add_property_override( - "DistributionConfig.Origins.0.OriginAccessControlId", - cfn_origin_access_control.ref, - ) - - # remove the OAI reference from the distribution - all_distribution_props = self.distribution.node.find_all() - for child in all_distribution_props: - if child.node.id == "S3Origin": - child.node.try_remove_child("Resource") - else: - self.distribution = cf.Distribution( - self, - stack_name, - comment=stack_name, - default_behavior=cf.BehaviorOptions( - origin=origins.HttpOrigin( - origin_bucket.bucket_website_domain_name, - protocol_policy=cf.OriginProtocolPolicy.HTTP_ONLY, - origin_id="stac-browser", - ), - cache_policy=cf.CachePolicy.CACHING_DISABLED, - ), - certificate=domain_cert, - default_root_object="index.html", - enable_logging=True, - domain_names=self.cf_domain_names - if veda_route_settings.domain_hosted_zone_name - else None, - ) - - self.distribution.add_behavior( - path_pattern="/api/stac*", - origin=origins.HttpOrigin( - f"{stac_api_id}.execute-api.{region}.amazonaws.com", - origin_id="stac-api", - ), - cache_policy=cf.CachePolicy.CACHING_DISABLED, - allowed_methods=cf.AllowedMethods.ALLOW_ALL, - origin_request_policy=cf.OriginRequestPolicy.ALL_VIEWER_EXCEPT_HOST_HEADER, - ) - - self.distribution.add_behavior( - path_pattern="/api/raster*", - origin=origins.HttpOrigin( - f"{raster_api_id}.execute-api.{region}.amazonaws.com", - origin_id="raster-api", - ), - cache_policy=cf.CachePolicy.CACHING_DISABLED, - allowed_methods=cf.AllowedMethods.ALLOW_ALL, - origin_request_policy=cf.OriginRequestPolicy.ALL_VIEWER_EXCEPT_HOST_HEADER, - ) - - self.hosted_zone = aws_route53.HostedZone.from_hosted_zone_attributes( - self, - "hosted-zone", - hosted_zone_id=veda_route_settings.domain_hosted_zone_id, - zone_name=veda_route_settings.domain_hosted_zone_name, - ) - - # Infer cloudfront arn to add to bucket resource policy - self.distribution_arn = f"arn:aws:cloudfront::{self.distribution.env.account}:distribution/{self.distribution.distribution_id}" - origin_bucket.add_to_resource_policy( - permission=iam.PolicyStatement( - actions=["s3:GetObject"], - conditions={ - "StringEquals": {"aws:SourceArn": self.distribution_arn} - }, - effect=iam.Effect("ALLOW"), - principals=[iam.ServicePrincipal("cloudfront.amazonaws.com")], - resources=[origin_bucket.arn_for_objects("*")], - sid="AllowCloudFrontServicePrincipal", - ) - ) - - CfnOutput(self, "Endpoint", value=self.distribution.domain_name) - - def add_ingest_behavior( - self, - ingest_api, - stage: str, - region: Optional[str] = "us-west-2", - ): - """Required as second step as ingest API depends on stac API route""" - if veda_route_settings.cloudfront: - self.distribution.add_behavior( - "/api/ingest*", - origin=origins.HttpOrigin( - f"{ingest_api.api_id}.execute-api.{region}.amazonaws.com", - origin_id="ingest-api", - ), - cache_policy=cf.CachePolicy.CACHING_DISABLED, - allowed_methods=cf.AllowedMethods.ALLOW_ALL, - origin_request_policy=cf.OriginRequestPolicy.ALL_VIEWER_EXCEPT_HOST_HEADER, - ) - - def create_route_records(self, stage: str): - """This is a seperate function so that it can be called after all behaviors are instantiated""" - if veda_route_settings.cloudfront: - aws_route53.ARecord( - self, - "cloudfront-dns-record", - zone=self.hosted_zone, - target=aws_route53.RecordTarget.from_alias( - aws_route53_targets.CloudFrontTarget(self.distribution) - ), - record_name=stage, - ) diff --git a/scripts/run-local-tests.sh b/scripts/run-local-tests.sh index 76434374..eccd1841 100755 --- a/scripts/run-local-tests.sh +++ b/scripts/run-local-tests.sh @@ -32,4 +32,7 @@ docker exec veda.db /tmp/scripts/bin/load-data.sh python -m pytest .github/workflows/tests/ -vv -s # Run ingest unit tests -NO_PYDANTIC_SSM_SETTINGS=1 python -m pytest --cov=ingest_api/runtime/src ingest_api/runtime/tests/ -vv -s \ No newline at end of file +NO_PYDANTIC_SSM_SETTINGS=1 python -m pytest --cov=ingest_api/runtime/src ingest_api/runtime/tests/ -vv -s + +# Transactions tests +python -m pytest stac_api/runtime/tests/ --asyncio-mode=auto -vv -s diff --git a/stac_api/infrastructure/construct.py b/stac_api/infrastructure/construct.py index f4848a41..c3ca56a0 100644 --- a/stac_api/infrastructure/construct.py +++ b/stac_api/infrastructure/construct.py @@ -1,8 +1,6 @@ """CDK Construct for a Lambda backed API implementing stac-fastapi.""" import os -import typing -from typing import Optional from aws_cdk import ( CfnOutput, @@ -18,9 +16,6 @@ from .config import veda_stac_settings -if typing.TYPE_CHECKING: - from domain.infrastructure.construct import DomainConstruct - class StacApiLambdaConstruct(Construct): """CDK Construct for a Lambda backed API implementing stac-fastapi.""" @@ -34,7 +29,6 @@ def __init__( database, raster_api, # TODO: typing! code_dir: str = "./", - domain: Optional["DomainConstruct"] = None, **kwargs, ) -> None: """.""" @@ -108,19 +102,10 @@ def __init__( ) ) - domain_mapping = None - # Legacy method to use a custom subdomain for this api (i.e. -stac..com) - # If using a custom root path and/or a proxy server, do not use a custom subdomain - if domain and domain.stac_domain_name: - domain_mapping = aws_apigatewayv2_alpha.DomainMappingOptions( - domain_name=domain.stac_domain_name - ) - self.stac_api = aws_apigatewayv2_alpha.HttpApi( self, f"{stack_name}-{construct_id}", default_integration=stac_api_integration, - default_domain_mapping=domain_mapping, ) CfnOutput( diff --git a/stac_api/runtime/src/validation.py b/stac_api/runtime/src/validation.py index 9f429e3c..b50d5c3a 100644 --- a/stac_api/runtime/src/validation.py +++ b/stac_api/runtime/src/validation.py @@ -6,16 +6,13 @@ from pydantic import BaseModel, Field from pystac import STACObjectType -from pystac.errors import STACValidationError +from pystac.errors import STACTypeError, STACValidationError from pystac.validation import validate_dict -from src.config import api_settings from fastapi import Request from fastapi.responses import JSONResponse from starlette.middleware.base import BaseHTTPMiddleware -path_prefix = api_settings.root_path or "" - class BulkItems(BaseModel): """Validation model for bulk-items endpoint request""" @@ -33,24 +30,25 @@ async def dispatch(self, request: Request, call_next): try: body = await request.body() request_data = json.loads(body) + if re.match( - f"^{path_prefix}/collections(?:/[^/]+)?$", + "^.*?/collections(?:/[^/]+)?$", request.url.path, ): validate_dict(request_data, STACObjectType.COLLECTION) elif re.match( - f"^{path_prefix}/collections/[^/]+/items(?:/[^/]+)?$", + "^.*?/collections/[^/]+/items(?:/[^/]+)?$", request.url.path, ): validate_dict(request_data, STACObjectType.ITEM) elif re.match( - f"^{path_prefix}/collections/[^/]+/bulk-items$", + "^.*?/collections/[^/]+/bulk_items$", request.url.path, ): bulk_items = BulkItems(**request_data) for item_data in bulk_items.items.values(): validate_dict(item_data, STACObjectType.ITEM) - except STACValidationError as e: + except (STACValidationError, STACTypeError) as e: return JSONResponse( status_code=422, content={"detail": "Validation Error", "errors": str(e)}, diff --git a/stac_api/runtime/tests/conftest.py b/stac_api/runtime/tests/conftest.py index e584bde6..b8a68269 100644 --- a/stac_api/runtime/tests/conftest.py +++ b/stac_api/runtime/tests/conftest.py @@ -9,8 +9,9 @@ import os import pytest +from httpx import ASGITransport, AsyncClient -from fastapi.testclient import TestClient +from stac_fastapi.pgstac.db import close_db_connection, connect_to_db VALID_COLLECTION = { "id": "CMIP245-winter-median-pr", @@ -209,7 +210,7 @@ } -@pytest.fixture +@pytest.fixture(autouse=True) def test_environ(): """ Set up the test environment with mocked AWS and PostgreSQL credentials. @@ -235,8 +236,8 @@ def test_environ(): os.environ["POSTGRES_USER"] = "username" os.environ["POSTGRES_PASS"] = "password" os.environ["POSTGRES_DBNAME"] = "postgis" - os.environ["POSTGRES_HOST_READER"] = "database" - os.environ["POSTGRES_HOST_WRITER"] = "database" + os.environ["POSTGRES_HOST_READER"] = "0.0.0.0" + os.environ["POSTGRES_HOST_WRITER"] = "0.0.0.0" os.environ["POSTGRES_PORT"] = "5432" @@ -251,7 +252,7 @@ def override_validated_token(): @pytest.fixture -def app(test_environ): +async def app(): """ Fixture to initialize the FastAPI application. @@ -266,11 +267,13 @@ def app(test_environ): """ from src.app import app - return app + await connect_to_db(app) + yield app + await close_db_connection(app) -@pytest.fixture -def api_client(app): +@pytest.fixture(scope="function") +async def api_client(app): """ Fixture to initialize the API client for making requests. @@ -286,7 +289,13 @@ def api_client(app): from src.app import auth app.dependency_overrides[auth.validated_token] = override_validated_token - yield TestClient(app) + base_url = "http://test" + + async with AsyncClient( + transport=ASGITransport(app=app), base_url=base_url + ) as client: + yield client + app.dependency_overrides.clear() diff --git a/stac_api/runtime/tests/test_transactions.py b/stac_api/runtime/tests/test_transactions.py index 6a5cd8e3..4961f287 100644 --- a/stac_api/runtime/tests/test_transactions.py +++ b/stac_api/runtime/tests/test_transactions.py @@ -53,59 +53,59 @@ def setup( self.invalid_stac_collection = invalid_stac_collection self.invalid_stac_item = invalid_stac_item - def test_post_invalid_collection(self): + async def test_post_invalid_collection(self): """ Test the API's response to posting an invalid STAC collection. Asserts that the response status code is 422 and the detail is "Validation Error". """ - response = self.api_client.post( + response = await self.api_client.post( collections_endpoint, json=self.invalid_stac_collection ) assert response.json()["detail"] == "Validation Error" assert response.status_code == 422 - def test_post_valid_collection(self): + async def test_post_valid_collection(self): """ Test the API's response to posting a valid STAC collection. Asserts that the response status code is 200. """ - response = self.api_client.post( + response = await self.api_client.post( collections_endpoint, json=self.valid_stac_collection ) # assert response.json() == {} assert response.status_code == 200 - def test_post_invalid_item(self): + async def test_post_invalid_item(self): """ Test the API's response to posting an invalid STAC item. Asserts that the response status code is 422 and the detail is "Validation Error". """ - response = self.api_client.post( + response = await self.api_client.post( items_endpoint.format(self.invalid_stac_item["collection"]), json=self.invalid_stac_item, ) assert response.json()["detail"] == "Validation Error" assert response.status_code == 422 - def test_post_valid_item(self): + async def test_post_valid_item(self): """ Test the API's response to posting a valid STAC item. Asserts that the response status code is 200. """ - response = self.api_client.post( + response = await self.api_client.post( items_endpoint.format(self.valid_stac_item["collection"]), json=self.valid_stac_item, ) # assert response.json() == {} assert response.status_code == 200 - def test_post_invalid_bulk_items(self): + async def test_post_invalid_bulk_items(self): """ Test the API's response to posting invalid bulk STAC items. @@ -117,12 +117,12 @@ def test_post_invalid_bulk_items(self): "items": {item_id: self.invalid_stac_item}, "method": "upsert", } - response = self.api_client.post( + response = await self.api_client.post( bulk_endpoint.format(collection_id), json=invalid_request ) assert response.status_code == 422 - def test_post_valid_bulk_items(self): + async def test_post_valid_bulk_items(self): """ Test the API's response to posting valid bulk STAC items. @@ -131,7 +131,7 @@ def test_post_valid_bulk_items(self): item_id = self.valid_stac_item["id"] collection_id = self.valid_stac_item["collection"] valid_request = {"items": {item_id: self.valid_stac_item}, "method": "upsert"} - response = self.api_client.post( + response = await self.api_client.post( bulk_endpoint.format(collection_id), json=valid_request ) assert response.status_code == 200 diff --git a/standalone_base_infrastructure/network_construct.py b/standalone_base_infrastructure/network_construct.py index d8785d4a..3911c5e3 100644 --- a/standalone_base_infrastructure/network_construct.py +++ b/standalone_base_infrastructure/network_construct.py @@ -42,6 +42,9 @@ def __init__( "cloudwatch-logs": aws_ec2.InterfaceVpcEndpointAwsService.CLOUDWATCH_LOGS, "s3": aws_ec2.GatewayVpcEndpointAwsService.S3, "dynamodb": aws_ec2.GatewayVpcEndpointAwsService.DYNAMODB, + "ecr": aws_ec2.InterfaceVpcEndpointAwsService.ECR, # allows airflow to pull task images + "ecr-docker": aws_ec2.InterfaceVpcEndpointAwsService.ECR_DOCKER, # allows airflow to pull task images + "sts": aws_ec2.InterfaceVpcEndpointAwsService.STS, # allows airflow tasks to assume access roles } for id, service in vpc_endpoints.items():