Feature / Azure platform storage #1232
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Integration | |
| on: | |
| # Integration tests always run on pull requests | |
| pull_request: | |
| # Re-run integration tests in main, to make sure there are no issues from the merge | |
| push: | |
| branches: | |
| - main | |
| # Allow manual triggering of integration tests | |
| workflow_dispatch: | |
| # Use baseline language versions for integration builds | |
| env: | |
| JAVA_VERSION: 17 | |
| JAVA_DISTRIBUTION: zulu | |
| PYTHON_VERSION: "3.11" | |
| NODE_VERSION: 18 | |
| jobs: | |
| web-api: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup - Java | |
| uses: actions/setup-java@v3 | |
| with: | |
| distribution: ${{ env.JAVA_DISTRIBUTION }} | |
| java-version: ${{ env.JAVA_VERSION }} | |
| # Turn on Gradle dependency caching | |
| cache: gradle | |
| - name: Setup -Node.js | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| - name: Platform - Build distribution files | |
| run: ./gradlew installDist | |
| - name: Platform - Install the sandbox | |
| run: | | |
| VERSION=`dev/version.sh` | |
| mkdir -p build/dist/tracdap-sandbox-${VERSION} | |
| for MODULE in build/modules/*/install/*; do | |
| cp -R $MODULE/* build/dist/tracdap-sandbox-${VERSION} | |
| done | |
| cd build/dist | |
| ln -s tracdap-sandbox-${VERSION}/ current | |
| # Currently not starting svc-orch, we'd need to build the runtime and deploy an execution venv | |
| # OUr JavaScript tests only do data and metadata so far, no spawning / checking jobs | |
| # We should aim to get there though! | |
| - name: Platform - Run setup tools | |
| run: | | |
| cd build/dist/current | |
| deploy_dir=`pwd` | |
| mkdir metadata | |
| mkdir -p data/storage1 | |
| echo SECRET_KEY=xHjFeFlo$k= >> etc/env.sh | |
| sed -i "s#/path/to#${deploy_dir}#g" etc/trac-platform.yaml | |
| bin/secret-tool run --task init_secrets | |
| bin/secret-tool run --task create_root_auth_key EC 256 | |
| bin/deploy-metadb run --task deploy_schema | |
| bin/deploy-metadb run --task add_tenant ACME_CORP "ACME Rockets always get the Coyote!" | |
| - name: Platform - Start | |
| run: | | |
| cd build/dist/current | |
| bin/tracdap-svc-meta start | |
| bin/tracdap-svc-data start | |
| bin/tracdap-gateway start | |
| - name: Web API - Install dependencies | |
| run: | | |
| cd tracdap-api/packages/web | |
| npm install | |
| - name: Web API - Set TRAC version | |
| run: | | |
| cd tracdap-api/packages/web | |
| npm run tracVersion:posix | |
| - name: Web API - Build package | |
| run: | | |
| cd tracdap-api/packages/web | |
| npm run buildApi | |
| - name: Examples - Install dependencies | |
| run: | | |
| cd examples/apps/javascript | |
| npm install | |
| - name: Examples - Run all | |
| run: | | |
| cd examples/apps/javascript | |
| npm run examples | |
| - name: Platform - Shut down | |
| if: always() | |
| run: | | |
| cd build/dist/current | |
| bin/tracdap-gateway stop | |
| bin/tracdap-svc-data stop | |
| bin/tracdap-svc-meta stop | |
| slow-tests: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| steps: | |
| # fetch-depth = 0 is needed to get tags for version info | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Java | |
| uses: actions/setup-java@v3 | |
| with: | |
| distribution: ${{ env.JAVA_DISTRIBUTION }} | |
| java-version: ${{ env.JAVA_VERSION }} | |
| cache: gradle | |
| - name: Build | |
| run: ./gradlew testClasses | |
| - name: Slow tests | |
| run: ./gradlew integration -DintegrationTags="slow" | |
| # If the tests fail, make the output available for download | |
| - name: Store failed test results | |
| uses: actions/upload-artifact@v3 | |
| if: failure() | |
| with: | |
| name: junit-test-results | |
| path: build/modules/*/reports/** | |
| retention-days: 7 | |
| end-to-end: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| steps: | |
| # fetch-depth = 0 is needed to get tags for version info | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Java - Setup | |
| uses: actions/setup-java@v3 | |
| with: | |
| distribution: ${{ env.JAVA_DISTRIBUTION }} | |
| java-version: ${{ env.JAVA_VERSION }} | |
| # Turn on Gradle dependency caching | |
| cache: gradle | |
| - name: Java - Build platform | |
| run: ./gradlew testClasses | |
| - name: Python - setup | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Python - PIP | |
| run: python -m pip install --upgrade pip | |
| - name: Python - Install build dependencies | |
| run: | | |
| pip install -r tracdap-runtime/python/requirements.txt | |
| - name: Python - Build runtime package | |
| run: python tracdap-runtime/python/build_runtime.py --target codegen dist | |
| - name: Run end to end tests | |
| run: ./gradlew integration -DintegrationTags="int-e2e" | |
| # If the tests fail, make the output available for download | |
| - name: Store failed test results | |
| uses: actions/upload-artifact@v3 | |
| if: failure() | |
| with: | |
| name: junit-test-results | |
| path: build/modules/*/reports/** | |
| retention-days: 7 | |
| int-metadb: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| container: | |
| image: ubuntu:latest | |
| strategy: | |
| # Try to finish all jobs - it can be helpful to see if some succeed and others fail | |
| fail-fast: false | |
| matrix: | |
| database: | |
| - { DB_NAME: MySQL, | |
| DB_IMAGE: 'mysql:8.0', | |
| DB_PORT: 3306, | |
| DB_OPTIONS: '--health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3', | |
| BUILD_sql_mysql: true, | |
| TRAC_CONFIG_FILE: '.github/config/int-metadb-mysql.yaml', | |
| TRAC_SECRET_KEY: wDeq3x-NjaLL7, | |
| MYSQL_DATABASE: trac, | |
| MYSQL_USER: trac_admin, | |
| MYSQL_PASSWORD: trac_admin, | |
| METADB_SECRET: trac_admin, | |
| MYSQL_ALLOW_EMPTY_PASSWORD: yes } | |
| - { DB_NAME: MariaDB, | |
| DB_IMAGE: 'mariadb:10.3', | |
| DB_PORT: 3306, | |
| DB_OPTIONS: '--health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3', | |
| BUILD_sql_mariadb: true, | |
| TRAC_CONFIG_FILE: '.github/config/int-metadb-mariadb.yaml', | |
| TRAC_SECRET_KEY: uYhnKwq8+esS, | |
| MYSQL_DATABASE: trac, | |
| MYSQL_USER: trac_admin, | |
| MYSQL_PASSWORD: trac_admin, | |
| METADB_SECRET: trac_admin, | |
| MYSQL_ALLOW_EMPTY_PASSWORD: yes } | |
| - { DB_NAME: PostgreSQL, | |
| DB_IMAGE: 'postgres:10-alpine', | |
| DB_PORT: 5432, | |
| DB_OPTIONS: '--health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5', | |
| BUILD_sql_postgresql: true, | |
| TRAC_CONFIG_FILE: '.github/config/int-metadb-postgresql.yaml', | |
| TRAC_SECRET_KEY: hjXks83bX=wxMr, | |
| POSTGRES_DB: trac, | |
| POSTGRES_USER: trac_admin, | |
| METADB_SECRET: trac_admin, | |
| POSTGRES_PASSWORD: trac_admin } | |
| - { DB_NAME: SQLServer, | |
| DB_IMAGE: 'mcr.microsoft.com/mssql/server:2019-latest', | |
| DB_PORT: 1433, | |
| DB_OPTIONS: '-e "NO_DB_OPTIONS=not_used"', # docker run -e flag sets an env variable, passing '' causes errors | |
| BUILD_sql_sqlserver: true, | |
| TRAC_CONFIG_FILE: '.github/config/int-metadb-sqlserver.yaml', | |
| TRAC_SECRET_KEY: unHkj>weN2jSl, | |
| MSSQL_PID: Developer, | |
| ACCEPT_EULA: Y, | |
| SA_PASSWORD: "tR4c_aDm!n", | |
| METADB_SECRET: "tR4c_aDm!n" } | |
| env: ${{ matrix.database }} | |
| services: | |
| metadb: | |
| image: ${{ matrix.database.DB_IMAGE }} | |
| env: ${{ matrix.database }} | |
| ports: | |
| - ${{ matrix.database.DB_PORT }}:${{ matrix.database.DB_PORT }} | |
| options: ${{ matrix.database.DB_OPTIONS }} | |
| steps: | |
| # fetch-depth = 0 is needed to get tags for version info | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Java | |
| uses: actions/setup-java@v3 | |
| with: | |
| distribution: ${{ env.JAVA_DISTRIBUTION }} | |
| java-version: ${{ env.JAVA_VERSION }} | |
| cache: gradle | |
| - name: Build | |
| run: ./gradlew trac-svc-meta:testClasses | |
| # Auth tool will also create the secrets file if it doesn't exist | |
| - name: Prepare secrets | |
| run: | | |
| ./gradlew secret-tool:run --args="--config ${{ env.TRAC_CONFIG_FILE }} --task init_secrets" | |
| ./gradlew secret-tool:run --args="--config ${{ env.TRAC_CONFIG_FILE }} --task create_root_auth_key EC 256" | |
| echo "${METADB_SECRET}" | ./gradlew secret-tool:run --args="--config ${{ env.TRAC_CONFIG_FILE }} --task add_secret metadb_secret" | |
| # The name and description of the test tenant are verified in one of the test cases so they need to match | |
| # MetadataReapApiTest listTenants() | |
| - name: Prepare database | |
| run: | | |
| ./gradlew deploy-metadb:run --args="\ | |
| --config ${{ env.TRAC_CONFIG_FILE }} \ | |
| --secret-key ${{ env.TRAC_SECRET_KEY }} \ | |
| --task deploy_schema \ | |
| --task add_tenant ACME_CORP 'Test tenant [ACME_CORP]'" | |
| - name: Integration tests | |
| run: ./gradlew trac-svc-meta:integration -DintegrationTags="int-metadb" | |
| # If the tests fail, make the output available for download | |
| - name: Store failed test results | |
| uses: actions/upload-artifact@v3 | |
| if: failure() | |
| with: | |
| name: junit-test-results | |
| path: build/modules/*/reports/** | |
| retention-days: 7 | |
| int-storage: | |
| strategy: | |
| # Try to finish all jobs - it can be helpful to see if some succeed and others fail | |
| fail-fast: false | |
| matrix: | |
| storage: | |
| - { SERVICE: storage, | |
| TARGET: aws, | |
| PROTOCOL: S3, | |
| ENABLE_IF: TRAC_AWS_BUCKET, | |
| BUILD_aws_storage: true, | |
| INSTALL_SCRIPT: ".github/scripts/int-storage-s3-install.sh", | |
| SETUP_SCRIPT: ".github/scripts/int-storage-s3-setup.sh", | |
| TRAC_CONFIG_FILE: '.github/config/int-storage-s3.yaml', | |
| TRAC_SECRET_KEY: short-lived-secret, | |
| PYTHON_TESTS: int_storage_aws*.py, | |
| JAVA_TESTS: int-storage } | |
| - { SERVICE: storage, | |
| TARGET: gcp, | |
| PROTOCOL: GCS, | |
| ENABLE_IF: TRAC_GCP_BUCKET, | |
| BUILD_gcp_storage: true, | |
| INSTALL_SCRIPT: ".github/scripts/int-storage-gcs-install.sh", | |
| SETUP_SCRIPT: ".github/scripts/int-storage-gcs-setup.sh", | |
| TRAC_CONFIG_FILE: '.github/config/int-storage-gcs.yaml', | |
| TRAC_SECRET_KEY: short-lived-secret, | |
| PYTHON_TESTS: int_storage_gcp*.py, | |
| JAVA_TESTS: int-storage } | |
| - { SERVICE: storage, | |
| TARGET: azure, | |
| PROTOCOL: BLOB, | |
| ENABLE_IF: TRAC_AZURE_CONTAINER, | |
| BUILD_azure_storage: true, | |
| INSTALL_SCRIPT: ".github/scripts/int-storage-azure-install.sh", | |
| SETUP_SCRIPT: ".github/scripts/int-storage-azure-setup.sh", | |
| TRAC_CONFIG_FILE: '.github/config/int-storage-azure.yaml', | |
| TRAC_SECRET_KEY: short-lived-secret, | |
| PYTHON_TESTS: int_storage_azure*.py, | |
| JAVA_TESTS: int-storage } | |
| uses: ./.github/workflows/integration-cloud.yaml | |
| with: | |
| service: ${{ matrix.storage.SERVICE }} | |
| target: ${{ matrix.storage.TARGET }} | |
| protocol: ${{ matrix.storage.PROTOCOL }} | |
| matrix: ${{ toJson( matrix.storage ) }} | |
| enabled: ${{ vars[ matrix.storage.ENABLE_IF ] != '' }} | |
| has_java: ${{ matrix.storage.JAVA_TESTS != '' }} | |
| has_python: ${{ matrix.storage.PYTHON_TESTS != '' }} | |
| secrets: inherit | |
| # Permissions required to use IdP to connect to cloud platforms | |
| permissions: | |
| id-token: write # This is required for requesting the JWT | |
| contents: read # This is required for actions/checkout |