diff --git a/.env b/.env new file mode 100644 index 000000000..7cb97cea6 --- /dev/null +++ b/.env @@ -0,0 +1,25 @@ +# Application +APP_PORT=3000 + +# Cognito +COGNITO_AWS_REGION=eu-west-1 +COGNITO_CLIENT_ID=10h1tga4i933buv25lelalmtrn +COGNITO_POOL_ID=eu-west-1_dbfEb2FuH + +# S3 +S3_AWS_REGION=eu-west-1 +AWS_ACCESS_KEY_ID=minio +AWS_SECRET_ACCESS_KEY=minio123 +S3_BUCKET=ngmpub-userdata-local +PROJECTS_S3_BUCKET=ngmpub-project-files-local +S3_ENDPOINT=http://minio:9000 + +# Database +PGUSER=www-data +PGPASSWORD=www-data +PGHOST=db +PGPORT=5432 +PGDATABASE=swissgeol-local + +# sqlx +DATABASE_URL=postgres://${PGUSER}:${PGPASSWORD}@${PGHOST}:${PGPORT}/${PGDATABASE} diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml index 948ced943..56a9e7c91 100644 --- a/.github/workflows/code-quality.yml +++ b/.github/workflows/code-quality.yml @@ -9,10 +9,7 @@ on: env: NODE_VERSION: "22.x" - DB_USERNAME: postgres - DB_PASSWORD: postgres - DB_DATABASE: postgres - DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgres?schema=public + RUST_VERSION: "1.73" jobs: # add_review_links: @@ -33,7 +30,9 @@ jobs: - name: Review Dependencies uses: actions/dependency-review-action@v4 + install-ui: + name: "Install UI" runs-on: ubuntu-latest steps: - name: Checkout repository @@ -59,19 +58,15 @@ jobs: key: "${{ runner.os }}-node_modules-${{ env.NODE_VERSION }}-${{ hashFiles('./ui/package-lock.json') }}" restore-keys: | ${{ runner.os }}-node_modules- -# - name: Cache e2e node modules -# uses: actions/cache@v4 -# with: -# path: ./e2e/node_modules -# key: "${{ runner.os }}-node_modules_e2e-${{ env.NODE_VERSION }}-${{ hashFiles('./e2e/package-lock.json') }}" -# restore-keys: | -# ${{ runner.os }}-node_modules_e2e- - name: Install node dependencies run: cd ui && npm install - test-ui: + + typecheck-ui: + name: "Type-check UI" runs-on: ubuntu-latest - needs: install-ui + needs: + - install-ui steps: - name: Checkout repository uses: actions/checkout@v4 @@ -84,12 +79,17 @@ jobs: with: path: ./ui/node_modules key: "${{ runner.os }}-node_modules-${{ env.NODE_VERSION }}-${{ hashFiles('./ui/package-lock.json') }}" - - name: Run tests - run: cd ui && npm run test + # - name: Reset nx + # run: npx nx reset + - name: Run build + run: cd ui && npm run check - lint-ui: + + test-ui: + name: "Test UI" runs-on: ubuntu-latest - needs: install-ui + needs: + - typecheck-ui steps: - name: Checkout repository uses: actions/checkout@v4 @@ -102,23 +102,15 @@ jobs: with: path: ./ui/node_modules key: "${{ runner.os }}-node_modules-${{ env.NODE_VERSION }}-${{ hashFiles('./ui/package-lock.json') }}" - - name: Run lint - run: cd ui && npm run lint -# - name: Run prettier -# run: npx prettier --check . + - name: Run tests + run: cd ui && npm run test - # It would be cleaner and probably more performant to replace this build step - # with either a non-emitting build or a simple type check. - # We only have `build` available for now, - # since the project is currently split across a multitude of small packages, - # all of which have to specify their own commands. - # (Daniel von Atzigen, 2024-04-12) - build-ui: + + lint-ui: + name: "Lint UI" runs-on: ubuntu-latest needs: - - test-ui - - lint-ui - - dependency-review + - typecheck-ui steps: - name: Checkout repository uses: actions/checkout@v4 @@ -131,148 +123,121 @@ jobs: with: path: ./ui/node_modules key: "${{ runner.os }}-node_modules-${{ env.NODE_VERSION }}-${{ hashFiles('./ui/package-lock.json') }}" -# - name: Reset nx -# run: npx nx reset - - name: Run build - run: cd ui && npm run build - + - name: Run lint + run: cd ui && npm run lint install-api: + name: "Install API" runs-on: ubuntu-latest steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Install Rust toolchain uses: actions-rs/toolchain@v1 with: - toolchain: 1.73 + toolchain: ${{ env.RUST_VERSION }} profile: minimal override: true - - name: Cache cargo registry uses: actions/cache@v4 with: path: ~/.cargo/registry - key: cargo-registry-${{ runner.os }}-${{ hashFiles('./api/Cargo.lock') }} + key: "${{ runner.os }}-cargo_registry-${{ env.RUST_VERSION }}-${{ hashFiles('./api/Cargo.lock') }}" restore-keys: | - cargo-registry-${{ runner.os }}- - + ${{ runner.os }}-cargo_registry- - name: Cache cargo index uses: actions/cache@v4 with: path: ~/.cargo/git - key: cargo-index-${{ runner.os }}-${{ hashFiles('./api/Cargo.lock') }} + key: "${{ runner.os }}-cargo_index-${{ env.RUST_VERSION }}-${{ hashFiles('./api/Cargo.lock') }}" restore-keys: | - cargo-index-${{ runner.os }}- - + ${{ runner.os }}-cargo_index- - name: Cache cargo build uses: actions/cache@v4 with: - path: target - key: cargo-build-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }} + path: ./api/target + key: "${{ runner.os }}-cargo_build-${{ env.RUST_VERSION }}-${{ hashFiles('./api/Cargo.lock') }}" restore-keys: | - cargo-build-${{ runner.os }}- - + ${{ runner.os }}-cargo_build- - name: Cache sqlx binary uses: actions/cache@v4 with: path: ~/.cargo/bin/sqlx - key: sqlx-${{ runner.os }}-${{ hashFiles('./api/Cargo.lock') }} + key: "${{ runner.os }}-sqlx-${{ env.RUST_VERSION }}-${{ hashFiles('./api/Cargo.lock') }}" restore-keys: | - sqlx-${{ runner.os }}- - - - name: Install Rust dependencies - env: - DATABASE_URL: "postgres://www-data:www-data@localhost:15432/swissgeol-local" + ${{ runner.os }}-sqlx- + - name: Start DB + run: + docker compose up -d db + sleep 10 + - name: Setup DB run: | - docker compose up -d db - sleep 10 cd api - if [ ! -f ~/.cargo/bin/sqlx ]; then cargo install sqlx-cli --version 0.7.3 --no-default-features --features native-tls,postgres --locked --quiet - fi -# sqlx database create -# sqlx migrate run - - - name: Cache shared volume - uses: actions/cache@v4 + sqlx database create + sqlx migrate run + - name: Upload DB data + uses: actions/upload-artifact@v4 with: - path: /var/lib/postgresql/data - key: db-data-${{ runner.os }}-${{ hashFiles('./api/Cargo.lock') }} - restore-keys: | - db-data-${{ runner.os }}- + name: db + path: /var/lib/docker/volumes/${{ github.workspace }}_db + - name: Install dependencies + run: | + cd api + rm -r src/* + echo "fn main() {}" > src/main.rs + cargo build --all-targets --locked --quiet + - name: Stop services + run: | + docker compose down - format-api: + lint-api: + name: "Lint API" runs-on: ubuntu-latest needs: - install-api steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Install Rust toolchain uses: actions-rs/toolchain@v1 with: - toolchain: 1.73 + toolchain: ${{ env.RUST_VERSION }} profile: minimal override: true - - - name: Restore cached cargo registry + - name: Cache cargo registry uses: actions/cache/restore@v4 with: path: ~/.cargo/registry - key: cargo-registry-${{ runner.os }}-${{ hashFiles('./api/Cargo.lock') }} - restore-keys: | - cargo-registry-${{ runner.os }}- - - - name: Restore cached cargo index + key: "${{ runner.os }}-cargo_registry-${{ env.RUST_VERSION }}-${{ hashFiles('./api/Cargo.lock') }}" + - name: Cache cargo index uses: actions/cache/restore@v4 with: path: ~/.cargo/git - key: cargo-index-${{ runner.os }}-${{ hashFiles('./api/Cargo.lock') }} - restore-keys: | - cargo-index-${{ runner.os }}- - - - name: Restore cached cargo build + key: "${{ runner.os }}-cargo_index-${{ env.RUST_VERSION }}-${{ hashFiles('./api/Cargo.lock') }}" + - name: Cache cargo build uses: actions/cache/restore@v4 with: - path: target - key: cargo-build-${{ runner.os }}-${{ hashFiles('./api/Cargo.lock') }} - restore-keys: | - cargo-build-${{ runner.os }}- - - - name: Restore cached sqlx binary + path: ./api/target + key: "${{ runner.os }}-cargo_build-${{ env.RUST_VERSION }}-${{ hashFiles('/api/Cargo.lock') }}" + - name: Cache sqlx binary uses: actions/cache/restore@v4 with: path: ~/.cargo/bin/sqlx - key: sqlx-${{ runner.os }}-${{ hashFiles('./api/Cargo.lock') }} - restore-keys: | - sqlx-${{ runner.os }}- - -# - name: Install Rust dependencies -# env: -# DATABASE_URL: "postgres://www-data:www-data@localhost:15432/swissgeol-local" -# run: | -# docker compose up -d db -# sleep 10 -# cd api -# sqlx database create -# sqlx migrate run - - name: Run container + key: "${{ runner.os }}-sqlx-${{ env.RUST_VERSION }}-${{ hashFiles('./api/Cargo.lock') }}" + - name: Run rustfmt run: | - docker compose up -d api - sleep 10 - - name: Run fmt - run: | - docker compose exec api rustup component add rustfmt - docker compose exec api cargo fmt + cd api + rustup component add rustfmt + cargo fmt --check - name: Run clippy run: | - docker compose exec api rustup component add clippy - docker compose exec api cargo clippy + cd api + rustup component add clippy + cargo clippy --frozen + test-api: runs-on: ubuntu-latest @@ -281,143 +246,117 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Install Rust toolchain uses: actions-rs/toolchain@v1 with: - toolchain: 1.73 + toolchain: ${{ env.RUST_VERSION }} profile: minimal override: true - - - name: Restore cached cargo registry + - name: Cache cargo registry uses: actions/cache/restore@v4 with: path: ~/.cargo/registry - key: cargo-registry-${{ runner.os }}-${{ hashFiles('./api/Cargo.lock') }} - restore-keys: | - cargo-registry-${{ runner.os }}- - - - name: Restore cached cargo index + key: "${{ runner.os }}-cargo_registry-${{ env.RUST_VERSION }}-${{ hashFiles('./api/Cargo.lock') }}" + - name: Cache cargo index uses: actions/cache/restore@v4 with: path: ~/.cargo/git - key: cargo-index-${{ runner.os }}-${{ hashFiles('./api/Cargo.lock') }} - restore-keys: | - cargo-index-${{ runner.os }}- - - - name: Restore cached cargo build + key: "${{ runner.os }}-cargo_index-${{ env.RUST_VERSION }}-${{ hashFiles('./api/Cargo.lock') }}" + - name: Cache cargo build uses: actions/cache/restore@v4 with: - path: target - key: cargo-build-${{ runner.os }}-${{ hashFiles('./api/Cargo.lock') }} - restore-keys: | - cargo-build-${{ runner.os }}- - - - name: Restore cached sqlx binary + path: ./api/target + key: "${{ runner.os }}-cargo_build-${{ env.RUST_VERSION }}-${{ hashFiles('/api/Cargo.lock') }}" + - name: Cache sqlx binary uses: actions/cache/restore@v4 with: path: ~/.cargo/bin/sqlx - key: sqlx-${{ runner.os }}-${{ hashFiles('./api/Cargo.lock') }} - restore-keys: | - sqlx-${{ runner.os }}- - -# - name: Install Rust dependencies -# env: -# DATABASE_URL: "postgres://www-data:www-data@localhost:15432/swissgeol-local" -# run: | -# docker compose up -d db -# sleep 10 -# cd api -# sqlx database create -# sqlx migrate run + key: "${{ runner.os }}-sqlx-${{ env.RUST_VERSION }}-${{ hashFiles('./api/Cargo.lock') }}" + - name: Download DB data + uses: actions/download-artifact@v4 + with: + name: db + path: /var/lib/docker/volumes/${{ github.workspace }}_db + - name: Start DB + run: + docker compose up -d db + sleep 10 + - name: Start API + run: + cd api + nohup cargo run --frozen > ../api.log 2>&1 & + echo $! > ../api.pid - name: Run test run: | - docker compose up -d api - sleep 10 cd api - docker compose exec api cargo test --offline + cargo test --frozen + - name: Stop services + run: | + kill $(cat api.pid) + docker compose down + - build-api: + check-api: runs-on: ubuntu-latest needs: -# - test-api -# - format-api - install-api - - dependency-review steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Install Rust toolchain uses: actions-rs/toolchain@v1 with: - toolchain: 1.73 + toolchain: ${{ env.RUST_VERSION }} profile: minimal override: true - - - name: Restore cached cargo registry + - name: Cache cargo registry uses: actions/cache/restore@v4 with: path: ~/.cargo/registry - key: cargo-registry-${{ runner.os }}-${{ hashFiles('./api/Cargo.lock') }} - restore-keys: | - cargo-registry-${{ runner.os }}- - - - name: Restore cached cargo index + key: "${{ runner.os }}-cargo_registry-${{ env.RUST_VERSION }}-${{ hashFiles('./api/Cargo.lock') }}" + - name: Cache cargo index uses: actions/cache/restore@v4 with: path: ~/.cargo/git - key: cargo-index-${{ runner.os }}-${{ hashFiles('./api/Cargo.lock') }} - restore-keys: | - cargo-index-${{ runner.os }}- - - - name: Restore cached cargo build + key: "${{ runner.os }}-cargo_index-${{ env.RUST_VERSION }}-${{ hashFiles('./api/Cargo.lock') }}" + - name: Cache cargo build uses: actions/cache/restore@v4 with: - path: target - key: cargo-build-${{ runner.os }}-${{ hashFiles('./api/Cargo.lock') }} - restore-keys: | - cargo-build-${{ runner.os }}- - - - name: Restore cached sqlx binary + path: ./api/target + key: "${{ runner.os }}-cargo_build-${{ env.RUST_VERSION }}-${{ hashFiles('/api/Cargo.lock') }}" + - name: Cache sqlx binary uses: actions/cache/restore@v4 with: path: ~/.cargo/bin/sqlx - key: sqlx-${{ runner.os }}-${{ hashFiles('./api/Cargo.lock') }} - restore-keys: | - sqlx-${{ runner.os }}- - - - name: Install Rust dependencies - env: - DATABASE_URL: "postgres://www-data:www-data@localhost:15432/swissgeol-local" - run: | - docker compose up -d db - sleep 10 - cd api -# sqlx database create -# sqlx migrate run - - name: Build API - env: - S3_AWS_REGION: eu-west-1 - AWS_ACCESS_KEY_ID: minio - AWS_SECRET_ACCESS_KEY: minio123 - S3_BUCKET: ngmpub-userdata-local - PROJECTS_S3_BUCKET: ngmpub-project-files-local - S3_ENDPOINT: http://minio:9000 - COGNITO_CLIENT_ID: 10h1tga4i933buv25lelalmtrn - COGNITO_POOL_ID: eu-west-1_dbfEb2FuH - PGUSER: www-data - PGPASSWORD: www-data - PGDATABASE: swissgeol-local - PGHOST: db - PGPORT: "5432" - APP_PORT: "3000" - SQLX_OFFLINE: "true" - DATABASE_URL: "postgres://www-data:www-data@db:5432/swissgeol-local" + key: "${{ runner.os }}-sqlx-${{ env.RUST_VERSION }}-${{ hashFiles('./api/Cargo.lock') }}" + - name: Run check run: | - apt update && apt install -y musl-tools musl-dev musl-gcc - rustup target add x86_64-unknown-linux-musl cd api - cargo build --target x86_64-unknown-linux-musl + cargo check --frozen + cleanup: + runs-on: ubuntu-latest + needs: + - build-api + - lint-ui + - test-ui + if: always() + steps: + - name: Find DB artifact + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + artifact_name="db" + artifacts=$(curl -s -H "Authorization: token $GITHUB_TOKEN" \ + "https://api.github.com/repos/${{ github.repository }}/actions/artifacts") + artifact_id=$(echo "$artifacts" | jq -r ".artifacts[] | select(.name==\"$artifact_name\") | .id") + echo "artifact_id=$artifact_id" >> $GITHUB_ENV + + - name: Delete DB artifact + if: ${{ env.artifact_id != '' }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + curl -X DELETE -H "Authorization: token $GITHUB_TOKEN" \ + "https://api.github.com/repos/${{ github.repository }}/actions/artifacts/${{ env.artifact_id }}" diff --git a/api/.env b/api/.env index 8dc3d9d6b..c7e39898a 100644 --- a/api/.env +++ b/api/.env @@ -8,11 +8,11 @@ COGNITO_POOL_ID=eu-west-1_dbfEb2FuH # S3 S3_AWS_REGION=eu-west-1 -AWS_ACCESS_KEY_ID=bla -AWS_SECRET_ACCESS_KEY=hu +AWS_ACCESS_KEY_ID=minio +AWS_SECRET_ACCESS_KEY=minio123 S3_BUCKET=ngmpub-userdata-local PROJECTS_S3_BUCKET=ngmpub-project-files-local -S3_ENDPOINT="http://minio:9000" +S3_ENDPOINT=http://minio:9000 # Database PGUSER=www-data @@ -21,5 +21,5 @@ PGHOST=localhost PGPORT=15432 PGDATABASE=swissgeol-local -# Required for using sqlx & tracing macros -DATABASE_URL="postgres://www-data:www-data@localhost:15432/swissgeol-local" +# sqlx +DATABASE_URL=postgres://${PGUSER}:${PGPASSWORD}@${PGHOST}:${PGPORT}/${PGDATABASE} diff --git a/docker-compose.yaml b/docker-compose.yaml index c526b861b..2c8f972ca 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -7,28 +7,9 @@ services: dockerfile: DockerfileDev platform: linux/amd64 ports: - - 8480:3000 + - "8480:3000" environment: - # S3 - S3_AWS_REGION: &minio_region eu-west-1 - AWS_ACCESS_KEY_ID: &minio_user minio - AWS_SECRET_ACCESS_KEY: &minio_pass minio123 - S3_BUCKET: ngmpub-userdata-local - PROJECTS_S3_BUCKET: ngmpub-project-files-local - S3_ENDPOINT: http://minio:9000 - # Cognito - COGNITO_CLIENT_ID: 10h1tga4i933buv25lelalmtrn - COGNITO_POOL_ID: eu-west-1_dbfEb2FuH - # Postgres - PGUSER: &db_user www-data - PGPASSWORD: &db_pass www-data - PGDATABASE: &db_name swissgeol-local - PGHOST: db - PGPORT: "5432" - # Api - APP_PORT: "3000" - SQLX_OFFLINE: "true" - DATABASE_URL: "postgres://www-data:www-data@db:5432/swissgeol-local" + SQLX_OFFLINE: true command: ["cargo", "watch", "--poll", "--shell", "cargo run --offline --target x86_64-unknown-linux-musl"] volumes: - ./api/src:/app/src:ro @@ -37,8 +18,6 @@ services: - ./api/Cargo.toml:/app/Cargo.toml:ro links: - db -# env_file: -# - ./api/.env ui: image: node:lts @@ -59,15 +38,15 @@ services: image: minio/minio:latest command: server /data --console-address :9001 volumes: - - miniodata:/data + - minio:/data ports: - "9000:9000" - "9001:9001" environment: MINIO_BROWSER: 'on' - MINIO_SITE_REGION: *minio_region - MINIO_ROOT_USER: *minio_user - MINIO_ROOT_PASSWORD: *minio_pass + MINIO_SITE_REGION: ${S3_AWS_REGION} + MINIO_ROOT_USER: ${AWS_ACCESS_KEY_ID} + MINIO_ROOT_PASSWORD: ${AWS_SECRET_ACCESS_KEY} abbreviator: image: ghcr.io/swisstopo/swissgeol-viewer-app-abbreviator:main @@ -83,15 +62,15 @@ services: image: camptocamp/postgres:14-postgis-3 platform: linux/amd64 environment: - POSTGRES_USER: *db_user - POSTGRES_PASSWORD: *db_pass - POSTGRES_DB: *db_name + POSTGRES_USER: ${PGUSER} + POSTGRES_PASSWORD: ${PGPASSWORD} + POSTGRES_DB: ${PGDATABASE} ports: - - 15432:5432 + - "15432:5432" volumes: - - db-data:/var/lib/postgresql/data + - db:/var/lib/postgresql/data # command: postgres -c log_statement=all for debugging volumes: - miniodata: - db-data: + db: + minio: