diff --git a/.env b/.env new file mode 100644 index 000000000..7cb97cea6 --- /dev/null +++ b/.env @@ -0,0 +1,25 @@ +# Application +APP_PORT=3000 + +# Cognito +COGNITO_AWS_REGION=eu-west-1 +COGNITO_CLIENT_ID=10h1tga4i933buv25lelalmtrn +COGNITO_POOL_ID=eu-west-1_dbfEb2FuH + +# S3 +S3_AWS_REGION=eu-west-1 +AWS_ACCESS_KEY_ID=minio +AWS_SECRET_ACCESS_KEY=minio123 +S3_BUCKET=ngmpub-userdata-local +PROJECTS_S3_BUCKET=ngmpub-project-files-local +S3_ENDPOINT=http://minio:9000 + +# Database +PGUSER=www-data +PGPASSWORD=www-data +PGHOST=db +PGPORT=5432 +PGDATABASE=swissgeol-local + +# sqlx +DATABASE_URL=postgres://${PGUSER}:${PGPASSWORD}@${PGHOST}:${PGPORT}/${PGDATABASE} diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml index 948ced943..010d9c62f 100644 --- a/.github/workflows/code-quality.yml +++ b/.github/workflows/code-quality.yml @@ -9,23 +9,21 @@ on: env: NODE_VERSION: "22.x" - DB_USERNAME: postgres - DB_PASSWORD: postgres - DB_DATABASE: postgres - DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgres?schema=public + RUST_VERSION: "1.73" jobs: -# add_review_links: -# runs-on: ubuntu-22.04 -# timeout-minutes: 3 -# steps: -# - uses: actions/checkout@v4 -# - name: Add review links -# env: -# GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} -# run: scripts/github_add_link_for_reviews.sh + # add_review_links: + # runs-on: ubuntu-22.04 + # timeout-minutes: 3 + # steps: + # - uses: actions/checkout@v4 + # - name: Add review links + # env: + # GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} + # run: scripts/github_add_link_for_reviews.sh dependency-review: + name: "Dependency Review" runs-on: ubuntu-latest steps: - name: Checkout repository @@ -33,8 +31,12 @@ jobs: - name: Review Dependencies uses: actions/dependency-review-action@v4 + install-ui: + name: "Install UI" runs-on: ubuntu-latest + needs: + - dependency-review steps: - name: Checkout repository uses: actions/checkout@v4 @@ -59,19 +61,15 @@ jobs: key: "${{ runner.os }}-node_modules-${{ env.NODE_VERSION }}-${{ hashFiles('./ui/package-lock.json') }}" restore-keys: | ${{ runner.os }}-node_modules- -# - name: Cache e2e node modules -# uses: actions/cache@v4 -# with: -# path: ./e2e/node_modules -# key: "${{ runner.os }}-node_modules_e2e-${{ env.NODE_VERSION }}-${{ hashFiles('./e2e/package-lock.json') }}" -# restore-keys: | -# ${{ runner.os }}-node_modules_e2e- - name: Install node dependencies run: cd ui && npm install - test-ui: + + check-ui: + name: "Check UI" runs-on: ubuntu-latest - needs: install-ui + needs: + - install-ui steps: - name: Checkout repository uses: actions/checkout@v4 @@ -84,12 +82,15 @@ jobs: with: path: ./ui/node_modules key: "${{ runner.os }}-node_modules-${{ env.NODE_VERSION }}-${{ hashFiles('./ui/package-lock.json') }}" - - name: Run tests - run: cd ui && npm run test + - name: Run build + run: cd ui && npm run check - lint-ui: + + test-ui: + name: "Test UI" runs-on: ubuntu-latest - needs: install-ui + needs: + - check-ui steps: - name: Checkout repository uses: actions/checkout@v4 @@ -102,23 +103,15 @@ jobs: with: path: ./ui/node_modules key: "${{ runner.os }}-node_modules-${{ env.NODE_VERSION }}-${{ hashFiles('./ui/package-lock.json') }}" - - name: Run lint - run: cd ui && npm run lint -# - name: Run prettier -# run: npx prettier --check . + - name: Run tests + run: cd ui && npm run test + - # It would be cleaner and probably more performant to replace this build step - # with either a non-emitting build or a simple type check. - # We only have `build` available for now, - # since the project is currently split across a multitude of small packages, - # all of which have to specify their own commands. - # (Daniel von Atzigen, 2024-04-12) - build-ui: + lint-ui: + name: "Lint UI" runs-on: ubuntu-latest needs: - - test-ui - - lint-ui - - dependency-review + - check-ui steps: - name: Checkout repository uses: actions/checkout@v4 @@ -131,293 +124,236 @@ jobs: with: path: ./ui/node_modules key: "${{ runner.os }}-node_modules-${{ env.NODE_VERSION }}-${{ hashFiles('./ui/package-lock.json') }}" -# - name: Reset nx -# run: npx nx reset - - name: Run build - run: cd ui && npm run build - + - name: Run lint + run: cd ui && npm run lint install-api: + name: "Install API" runs-on: ubuntu-latest + needs: + - dependency-review steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Install Rust toolchain uses: actions-rs/toolchain@v1 with: - toolchain: 1.73 + toolchain: ${{ env.RUST_VERSION }} profile: minimal override: true - - name: Cache cargo registry uses: actions/cache@v4 with: path: ~/.cargo/registry - key: cargo-registry-${{ runner.os }}-${{ hashFiles('./api/Cargo.lock') }} + key: "${{ runner.os }}-cargo_registry-${{ env.RUST_VERSION }}-${{ hashFiles('./api/Cargo.lock') }}" restore-keys: | - cargo-registry-${{ runner.os }}- - + ${{ runner.os }}-cargo_registry- - name: Cache cargo index uses: actions/cache@v4 with: path: ~/.cargo/git - key: cargo-index-${{ runner.os }}-${{ hashFiles('./api/Cargo.lock') }} + key: "${{ runner.os }}-cargo_index-${{ env.RUST_VERSION }}-${{ hashFiles('./api/Cargo.lock') }}" restore-keys: | - cargo-index-${{ runner.os }}- - + ${{ runner.os }}-cargo_index- - name: Cache cargo build uses: actions/cache@v4 with: - path: target - key: cargo-build-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }} + path: ./api/target + key: "${{ runner.os }}-cargo_build-${{ env.RUST_VERSION }}-${{ hashFiles('./api/Cargo.lock') }}" restore-keys: | - cargo-build-${{ runner.os }}- - + ${{ runner.os }}-cargo_build- - name: Cache sqlx binary uses: actions/cache@v4 with: path: ~/.cargo/bin/sqlx - key: sqlx-${{ runner.os }}-${{ hashFiles('./api/Cargo.lock') }} + key: "${{ runner.os }}-sqlx-${{ env.RUST_VERSION }}-${{ hashFiles('./api/Cargo.lock') }}" restore-keys: | - sqlx-${{ runner.os }}- - - - name: Install Rust dependencies - env: - DATABASE_URL: "postgres://www-data:www-data@localhost:15432/swissgeol-local" + ${{ runner.os }}-sqlx- + - name: Start DB + run: | + docker compose up --quiet-pull -d db + sleep 10 + - name: Setup DB run: | - docker compose up -d db - sleep 10 cd api - if [ ! -f ~/.cargo/bin/sqlx ]; then cargo install sqlx-cli --version 0.7.3 --no-default-features --features native-tls,postgres --locked --quiet - fi -# sqlx database create -# sqlx migrate run - - - name: Cache shared volume - uses: actions/cache@v4 + sqlx database create + sqlx migrate run + - name: Upload DB data + uses: actions/upload-artifact@v4 with: - path: /var/lib/postgresql/data - key: db-data-${{ runner.os }}-${{ hashFiles('./api/Cargo.lock') }} - restore-keys: | - db-data-${{ runner.os }}- + name: db + path: /var/lib/docker/volumes/swissgeol-viewer-app_db + - name: Install dependencies + run: | + cd api + rm -r src/* + echo "fn main() {}" > src/main.rs + cargo build --all-targets --locked --quiet + - name: Stop services + run: | + docker compose down - format-api: + check-api: + name: "Check API" runs-on: ubuntu-latest needs: - install-api steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Install Rust toolchain uses: actions-rs/toolchain@v1 with: - toolchain: 1.73 + toolchain: ${{ env.RUST_VERSION }} profile: minimal override: true - - - name: Restore cached cargo registry + - name: Restore cargo registry uses: actions/cache/restore@v4 with: path: ~/.cargo/registry - key: cargo-registry-${{ runner.os }}-${{ hashFiles('./api/Cargo.lock') }} - restore-keys: | - cargo-registry-${{ runner.os }}- - - - name: Restore cached cargo index + key: "${{ runner.os }}-cargo_registry-${{ env.RUST_VERSION }}-${{ hashFiles('./api/Cargo.lock') }}" + - name: Restore cargo index uses: actions/cache/restore@v4 with: path: ~/.cargo/git - key: cargo-index-${{ runner.os }}-${{ hashFiles('./api/Cargo.lock') }} - restore-keys: | - cargo-index-${{ runner.os }}- - - - name: Restore cached cargo build + key: "${{ runner.os }}-cargo_index-${{ env.RUST_VERSION }}-${{ hashFiles('./api/Cargo.lock') }}" + - name: Restore cargo build uses: actions/cache/restore@v4 with: - path: target - key: cargo-build-${{ runner.os }}-${{ hashFiles('./api/Cargo.lock') }} - restore-keys: | - cargo-build-${{ runner.os }}- - - - name: Restore cached sqlx binary - uses: actions/cache/restore@v4 - with: - path: ~/.cargo/bin/sqlx - key: sqlx-${{ runner.os }}-${{ hashFiles('./api/Cargo.lock') }} - restore-keys: | - sqlx-${{ runner.os }}- - -# - name: Install Rust dependencies -# env: -# DATABASE_URL: "postgres://www-data:www-data@localhost:15432/swissgeol-local" -# run: | -# docker compose up -d db -# sleep 10 -# cd api -# sqlx database create -# sqlx migrate run - - name: Run container - run: | - docker compose up -d api - sleep 10 - - name: Run fmt + path: ./api/target + key: "${{ runner.os }}-cargo_build-${{ env.RUST_VERSION }}-${{ hashFiles('/api/Cargo.lock') }}" + - name: Run check run: | - docker compose exec api rustup component add rustfmt - docker compose exec api cargo fmt - - name: Run clippy - run: | - docker compose exec api rustup component add clippy - docker compose exec api cargo clippy + cd api + cargo check --frozen - test-api: + lint-api: + name: "Lint API" runs-on: ubuntu-latest needs: - - install-api + - check-api steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Install Rust toolchain uses: actions-rs/toolchain@v1 with: - toolchain: 1.73 + toolchain: ${{ env.RUST_VERSION }} profile: minimal override: true - - - name: Restore cached cargo registry + - name: Restore cargo registry uses: actions/cache/restore@v4 with: path: ~/.cargo/registry - key: cargo-registry-${{ runner.os }}-${{ hashFiles('./api/Cargo.lock') }} - restore-keys: | - cargo-registry-${{ runner.os }}- - - - name: Restore cached cargo index + key: "${{ runner.os }}-cargo_registry-${{ env.RUST_VERSION }}-${{ hashFiles('./api/Cargo.lock') }}" + - name: Restore cargo index uses: actions/cache/restore@v4 with: path: ~/.cargo/git - key: cargo-index-${{ runner.os }}-${{ hashFiles('./api/Cargo.lock') }} - restore-keys: | - cargo-index-${{ runner.os }}- - - - name: Restore cached cargo build + key: "${{ runner.os }}-cargo_index-${{ env.RUST_VERSION }}-${{ hashFiles('./api/Cargo.lock') }}" + - name: Restore cargo build uses: actions/cache/restore@v4 with: - path: target - key: cargo-build-${{ runner.os }}-${{ hashFiles('./api/Cargo.lock') }} - restore-keys: | - cargo-build-${{ runner.os }}- - - - name: Restore cached sqlx binary - uses: actions/cache/restore@v4 - with: - path: ~/.cargo/bin/sqlx - key: sqlx-${{ runner.os }}-${{ hashFiles('./api/Cargo.lock') }} - restore-keys: | - sqlx-${{ runner.os }}- - -# - name: Install Rust dependencies -# env: -# DATABASE_URL: "postgres://www-data:www-data@localhost:15432/swissgeol-local" -# run: | -# docker compose up -d db -# sleep 10 -# cd api -# sqlx database create -# sqlx migrate run - - name: Run test + path: ./api/target + key: "${{ runner.os }}-cargo_build-${{ env.RUST_VERSION }}-${{ hashFiles('/api/Cargo.lock') }}" + - name: Run rustfmt + run: | + cd api + rustup component add rustfmt + cargo fmt --check + - name: Run clippy run: | - docker compose up -d api - sleep 10 cd api - docker compose exec api cargo test --offline + rustup component add clippy + cargo clippy --frozen + - build-api: + test-api: + name: "Test API" runs-on: ubuntu-latest needs: -# - test-api -# - format-api - - install-api - - dependency-review + - check-api steps: + - name: Checkout repository uses: actions/checkout@v4 - - name: Install Rust toolchain uses: actions-rs/toolchain@v1 with: - toolchain: 1.73 + toolchain: ${{ env.RUST_VERSION }} profile: minimal override: true - - - name: Restore cached cargo registry + - name: Restore cargo registry uses: actions/cache/restore@v4 with: path: ~/.cargo/registry - key: cargo-registry-${{ runner.os }}-${{ hashFiles('./api/Cargo.lock') }} - restore-keys: | - cargo-registry-${{ runner.os }}- - - - name: Restore cached cargo index + key: "${{ runner.os }}-cargo_registry-${{ env.RUST_VERSION }}-${{ hashFiles('./api/Cargo.lock') }}" + - name: Restore cargo index uses: actions/cache/restore@v4 with: path: ~/.cargo/git - key: cargo-index-${{ runner.os }}-${{ hashFiles('./api/Cargo.lock') }} - restore-keys: | - cargo-index-${{ runner.os }}- - - - name: Restore cached cargo build + key: "${{ runner.os }}-cargo_index-${{ env.RUST_VERSION }}-${{ hashFiles('./api/Cargo.lock') }}" + - name: Restore cargo build uses: actions/cache/restore@v4 with: - path: target - key: cargo-build-${{ runner.os }}-${{ hashFiles('./api/Cargo.lock') }} - restore-keys: | - cargo-build-${{ runner.os }}- - - - name: Restore cached sqlx binary + path: ./api/target + key: "${{ runner.os }}-cargo_build-${{ env.RUST_VERSION }}-${{ hashFiles('/api/Cargo.lock') }}" + - name: Restore sqlx binary uses: actions/cache/restore@v4 with: path: ~/.cargo/bin/sqlx - key: sqlx-${{ runner.os }}-${{ hashFiles('./api/Cargo.lock') }} - restore-keys: | - sqlx-${{ runner.os }}- + key: "${{ runner.os }}-sqlx-${{ env.RUST_VERSION }}-${{ hashFiles('./api/Cargo.lock') }}" + - name: Download DB data + uses: actions/download-artifact@v4 + with: + name: db + path: /var/lib/docker/volumes/${{ github.workspace }}_db + - name: Start DB + run: + docker compose up --quiet-pull -d db + sleep 10 + - name: Start API + run: + cd api + nohup cargo run --frozen > ../api.log 2>&1 & + echo $! > ../api.pid + - name: Run test + run: | + cd api + cargo test --frozen + - name: Stop services + run: | + kill $(cat api.pid) + docker compose down + - - name: Install Rust dependencies + cleanup: + name: "Cleanup" + runs-on: ubuntu-latest + needs: + - lint-ui + - test-ui + - lint-api + - test-api + if: always() + steps: + - name: Find DB artifact env: - DATABASE_URL: "postgres://www-data:www-data@localhost:15432/swissgeol-local" + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - docker compose up -d db - sleep 10 - cd api -# sqlx database create -# sqlx migrate run - - name: Build API + artifact_name="db" + artifacts=$(curl -s -H "Authorization: token $GITHUB_TOKEN" \ + "https://api.github.com/repos/${{ github.repository }}/actions/artifacts") + artifact_id=$(echo "$artifacts" | jq -r ".artifacts[] | select(.name==\"$artifact_name\") | .id") + echo "artifact_id=$artifact_id" >> $GITHUB_ENV + + - name: Delete DB artifact + if: ${{ env.artifact_id != '' }} env: - S3_AWS_REGION: eu-west-1 - AWS_ACCESS_KEY_ID: minio - AWS_SECRET_ACCESS_KEY: minio123 - S3_BUCKET: ngmpub-userdata-local - PROJECTS_S3_BUCKET: ngmpub-project-files-local - S3_ENDPOINT: http://minio:9000 - COGNITO_CLIENT_ID: 10h1tga4i933buv25lelalmtrn - COGNITO_POOL_ID: eu-west-1_dbfEb2FuH - PGUSER: www-data - PGPASSWORD: www-data - PGDATABASE: swissgeol-local - PGHOST: db - PGPORT: "5432" - APP_PORT: "3000" - SQLX_OFFLINE: "true" - DATABASE_URL: "postgres://www-data:www-data@db:5432/swissgeol-local" + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - apt update && apt install -y musl-tools musl-dev musl-gcc - rustup target add x86_64-unknown-linux-musl - cd api - cargo build --target x86_64-unknown-linux-musl - - + curl -X DELETE -H "Authorization: token $GITHUB_TOKEN" \ + "https://api.github.com/repos/${{ github.repository }}/actions/artifacts/${{ env.artifact_id }}" diff --git a/.github/workflows/publish-edge.yml b/.github/workflows/publish-edge.yml index 2ccd09778..58acb9797 100644 --- a/.github/workflows/publish-edge.yml +++ b/.github/workflows/publish-edge.yml @@ -4,8 +4,6 @@ on: push: branches: - "develop" - - "**" - - "!main" workflow_dispatch: inputs: diff --git a/.gitignore b/.gitignore index fa8b65b29..e883fe81f 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ /ui/types /docs /secrets.txt + +/.idea/ diff --git a/api/.env b/api/.env index 8dc3d9d6b..c7e39898a 100644 --- a/api/.env +++ b/api/.env @@ -8,11 +8,11 @@ COGNITO_POOL_ID=eu-west-1_dbfEb2FuH # S3 S3_AWS_REGION=eu-west-1 -AWS_ACCESS_KEY_ID=bla -AWS_SECRET_ACCESS_KEY=hu +AWS_ACCESS_KEY_ID=minio +AWS_SECRET_ACCESS_KEY=minio123 S3_BUCKET=ngmpub-userdata-local PROJECTS_S3_BUCKET=ngmpub-project-files-local -S3_ENDPOINT="http://minio:9000" +S3_ENDPOINT=http://minio:9000 # Database PGUSER=www-data @@ -21,5 +21,5 @@ PGHOST=localhost PGPORT=15432 PGDATABASE=swissgeol-local -# Required for using sqlx & tracing macros -DATABASE_URL="postgres://www-data:www-data@localhost:15432/swissgeol-local" +# sqlx +DATABASE_URL=postgres://${PGUSER}:${PGPASSWORD}@${PGHOST}:${PGPORT}/${PGDATABASE} diff --git a/docker-compose.yaml b/docker-compose.yaml index c526b861b..2c8f972ca 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -7,28 +7,9 @@ services: dockerfile: DockerfileDev platform: linux/amd64 ports: - - 8480:3000 + - "8480:3000" environment: - # S3 - S3_AWS_REGION: &minio_region eu-west-1 - AWS_ACCESS_KEY_ID: &minio_user minio - AWS_SECRET_ACCESS_KEY: &minio_pass minio123 - S3_BUCKET: ngmpub-userdata-local - PROJECTS_S3_BUCKET: ngmpub-project-files-local - S3_ENDPOINT: http://minio:9000 - # Cognito - COGNITO_CLIENT_ID: 10h1tga4i933buv25lelalmtrn - COGNITO_POOL_ID: eu-west-1_dbfEb2FuH - # Postgres - PGUSER: &db_user www-data - PGPASSWORD: &db_pass www-data - PGDATABASE: &db_name swissgeol-local - PGHOST: db - PGPORT: "5432" - # Api - APP_PORT: "3000" - SQLX_OFFLINE: "true" - DATABASE_URL: "postgres://www-data:www-data@db:5432/swissgeol-local" + SQLX_OFFLINE: true command: ["cargo", "watch", "--poll", "--shell", "cargo run --offline --target x86_64-unknown-linux-musl"] volumes: - ./api/src:/app/src:ro @@ -37,8 +18,6 @@ services: - ./api/Cargo.toml:/app/Cargo.toml:ro links: - db -# env_file: -# - ./api/.env ui: image: node:lts @@ -59,15 +38,15 @@ services: image: minio/minio:latest command: server /data --console-address :9001 volumes: - - miniodata:/data + - minio:/data ports: - "9000:9000" - "9001:9001" environment: MINIO_BROWSER: 'on' - MINIO_SITE_REGION: *minio_region - MINIO_ROOT_USER: *minio_user - MINIO_ROOT_PASSWORD: *minio_pass + MINIO_SITE_REGION: ${S3_AWS_REGION} + MINIO_ROOT_USER: ${AWS_ACCESS_KEY_ID} + MINIO_ROOT_PASSWORD: ${AWS_SECRET_ACCESS_KEY} abbreviator: image: ghcr.io/swisstopo/swissgeol-viewer-app-abbreviator:main @@ -83,15 +62,15 @@ services: image: camptocamp/postgres:14-postgis-3 platform: linux/amd64 environment: - POSTGRES_USER: *db_user - POSTGRES_PASSWORD: *db_pass - POSTGRES_DB: *db_name + POSTGRES_USER: ${PGUSER} + POSTGRES_PASSWORD: ${PGPASSWORD} + POSTGRES_DB: ${PGDATABASE} ports: - - 15432:5432 + - "15432:5432" volumes: - - db-data:/var/lib/postgresql/data + - db:/var/lib/postgresql/data # command: postgres -c log_statement=all for debugging volumes: - miniodata: - db-data: + db: + minio: diff --git a/ui/Dockerfile b/ui/Dockerfile new file mode 100644 index 000000000..9cb0f0cf3 --- /dev/null +++ b/ui/Dockerfile @@ -0,0 +1,26 @@ +FROM node:22-alpine as app-builder + +ENV CYPRESS_INSTALL_BINARY=0 + +RUN apk add python3 make gcc g++ + +WORKDIR /app +COPY . . + +RUN npm install --no-scripts +RUN npm run build + + + +FROM nginx:mainline-alpine + +WORKDIR /usr/share/nginx/html +COPY --from=app-builder /app/dist/apps/ngm . + +# this nginx base image will parse the template and will move it to +# /etc/nginx/conf.d/default.conf before it starts nginx process +COPY container/nginx.conf /etc/nginx/templates/default.conf.template + +HEALTHCHECK --interval=10s --timeout=5s --start-period=10s --retries=3 CMD wget localhost -q -O - > /dev/null 2>&1 + +EXPOSE 80