From a4947bbb6a8dcd7b65e0f56aab8fc2ad6238d48c Mon Sep 17 00:00:00 2001 From: Daniel von Atzigen Date: Wed, 6 Nov 2024 10:12:31 +0100 Subject: [PATCH 1/6] wip: Fix docker compose --- .env | 23 ++++++-------------- api/.env | 33 +++++++++++++++++----------- api/DockerfileDev | 20 ++++------------- api/DockerfileDevMac | 23 -------------------- api/start.sh | 11 ++++++++++ docker-compose.yaml | 51 +++++++++++++++++++++++++------------------- 6 files changed, 70 insertions(+), 91 deletions(-) delete mode 100644 api/DockerfileDevMac create mode 100644 api/start.sh diff --git a/.env b/.env index 7cb97cea6..82cf0a6d9 100644 --- a/.env +++ b/.env @@ -1,19 +1,3 @@ -# Application -APP_PORT=3000 - -# Cognito -COGNITO_AWS_REGION=eu-west-1 -COGNITO_CLIENT_ID=10h1tga4i933buv25lelalmtrn -COGNITO_POOL_ID=eu-west-1_dbfEb2FuH - -# S3 -S3_AWS_REGION=eu-west-1 -AWS_ACCESS_KEY_ID=minio -AWS_SECRET_ACCESS_KEY=minio123 -S3_BUCKET=ngmpub-userdata-local -PROJECTS_S3_BUCKET=ngmpub-project-files-local -S3_ENDPOINT=http://minio:9000 - # Database PGUSER=www-data PGPASSWORD=www-data @@ -21,5 +5,10 @@ PGHOST=db PGPORT=5432 PGDATABASE=swissgeol-local -# sqlx +# SQLx DATABASE_URL=postgres://${PGUSER}:${PGPASSWORD}@${PGHOST}:${PGPORT}/${PGDATABASE} + +# S3 +S3_AWS_REGION=eu-west-1 +AWS_ACCESS_KEY_ID=minio +AWS_SECRET_ACCESS_KEY=minio123 \ No newline at end of file diff --git a/api/.env b/api/.env index c7e39898a..e2525a40c 100644 --- a/api/.env +++ b/api/.env @@ -1,10 +1,22 @@ +# This is the base configuration for the API. +# It is configured so that the API can be run on the Docker host machine, +# while still being able to access the Docker services (e.g. the database). +# +# To instead run the API itself as a Docker service, use this configuration, +# but load `/.env` right after it. + # Application APP_PORT=3000 -# Cognito -COGNITO_AWS_REGION=eu-west-1 -COGNITO_CLIENT_ID=10h1tga4i933buv25lelalmtrn -COGNITO_POOL_ID=eu-west-1_dbfEb2FuH +# Database +PGUSER=www-data +PGPASSWORD=www-data +PGHOST=localhost +PGPORT=5432 +PGDATABASE=swissgeol-local + +# SQLx +DATABASE_URL=postgres://${PGUSER}:${PGPASSWORD}@${PGHOST}:${PGPORT}/${PGDATABASE} # S3 S3_AWS_REGION=eu-west-1 @@ -14,12 +26,7 @@ S3_BUCKET=ngmpub-userdata-local PROJECTS_S3_BUCKET=ngmpub-project-files-local S3_ENDPOINT=http://minio:9000 -# Database -PGUSER=www-data -PGPASSWORD=www-data -PGHOST=localhost -PGPORT=15432 -PGDATABASE=swissgeol-local - -# sqlx -DATABASE_URL=postgres://${PGUSER}:${PGPASSWORD}@${PGHOST}:${PGPORT}/${PGDATABASE} +# Cognito +COGNITO_AWS_REGION=eu-west-1 +COGNITO_CLIENT_ID=10h1tga4i933buv25lelalmtrn +COGNITO_POOL_ID=eu-west-1_dbfEb2FuH diff --git a/api/DockerfileDev b/api/DockerfileDev index 5dc3fca6e..01291be43 100644 --- a/api/DockerfileDev +++ b/api/DockerfileDev @@ -3,22 +3,10 @@ FROM rust:1.73 -RUN rustup target add x86_64-unknown-linux-musl -RUN apt update && apt install -y gnupg2 wget lsb-release -RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - -RUN sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' -RUN apt update && apt install -y musl-tools musl-dev postgresql-client-13 -RUN curl -L https://github.com/watchexec/cargo-watch/releases/download/v8.1.2/cargo-watch-v8.1.2-x86_64-unknown-linux-gnu.tar.xz -o cargo-watch.tar.xz -RUN tar xvf cargo-watch.tar.xz -RUN cp cargo-watch-v8.1.2-x86_64-unknown-linux-gnu/cargo-watch /usr/bin - WORKDIR /app -# First we handle fetching and building our dependencies -# We need a stub entry point for that -COPY Cargo.toml Cargo.lock ./ -RUN mkdir src; echo "fn main() {}" > src/main.rs -RUN cargo build --target x86_64-unknown-linux-musl +RUN cargo install cargo-watch --version 7.8.1 + +COPY . . -# Now we copy all files, building will be handled on run -COPY ./ . +CMD ["/app/start.sh"] \ No newline at end of file diff --git a/api/DockerfileDevMac b/api/DockerfileDevMac deleted file mode 100644 index ebe1c81ad..000000000 --- a/api/DockerfileDevMac +++ /dev/null @@ -1,23 +0,0 @@ -## Use a base image that supports Rust and cross-compilation for aarch64 (less configuration then default image) -FROM messense/rust-musl-cross:x86_64-musl as builder - -RUN rustup target add x86_64-unknown-linux-musl -RUN apt update && apt install -y gnupg2 wget lsb-release -RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - -RUN sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' - -RUN apt update && apt install -y musl-tools musl-dev postgresql-client-13 -RUN curl -L https://github.com/watchexec/cargo-watch/releases/download/v8.1.2/cargo-watch-v8.1.2-x86_64-unknown-linux-gnu.tar.xz -o cargo-watch.tar.xz -RUN tar xvf cargo-watch.tar.xz -RUN cp cargo-watch-v8.1.2-x86_64-unknown-linux-gnu/cargo-watch /usr/bin - -WORKDIR /app -# -## First we handle fetching and building our dependencies -## We need a stub entry point for that -COPY Cargo.toml Cargo.lock ./ -RUN mkdir src; echo "fn main() {}" > src/main.rs -RUN cargo build --target x86_64-unknown-linux-musl - -## Now we copy all files, building will be handled on run -COPY ./ . \ No newline at end of file diff --git a/api/start.sh b/api/start.sh new file mode 100644 index 000000000..6b1808863 --- /dev/null +++ b/api/start.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +SQLX_VERSION=0.7.3 +if [[ ! -f ~/.cargo/bin/sqlx ]] || [[ $(sqlx --version) != "sqlx-cli $SQLX_VERSION" ]]; then + cargo install sqlx-cli --version $SQLX_VERSION --no-default-features --features native-tls,postgres --locked +fi + +sqlx database create +sqlx migrate run + +cargo watch --poll --shell "cargo run" \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index 538235024..724939127 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,52 +1,56 @@ services: api: - init: true # handle kill signals for rust - image: local_api:latest + image: swissgeol-viewer-app/api:local build: context: ./api dockerfile: DockerfileDev - platform: linux/amd64 + init: true ports: - "8480:3000" - environment: - SQLX_OFFLINE: true - command: ["cargo", "watch", "--poll", "--shell", "cargo run --offline --target x86_64-unknown-linux-musl"] volumes: - - ./api/src:/app/src:ro - - ./api/migrations:/app/migrations:ro - - ./api/.sqlx:/app/.sqlx:ro - - ./api/Cargo.toml:/app/Cargo.toml:ro + - ./api:/app + - api.cargo:/usr/local/cargo + - api.target:/app/target depends_on: - db + - minio + env_file: + - ./api/.env + - .env ui: - image: node:lts + image: node:22 + init: true ports: - - 8000:8000 + - "8000:8000" environment: - ENVIRONMENT_NAME: local-dev + NODE_ENV: development working_dir: /app - command: ['node_modules/.bin/webpack', 'serve'] + command: ["sh", "-c", "npm install && npm run start"] depends_on: - api - abbreviator volumes: - - "./ui:/app:ro" + - ./ui:/app + - ui.node_modules:/app/node_modules minio: - init: true # handle kill signals image: minio/minio:latest command: server /data --console-address :9001 + init: true volumes: - minio:/data ports: - "9000:9000" - "9001:9001" environment: - MINIO_BROWSER: 'on' + MINIO_BROWSER: "on" MINIO_SITE_REGION: ${S3_AWS_REGION} MINIO_ROOT_USER: ${AWS_ACCESS_KEY_ID} MINIO_ROOT_PASSWORD: ${AWS_SECRET_ACCESS_KEY} + env_file: + - ./api/.env + - .env abbreviator: image: ghcr.io/swisstopo/swissgeol-viewer-app-abbreviator:main @@ -61,20 +65,23 @@ services: db: image: camptocamp/postgres:14-postgis-3 platform: linux/amd64 + ports: + - "5432:5432" environment: POSTGRES_USER: ${PGUSER} POSTGRES_PASSWORD: ${PGPASSWORD} POSTGRES_DB: ${PGDATABASE} - ports: - - "15432:5432" volumes: - - ./volumes/db/:/var/lib/postgresql/data + - db:/var/lib/postgresql/data healthcheck: - test: ["CMD", "pg_isready"] + test: ["CMD", "pg_isready", "-U", "${PGUSER}", "${PGDATABASE}"] interval: 10s timeout: 5s retries: 5 - # command: postgres -c log_statement=all for debugging volumes: minio: + db: + api.cargo: + api.target: + ui.node_modules: \ No newline at end of file From 5a6cb13569f5d1ce76a71fb280ead9ec2705c37b Mon Sep 17 00:00:00 2001 From: Daniel von Atzigen Date: Wed, 6 Nov 2024 11:07:06 +0100 Subject: [PATCH 2/6] Replace `ui/scripts/create_versions.sh` with js script --- ui/package.json | 2 +- ui/scripts/create_versions.sh | 10 ---------- ui/scripts/extract_versions.js | 30 ++++++++++++++++++++++++++++++ 3 files changed, 31 insertions(+), 11 deletions(-) delete mode 100755 ui/scripts/create_versions.sh create mode 100644 ui/scripts/extract_versions.js diff --git a/ui/package.json b/ui/package.json index ffbe9e770..03279faa5 100644 --- a/ui/package.json +++ b/ui/package.json @@ -9,7 +9,7 @@ "start": "npm run webpack-start", "build-static": "node scripts/environment.js $RELEASE_NAME && node scripts/generateManuals.js && bash scripts/legal_md_to_html.sh", "webpack-start": "npm run build-static && webpack serve", - "build": "rm -rf dist/* && npm run build-static && rollup --environment mode:production -c && bash scripts/create_versions.sh; find dist -type l -exec rm {} +", + "build": "rm -rf dist/* && npm run build-static && rollup --environment mode:production -c && node scripts/extract_versions.js; find dist -type l -exec rm {} +", "check": "npm run check-main && npm run check-ut", "check-main": "tsc src/index.ts --noEmit --allowJs --skipLibCheck --target es2020 --moduleResolution node", "check-ut": "tsc src/test/*.ts src/test/*.js --noEmit --allowJs --skipLibCheck --esModuleInterop true --target es2020 --moduleResolution node", diff --git a/ui/scripts/create_versions.sh b/ui/scripts/create_versions.sh deleted file mode 100755 index 828bd4657..000000000 --- a/ui/scripts/create_versions.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -e - -CSV="`jq '.version' node_modules/cesium/package.json`" -cat > dist/versions.json < Date: Wed, 6 Nov 2024 11:28:15 +0100 Subject: [PATCH 3/6] Replace `ui/scripts/legal_md_to_html.sh` with js script --- ui/package.json | 2 +- ui/scripts/legal_md_to_html.js | 30 ++++++++++++++++++++++++++++++ ui/scripts/legal_md_to_html.sh | 11 ----------- 3 files changed, 31 insertions(+), 12 deletions(-) create mode 100644 ui/scripts/legal_md_to_html.js delete mode 100755 ui/scripts/legal_md_to_html.sh diff --git a/ui/package.json b/ui/package.json index 03279faa5..ea7864586 100644 --- a/ui/package.json +++ b/ui/package.json @@ -7,7 +7,7 @@ "extract-from-assets": "node scripts/get_assets_properties.js src/assets_properties.js", "extract-i18n": "i18next -c i18next-parser.config.cjs", "start": "npm run webpack-start", - "build-static": "node scripts/environment.js $RELEASE_NAME && node scripts/generateManuals.js && bash scripts/legal_md_to_html.sh", + "build-static": "node scripts/environment.js $RELEASE_NAME && node scripts/generateManuals.js && node scripts/legal_md_to_html.js", "webpack-start": "npm run build-static && webpack serve", "build": "rm -rf dist/* && npm run build-static && rollup --environment mode:production -c && node scripts/extract_versions.js; find dist -type l -exec rm {} +", "check": "npm run check-main && npm run check-ut", diff --git a/ui/scripts/legal_md_to_html.js b/ui/scripts/legal_md_to_html.js new file mode 100644 index 000000000..5d7959f40 --- /dev/null +++ b/ui/scripts/legal_md_to_html.js @@ -0,0 +1,30 @@ +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import {marked} from 'marked'; + +const legalDir = path.resolve(import.meta.dirname, '../legal/'); +const legalDistDir = path.resolve(import.meta.dirname, '../dist/legal/'); +fs.mkdirSync(legalDistDir, {recursive: true}); +fs.copyFileSync( + path.resolve(legalDir, 'index.css'), + path.resolve(legalDistDir, 'index.css'), +); + +const template = fs.readFileSync(path.resolve(legalDir, 'template.html'), 'utf-8'); + +const files = await fs.promises.readdir(legalDir, {withFileTypes: true}); +for (const file of files) { + if (!file.isFile()) { + continue; + } + if (!file.name.endsWith('.md')) { + continue; + } + + const fileMarkdown = fs.readFileSync(path.resolve(file.parentPath, file.name), 'utf-8'); + const fileHtml = marked(fileMarkdown); + const legalHtml = template.replace('_CONTENT_', fileHtml); + + const fileNameHtml = `${file.name.slice(0, -3)}.html`; + fs.writeFileSync(path.resolve(legalDistDir, fileNameHtml), legalHtml); +} diff --git a/ui/scripts/legal_md_to_html.sh b/ui/scripts/legal_md_to_html.sh deleted file mode 100755 index 0d87a4302..000000000 --- a/ui/scripts/legal_md_to_html.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -e - -mkdir -p dist/legal/ -cp legal/index.css dist/legal/ - -template=$( dist/${file/.md/.html} -done From 7df6bcee624273393fb9ae0d6893801274d433b8 Mon Sep 17 00:00:00 2001 From: till_schuetze Date: Tue, 12 Nov 2024 08:25:31 +0100 Subject: [PATCH 4/6] enable local development inside docker container --- api/.env | 2 +- api/DockerfileDev | 4 +++- docker-compose.yaml | 2 +- ui/package.json | 2 +- ui/scripts/environment.js | 9 ++++++++- ui/webpack.config.js | 1 + 6 files changed, 15 insertions(+), 5 deletions(-) diff --git a/api/.env b/api/.env index e2525a40c..3175b3888 100644 --- a/api/.env +++ b/api/.env @@ -11,7 +11,7 @@ APP_PORT=3000 # Database PGUSER=www-data PGPASSWORD=www-data -PGHOST=localhost +PGHOST=db PGPORT=5432 PGDATABASE=swissgeol-local diff --git a/api/DockerfileDev b/api/DockerfileDev index 01291be43..7ad2f70fe 100644 --- a/api/DockerfileDev +++ b/api/DockerfileDev @@ -9,4 +9,6 @@ RUN cargo install cargo-watch --version 7.8.1 COPY . . -CMD ["/app/start.sh"] \ No newline at end of file +RUN chmod +x start.sh + +CMD ["./start.sh"] \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index 724939127..cdf40843c 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -26,7 +26,7 @@ services: environment: NODE_ENV: development working_dir: /app - command: ["sh", "-c", "npm install && npm run start"] + command: ["sh", "-c", "npm ci && npm run start"] depends_on: - api - abbreviator diff --git a/ui/package.json b/ui/package.json index ea7864586..8222ce308 100644 --- a/ui/package.json +++ b/ui/package.json @@ -8,7 +8,7 @@ "extract-i18n": "i18next -c i18next-parser.config.cjs", "start": "npm run webpack-start", "build-static": "node scripts/environment.js $RELEASE_NAME && node scripts/generateManuals.js && node scripts/legal_md_to_html.js", - "webpack-start": "npm run build-static && webpack serve", + "webpack-start": "npm run build-static && webpack serve --config webpack.config.js", "build": "rm -rf dist/* && npm run build-static && rollup --environment mode:production -c && node scripts/extract_versions.js; find dist -type l -exec rm {} +", "check": "npm run check-main && npm run check-ut", "check-main": "tsc src/index.ts --noEmit --allowJs --skipLibCheck --target es2020 --moduleResolution node", diff --git a/ui/scripts/environment.js b/ui/scripts/environment.js index 28d6f7de4..891996a81 100644 --- a/ui/scripts/environment.js +++ b/ui/scripts/environment.js @@ -13,7 +13,14 @@ const execPromise = promisify(exec); const filename = 'environment'; if (!branch) { - branch = (await execPromise('git rev-parse --abbrev-ref HEAD')).stdout.toString().trim(); + try { + // Check if the current directory is a Git repository + await execPromise('git rev-parse --is-inside-work-tree'); + branch = (await execPromise('git rev-parse --abbrev-ref HEAD')).stdout.toString().trim(); + } catch (gitError) { + console.warn('Not a git repository. Using default branch name.'); + branch = 'default-branch'; + } } const jsonContent = `{'branch': '${branch}'}`; diff --git a/ui/webpack.config.js b/ui/webpack.config.js index a54eb7b94..4a41584e7 100644 --- a/ui/webpack.config.js +++ b/ui/webpack.config.js @@ -128,6 +128,7 @@ export default { ], compress: true, port: 8000, + hot: true, }, plugins: [ From c7cae8118541533a93b57818ae1d9f1b9721ac5b Mon Sep 17 00:00:00 2001 From: till_schuetze Date: Wed, 13 Nov 2024 11:34:37 +0100 Subject: [PATCH 5/6] fix line endings and add comment --- api/DockerfileDev | 1 + 1 file changed, 1 insertion(+) diff --git a/api/DockerfileDev b/api/DockerfileDev index 7ad2f70fe..46f5dc363 100644 --- a/api/DockerfileDev +++ b/api/DockerfileDev @@ -11,4 +11,5 @@ COPY . . RUN chmod +x start.sh +# Make sure the start.sh script has LF line endings! CMD ["./start.sh"] \ No newline at end of file From 5421a263ef249c95ed0e461e64b09a7dbc95afca Mon Sep 17 00:00:00 2001 From: till_schuetze Date: Thu, 14 Nov 2024 09:17:49 +0100 Subject: [PATCH 6/6] Set pghost to localhost in pipeline --- .github/workflows/code-quality.yml | 1 + docker-compose.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml index 0825a6ca3..d7ff5d52f 100644 --- a/.github/workflows/code-quality.yml +++ b/.github/workflows/code-quality.yml @@ -12,6 +12,7 @@ env: RUST_VERSION: "1.73" SQLX_OFFLINE: true SQLX_VERSION: 0.7.3 + PGHOST: localhost # Overrides the default value in .env jobs: # add_review_links: diff --git a/docker-compose.yaml b/docker-compose.yaml index cdf40843c..1ac01c435 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -74,7 +74,7 @@ services: volumes: - db:/var/lib/postgresql/data healthcheck: - test: ["CMD", "pg_isready", "-U", "${PGUSER}", "${PGDATABASE}"] + test: ["CMD-SHELL", "pg_isready -U ${PGUSER} -d ${PGDATABASE}"] interval: 10s timeout: 5s retries: 5