From c04a98d9d4a2bfcdf7a897be7f8bca87a9b6fe19 Mon Sep 17 00:00:00 2001 From: Anass Bouassaba Date: Thu, 5 Dec 2024 08:50:56 +0100 Subject: [PATCH] chore(misc): switch to PostgreSQL (#421) --- .env | 6 +++-- DEVELOPMENT.md | 16 ++++-------- api/.env | 2 +- console/README.md | 6 ----- docker-compose.yml | 58 +++++++++++++++++++++++--------------------- idp/.env | 2 +- language/models.yaml | 10 ++++---- migrations/README.md | 4 +-- 8 files changed, 48 insertions(+), 56 deletions(-) diff --git a/.env b/.env index 2720910dc..89ed5978b 100644 --- a/.env +++ b/.env @@ -1,6 +1,9 @@ # Docker Config COMPOSE_PROJECT_NAME="voltaserve" +# Version +VOLTASERVE_VERSION="main" + # Hostname VOLTASERVE_HOSTNAME="localhost" @@ -15,8 +18,7 @@ VOLTASERVE_MOSAIC_PORT=8085 VOLTASERVE_CONSOLE_PORT=8086 # Infrastructure Ports -VOLTASERVE_POSTGRES_PORT=26257 -VOLTASERVE_COCKROACH_CONSOLE_PORT=18080 +VOLTASERVE_POSTGRES_PORT=5432 VOLTASERVE_MINIO_PORT=9000 VOLTASERVE_MINIO_CONSOLE_PORT=9001 VOLTASERVE_REDISINSIGHT_PORT=8001 diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 28800ab42..88ca987bd 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -18,7 +18,7 @@ Install [Python](https://www.python.org) 3.12 with [pyenv](https://github.com/py ```shell docker compose up -d \ - cockroach \ + postgres \ minio \ meilisearch \ redis \ @@ -29,22 +29,16 @@ Run the [migrations/migrate]() tool in the newly created database. ### From Binaries -#### CockroachDB +#### PostgreSQL -Download the [binary archive](https://www.cockroachlabs.com/docs/releases) and extract the archive. - -Start CockroachDB: - -```shell -./cockroach start-single-node --insecure --http-addr=0.0.0.0:18080 -``` +Download and run [PostgreSQL](https://www.postgresql.org). Using DBeaver or any PostgreSQL GUI, connect with `root` and no password, then create a user and database: ```sql CREATE DATABASE voltaserve; -CREATE USER voltaserve; -GRANT ALL PRIVILEGES ON DATABASE voltaserve TO voltaserve; +CREATE USER voltaserve WITH PASSWORD 'voltaserve'; +ALTER DATABASE voltaserve OWNER TO voltaserve; ``` Run the [migrations/migrate]() tool in the newly created database. diff --git a/api/.env b/api/.env index 811ba03a1..a2cd8e4b9 100644 --- a/api/.env +++ b/api/.env @@ -5,7 +5,7 @@ PUBLIC_UI_URL="http://127.0.0.1:3000" CONVERSION_URL="http://127.0.0.1:8083" LANGUAGE_URL="http://127.0.0.1:8084" MOSAIC_URL="http://127.0.0.1:8085" -POSTGRES_URL="postgresql://voltaserve@127.0.0.1:26257/voltaserve" +POSTGRES_URL="postgresql://voltaserve:voltaserve@127.0.0.1:5432/voltaserve" # Security SECURITY_JWT_SIGNING_KEY="586cozl1x9m6zmu4fg8iwi6ajazguehcm9qdfgd5ndo2pc3pcn" diff --git a/console/README.md b/console/README.md index 3650dc569..6403903ec 100644 --- a/console/README.md +++ b/console/README.md @@ -20,12 +20,6 @@ Add `--reload` flag for development: poetry run python -m api.uvi --reload ``` -To use CockroachDB, add the `POSTGRES_PORT` environment variable: - -```shell -POSTGRES_PORT=26257 poetry run python -m api.uvi --reload -``` - Lint code: ```shell diff --git a/docker-compose.yml b/docker-compose.yml index d99f720ad..0743cd2af 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,25 +9,24 @@ # AGPL-3.0-only in the root of this repository. volumes: - cockroach: + postgres: minio: meilisearch: redis: services: - cockroach: - image: cockroachdb/cockroach:latest-v24.2 + postgres: + image: postgres:17.2 ports: - - ${VOLTASERVE_POSTGRES_PORT}:26257 - - ${VOLTASERVE_COCKROACH_CONSOLE_PORT}:8080 + - ${VOLTASERVE_POSTGRES_PORT}:5432 environment: - COCKROACH_DATABASE: voltaserve - COCKROACH_USER: voltaserve + POSTGRES_USER: voltaserve + POSTGRES_DB: voltaserve + POSTGRES_PASSWORD: voltaserve volumes: - - cockroach:/cockroach/cockroach-data - command: start-single-node --insecure + - postgres:/var/lib/postgresql/data healthcheck: - test: cockroach sql --insecure --execute='SELECT 1;' || exit 1 + test: pg_isready -U postgres || exit 1 minio: image: minio/minio:RELEASE.2024-09-09T16-59-28Z ports: @@ -40,6 +39,8 @@ services: volumes: - minio:/data command: server /data --console-address ":9001" + healthcheck: + test: curl --fail http://127.0.0.1:9000/minio/health/live || exit 1 meilisearch: image: getmeili/meilisearch:v1.11.3 ports: @@ -64,7 +65,7 @@ services: healthcheck: test: wget --quiet --spider http://127.0.0.1:1080 || exit 1 api: - image: voltaserve/api + image: voltaserve/api:${VOLTASERVE_VERSION} build: context: ./api ports: @@ -74,7 +75,7 @@ services: - CONVERSION_URL=http://conversion:8083 - LANGUAGE_URL=http://language:8084 - MOSAIC_URL=http://mosaic:8085 - - POSTGRES_URL=postgresql://voltaserve@cockroach:26257/voltaserve + - POSTGRES_URL=postgresql://voltaserve:voltaserve@postgres:5432/voltaserve - S3_URL=minio:9000 - SEARCH_URL=http://meilisearch:7700 - PUBLIC_UI_URL=http://${VOLTASERVE_HOSTNAME}:${VOLTASERVE_UI_PORT} @@ -89,20 +90,20 @@ services: healthcheck: test: wget --quiet --spider http://127.0.0.1:8080/v3/health || exit 1 depends_on: - - cockroach + - postgres - redis - minio - meilisearch restart: on-failure idp: - image: voltaserve/idp + image: voltaserve/idp:${VOLTASERVE_VERSION} build: context: ./idp ports: - ${VOLTASERVE_IDP_PORT}:8081 environment: - PORT=8081 - - POSTGRES_URL=postgresql://voltaserve@cockroach:26257/voltaserve + - POSTGRES_URL=postgresql://voltaserve:voltaserve@postgres:5432/voltaserve - SEARCH_URL=http://meilisearch:7700 - PUBLIC_UI_URL=http://${VOLTASERVE_HOSTNAME}:${VOLTASERVE_UI_PORT} - SMTP_HOST=${VOLTASERVE_SMTP_HOST} @@ -115,12 +116,12 @@ services: healthcheck: test: wget --quiet --spider http://127.0.0.1:8081/v3/health || exit 1 depends_on: - - cockroach + - postgres - meilisearch - minio restart: on-failure ui: - image: voltaserve/ui + image: voltaserve/ui:${VOLTASERVE_VERSION} build: context: ./ui ports: @@ -135,7 +136,7 @@ services: - api restart: on-failure webdav: - image: voltaserve/webdav + image: voltaserve/webdav:${VOLTASERVE_VERSION} build: context: ./webdav ports: @@ -153,7 +154,7 @@ services: - api restart: on-failure conversion: - image: voltaserve/conversion + image: voltaserve/conversion:${VOLTASERVE_VERSION} build: context: ./conversion ports: @@ -172,7 +173,7 @@ services: - minio restart: on-failure language: - image: voltaserve/language + image: voltaserve/language:${VOLTASERVE_VERSION} build: context: ./language ports: @@ -181,7 +182,7 @@ services: test: wget --quiet --spider http://127.0.0.1:8084/v3/health || exit 1 restart: on-failure mosaic: - image: voltaserve/mosaic + image: voltaserve/mosaic:${VOLTASERVE_VERSION} build: context: ./mosaic ports: @@ -192,7 +193,7 @@ services: test: wget --quiet --spider http://127.0.0.1:8085/v3/health || exit 1 restart: on-failure console: - image: voltaserve/console + image: voltaserve/console:${VOLTASERVE_VERSION} build: context: ./console dockerfile: Dockerfile @@ -201,10 +202,11 @@ services: environment: - PORT=8086 - HOST=0.0.0.0 - - POSTGRES_URL=cockroach - - POSTGRES_PORT=26257 + - POSTGRES_URL=postgres + - POSTGRES_PORT=5432 - POSTGRES_NAME=voltaserve - POSTGRES_USER=voltaserve + - POSTGRES_PASSWORD=voltaserve - WORKERS=4 - SECURITY_JWT_SIGNING_KEY=586cozl1x9m6zmu4fg8iwi6ajazguehcm9qdfgd5ndo2pc3pcn - SECURITY_CORS_ORIGINS=http://localhost:3000 @@ -225,14 +227,14 @@ services: - conversion - language - mosaic - - cockroach + - postgres restart: on-failure migrations: - image: voltaserve/migrations + image: voltaserve/migrations:${VOLTASERVE_VERSION} build: context: ./migrations environment: - - DATABASE_URL=postgresql://voltaserve@cockroach:26257/voltaserve + - DATABASE_URL=postgresql://voltaserve:voltaserve@postgres:5432/voltaserve depends_on: - - cockroach + - postgres restart: on-failure diff --git a/idp/.env b/idp/.env index 991eaf583..36988b437 100644 --- a/idp/.env +++ b/idp/.env @@ -2,7 +2,7 @@ PORT=8081 # URLs PUBLIC_UI_URL="http://127.0.0.1:3000" -POSTGRES_URL="postgresql://voltaserve@127.0.0.1:26257/voltaserve" +POSTGRES_URL="postgresql://voltaserve:voltaserve@127.0.0.1:5432/voltaserve" # Token TOKEN_JWT_SIGNING_KEY="586cozl1x9m6zmu4fg8iwi6ajazguehcm9qdfgd5ndo2pc3pcn" diff --git a/language/models.yaml b/language/models.yaml index e9596324f..4b703ff97 100644 --- a/language/models.yaml +++ b/language/models.yaml @@ -11,8 +11,8 @@ ita: package: "it_core_news_sm" url: "https://huggingface.co/spacy/it_core_news_sm/resolve/main/it_core_news_sm-any-py3-none-any.whl" jpn: - package: "ja_core_news_sm" - url: "https://huggingface.co/spacy/ja_core_news_sm/resolve/main/ja_core_news_sm-any-py3-none-any.whl" + package: "xx_ent_wiki_sm" + url: "https://huggingface.co/spacy/xx_ent_wiki_sm/resolve/main/xx_ent_wiki_sm-any-py3-none-any.whl" nld: package: "nl_core_news_sm" url: "https://huggingface.co/spacy/nl_core_news_sm/resolve/main/nl_core_news_sm-any-py3-none-any.whl" @@ -41,11 +41,11 @@ chi_tra: package: "zh_core_web_sm" url: "https://huggingface.co/spacy/zh_core_web_sm/resolve/main/zh_core_web_sm-any-py3-none-any.whl" rus: - package: "ru_core_news_sm" - url: "https://huggingface.co/spacy/ru_core_news_sm/resolve/main/ru_core_news_sm-any-py3-none-any.whl" + package: "xx_ent_wiki_sm" + url: "https://huggingface.co/spacy/xx_ent_wiki_sm/resolve/main/xx_ent_wiki_sm-any-py3-none-any.whl" hin: package: "xx_ent_wiki_sm" url: "https://huggingface.co/spacy/xx_ent_wiki_sm/resolve/main/xx_ent_wiki_sm-any-py3-none-any.whl" ara: package: "xx_ent_wiki_sm" - url: "https://huggingface.co/spacy/xx_ent_wiki_sm/resolve/main/xx_ent_wiki_sm-any-py3-none-any.whl" \ No newline at end of file + url: "https://huggingface.co/spacy/xx_ent_wiki_sm/resolve/main/xx_ent_wiki_sm-any-py3-none-any.whl" diff --git a/migrations/README.md b/migrations/README.md index d6c2e8e55..840462cdc 100644 --- a/migrations/README.md +++ b/migrations/README.md @@ -9,5 +9,5 @@ cargo build --release Run locally: ```shell -DATABASE_URL=postgresql://voltaserve@localhost:5432/voltaserve ./target/release/migrate up -``` \ No newline at end of file +DATABASE_URL=postgresql://voltaserve:voltaserve@localhost:5432/voltaserve ./target/release/migrate up +```