Skip to content

Commit

Permalink
add docker-compose for distributed mode testing (#777)
Browse files Browse the repository at this point in the history 
This PR adds docker-compose for distributed
workflow updated to test both standalone and distributed 
modes.

fixes : #644
  • Loading branch information
@nikhilsinhaparseable
nikhilsinhaparseable authored Apr 23, 2024
1 parent 5059d71 commit 6773190
Show file tree
Hide file tree
Showing 2 changed files with 269 additions and 1 deletion.
14 changes: 13 additions & 1 deletion .github/workflows/integration-test.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ on:
jobs:

docker-compose-test:
name: Quest Smoke and Load Tests
name: Quest Smoke and Load Tests for Standalone deployments
runs-on: ubuntu-latest
steps:
- name: Checkout
Expand All @@ -21,3 +21,15 @@ jobs:
- name: Stop compose
if: always()
run: docker-compose -f docker-compose-test.yaml down

docker-compose-distributed-test:
name: Quest Smoke and Load Tests for Distributed deployments
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Start compose
run: docker-compose -f docker-compose-distributed-test.yaml up --build --exit-code-from quest
- name: Stop compose
if: always()
run: docker-compose -f docker-compose-distributed-test.yaml down
256 changes: 256 additions & 0 deletions docker-compose-distributed-test.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,256 @@
version: "3.7"
networks:
parseable-internal:
services:
# minio
minio:
image: minio/minio:RELEASE.2023-02-10T18-48-39Z
entrypoint:
- sh
- -euc
- |
mkdir -p /tmp/minio/parseable && \
minio server /tmp/minio
environment:
- MINIO_ROOT_USER=parseable
- MINIO_ROOT_PASSWORD=supersecret
- MINIO_UPDATE=off
ports:
- 9000:9000
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
interval: 15s
timeout: 20s
retries: 5
networks:
- parseable-internal
# query server
parseable-query:
build:
context: .
dockerfile: Dockerfile
command: ["parseable", "s3-store"]
ports:
- 8000:8000
environment:
- P_S3_URL=http://minio:9000
- P_S3_ACCESS_KEY=parseable
- P_S3_SECRET_KEY=supersecret
- P_S3_REGION=us-east-1
- P_S3_BUCKET=parseable
- P_STAGING_DIR=/tmp/data
- P_USERNAME=parseableadmin
- P_PASSWORD=parseableadmin
- P_CHECK_UPDATE=false
- P_PARQUET_COMPRESSION_ALGO=snappy
- P_MODE=query
networks:
- parseable-internal
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8000/api/v1/liveness"]
interval: 15s
timeout: 20s
retries: 5
depends_on:
- minio
deploy:
restart_policy:
condition: on-failure
delay: 20s
max_attempts: 3
# ingest server one
parseable-ingest-one:
build:
context: .
dockerfile: Dockerfile
command: ["parseable", "s3-store"]
ports:
- 8000
environment:
- P_S3_URL=http://minio:9000
- P_S3_ACCESS_KEY=parseable
- P_S3_SECRET_KEY=supersecret
- P_S3_REGION=us-east-1
- P_S3_BUCKET=parseable
- P_STAGING_DIR=/tmp/data
- P_USERNAME=parseableadmin
- P_PASSWORD=parseableadmin
- P_CHECK_UPDATE=false
- P_PARQUET_COMPRESSION_ALGO=snappy
- P_MODE=ingest
- P_INGESTOR_ENDPOINT=parseable-ingest-one:8000
networks:
- parseable-internal
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8000/api/v1/liveness"]
interval: 15s
timeout: 20s
retries: 5
depends_on:
- parseable-query
- minio
deploy:
restart_policy:
condition: on-failure
delay: 20s
max_attempts: 3
# ingest server two
parseable-ingest-two:
build:
context: .
dockerfile: Dockerfile
command: ["parseable", "s3-store"]
ports:
- 8000
environment:
- P_S3_URL=http://minio:9000
- P_S3_ACCESS_KEY=parseable
- P_S3_SECRET_KEY=supersecret
- P_S3_REGION=us-east-1
- P_S3_BUCKET=parseable
- P_STAGING_DIR=/tmp/data
- P_USERNAME=parseableadmin
- P_PASSWORD=parseableadmin
- P_CHECK_UPDATE=false
- P_PARQUET_COMPRESSION_ALGO=snappy
- P_MODE=ingest
- P_INGESTOR_ENDPOINT=parseable-ingest-two:8000
networks:
- parseable-internal
healthcheck:
test: [ "CMD", "curl", "-f", "http://localhost:8000/api/v1/liveness" ]
interval: 15s
timeout: 20s
retries: 5
depends_on:
- parseable-query
- minio
deploy:
restart_policy:
condition: on-failure
delay: 20s
max_attempts: 3
# ingest server three
parseable-ingest-three:
build:
context: .
dockerfile: Dockerfile
command: ["parseable", "s3-store"]
ports:
- 8000
environment:
- P_S3_URL=http://minio:9000
- P_S3_ACCESS_KEY=parseable
- P_S3_SECRET_KEY=supersecret
- P_S3_REGION=us-east-1
- P_S3_BUCKET=parseable
- P_STAGING_DIR=/tmp/data
- P_USERNAME=parseableadmin
- P_PASSWORD=parseableadmin
- P_CHECK_UPDATE=false
- P_PARQUET_COMPRESSION_ALGO=snappy
- P_MODE=ingest
- P_INGESTOR_ENDPOINT=parseable-ingest-three:8000
networks:
- parseable-internal
healthcheck:
test: [ "CMD", "curl", "-f", "http://localhost:8000/api/v1/liveness" ]
interval: 15s
timeout: 20s
retries: 5
depends_on:
- parseable-query
- minio
deploy:
restart_policy:
condition: on-failure
delay: 20s
max_attempts: 3
# ingest server four
parseable-ingest-four:
build:
context: .
dockerfile: Dockerfile
command: ["parseable", "s3-store"]
ports:
- 8000
environment:
- P_S3_URL=http://minio:9000
- P_S3_ACCESS_KEY=parseable
- P_S3_SECRET_KEY=supersecret
- P_S3_REGION=us-east-1
- P_S3_BUCKET=parseable
- P_STAGING_DIR=/tmp/data
- P_USERNAME=parseableadmin
- P_PASSWORD=parseableadmin
- P_CHECK_UPDATE=false
- P_PARQUET_COMPRESSION_ALGO=snappy
- P_MODE=ingest
- P_INGESTOR_ENDPOINT=parseable-ingest-four:8000
networks:
- parseable-internal
healthcheck:
test: [ "CMD", "curl", "-f", "http://localhost:8000/api/v1/liveness" ]
interval: 15s
timeout: 20s
retries: 5
depends_on:
- parseable-query
- minio
deploy:
restart_policy:
condition: on-failure
delay: 20s
max_attempts: 3
# penguin lb
penguin:
platform: linux/amd64
image: parseable/penguin:edge
command: ["./target/release/penguin"]
environment:
- P_ADDR_ONE=parseable-ingest-one:8000
- P_ADDR_TWO=parseable-ingest-two:8000
- P_ADDR_THREE=parseable-ingest-three:8000
- P_ADDR_FOUR=parseable-ingest-four:8000
ports:
- 6188:6188
depends_on:
- parseable-query
- minio
- parseable-ingest-one
- parseable-ingest-two
- parseable-ingest-three
- parseable-ingest-four
networks:
- parseable-internal
quest:
platform: linux/amd64
image: ghcr.io/parseablehq/quest:main
command:
[
"load",
"http://parseable-query:8000",
"parseableadmin",
"parseableadmin",
"20",
"10",
"5m",
"minio:9000",
"parseable",
"supersecret",
"parseable",
"http://penguin:6188",
"parseableadmin",
"parseableadmin",
]
networks:
- parseable-internal
depends_on:
- parseable-query
- penguin
- minio
deploy:
restart_policy:
condition: on-failure
delay: 20s
max_attempts: 3

0 comments on commit 6773190

Please sign in to comment.