Merge pull request #62 from qdrant/ci/manual-benchmarking-workflow

ci: Add manual benchmarking workflow
qdrant · Oct 5, 2023 · d3daf00 · d3daf00
2 parents b656d4c + 4ac5ca5
commit d3daf00
Show file tree

Hide file tree

Showing 4 changed files with 45 additions and 5 deletions.
diff --git a/.github/workflows/manual-benchmark.yaml b/.github/workflows/manual-benchmark.yaml
@@ -0,0 +1,32 @@
+name: Manual Benchmark
+
+on:
+ repository_dispatch:
+ workflow_dispatch:
+ inputs:
+ qdrant_branch:
+ description: "Branch of qdrant to benchmark"
+ default: dev
+ benchmark_table:
+ description: "Postgres table to store benchmark results"
+ default: benchmark_manual
+
+jobs:
+ runManualBenchmark:
+ name: manual benchmark - ${{ github.ref_name }}
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v3
+ - uses: webfactory/[email protected]
+ with:
+ ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
+ - name: Benches
+ run: |
+ export HCLOUD_TOKEN=${{ secrets.HCLOUD_TOKEN }}
+ export GCS_KEY=${{ secrets.GCS_KEY }}
+ export GCS_SECRET=${{ secrets.GCS_SECRET }}
+ export POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }}
+ export POSTGRES_HOST=${{ secrets.POSTGRES_HOST }}
+ export QDRANT_VERSION=${{ inputs.qdrant_branch }}
+ export POSTGRES_TABLE=${{ inputs.benchmark_table }}
+ bash -x tools/run_ci.sh
diff --git a/engine/servers/qdrant-continuous-benchmarks/docker-compose.yaml b/engine/servers/qdrant-continuous-benchmarks/docker-compose.yaml
@@ -3,6 +3,7 @@ version: '3.7'
 services:
  qdrant_bench:
  image: qdrant/qdrant:${QDRANT_VERSION}
+ container_name: qdrant-continuous
  network_mode: host
  logging:
  driver: "json-file"

diff --git a/tools/run_server_container.sh b/tools/run_server_container.sh
@@ -20,7 +20,13 @@ IP_OF_THE_SERVER=$(bash "${SCRIPT_PATH}/${CLOUD_NAME}/get_public_ip.sh" "$BENCH_
 
 bash -x "${SCRIPT_PATH}/sync_servers.sh" "root@$IP_OF_THE_SERVER"
 
-DOCKER_COMPOSE="export QDRANT_VERSION=${QDRANT_VERSION}; docker compose down ; docker rmi qdrant/qdrant:${QDRANT_VERSION} || true ; docker compose up -d"
-
-ssh -t "${SERVER_USERNAME}@${IP_OF_THE_SERVER}" "cd ./projects/vector-db-benchmark/engine/servers/${CONTAINER_NAME} ; $DOCKER_COMPOSE"
-
+if [ "${QDRANT_VERSION}" == "dev" ]; then
+ # if version is dev, run in docker
+ DOCKER_COMPOSE="export QDRANT_VERSION=${QDRANT_VERSION}; docker compose down ; docker rmi qdrant/qdrant:${QDRANT_VERSION} || true ; docker compose up -d"
+ ssh -t "${SERVER_USERNAME}@${IP_OF_THE_SERVER}" "cd ./projects/vector-db-benchmark/engine/servers/${CONTAINER_NAME} ; $DOCKER_COMPOSE"
+else
+ # else run natively in the server
+ DOCKER_QDRANT_STOP="docker stop qdrant-continuous || true"
+ QDRANT_BUILD="git checkout ${QDRANT_VERSION}; nohup mold -run cargo run --bin qdrant --release &"
+ ssh -t "${SERVER_USERNAME}@${IP_OF_THE_SERVER}" "cd ./projects/qdrant; ${DOCKER_QDRANT_STOP}; $QDRANT_BUILD"
+fi
diff --git a/tools/upload_results_postgres.sh b/tools/upload_results_postgres.sh
@@ -20,6 +20,7 @@
 SEARCH_RESULTS_FILE=${SEARCH_RESULTS_FILE:-""}
 UPLOAD_RESULTS_FILE=${UPLOAD_RESULTS_FILE:-""}
 MEMORY_USAGE_FILE=${MEMORY_USAGE_FILE:-""}
+POSTGRES_TABLE=${POSTGRES_TABLE:-"benchmark"}
 
 if [[ -z "$SEARCH_RESULTS_FILE" ]]; then
  echo "SEARCH_RESULTS_FILE is not set"
@@ -52,7 +53,7 @@ MEASURE_TIMESTAMP=${MEASURE_TIMESTAMP:-$(date -u +"%Y-%m-%dT%H:%M:%SZ")}
 
 
 docker run --rm jbergknoff/postgresql-client "postgresql://qdrant:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:5432/postgres" -c "
-INSERT INTO benchmark (engine, measure_timestamp, upload_time, indexing_time, rps, mean_precisions, p95_time, p99_time, memory_usage)
+INSERT INTO ${POSTGRES_TABLE} (engine, measure_timestamp, upload_time, indexing_time, rps, mean_precisions, p95_time, p99_time, memory_usage)
 VALUES ('qdrant-ci', '${MEASURE_TIMESTAMP}', ${UPLOAD_TIME}, ${INDEXING_TIME}, ${RPS}, ${MEAN_PRECISIONS}, ${P95_TIME}, ${P99_TIME}, ${MEMORY_USAGE});
 "