Skip to content
GitHub-Hauptaktion

Run Hyrise master with GH actions #196

Sign in to view logs

Run Hyrise master with GH actions

Run Hyrise master with GH actions #196

Workflow file for this run

.github/workflows/haupt.yml at 804ea12
name: GitHub-Hauptaktion
on:
push:
branches:
- 'main'
pull_request:
schedule:
- cron: "0 3 17 * *"
defaults:
run:
shell: bash
env:
SKIP_HYRISE: true
SKIP_HYRISE_MASTER: false
SKIP_MONETDB: true
SKIP_DUCKDB: true
SCALE_FACTOR: 0.5
CMAKE_GENERATOR: Ninja
jobs:
hyrise_full_pipeline:
name: Hyrise - Full calibration and evaluation pipeline
runs-on: ubuntu-24.04
outputs:
core_count: ${{ steps.core_client_counts.outputs.core_count }}
client_count: ${{ steps.core_client_counts.outputs.client_count }}
calibration_run: ${{ steps.calibration.outputs.calibration_run }}
steps:
- uses: actions/checkout@master
with:
submodules: recursive
- name: Install dependencies for Act setup
if: ${{ env.ACT }}
run: |
sudo apt-get update -y -qq
sudo apt-get install -y -qq git build-essential cmake python3-pip
- name: Install dependencies
run: |
sudo apt-get update -y -qq
# Add repository for older python versions. We use 3.11 as there are several issues with 3.12 (e.g., removed distutils and pip problems).
sudo add-apt-repository ppa:deadsnakes/ppa --yes
# We don't use Hyrise's install_dependencies script as it includes much more than needed for this small setup here.
sudo apt-get install -y ninja-build libboost-all-dev postgresql-server-dev-16 libtbb-dev libreadline-dev libsqlite3-dev systemtap-sdt-dev lld numactl python3.11-full python3.11-venv
python3.11 -m venv ~/venv
source ~/venv/bin/activate
python -m pip install -r python/requirements.txt --quiet
- name: Determine core and client counts for database comparison
id: core_client_counts
run: |
core_count=`grep -Pc '^processor\t' /proc/cpuinfo`
client_count=$(python -c "import math; print(int(math.ceil(${core_count}*0.75)))")
comparison_runtime=$(python -c "print(min(1800, max(300, int(${{ env.SCALE_FACTOR }}*3500))))")
echo "Using ${core_count} cores and ${client_count} clients, comparison benchmarks running for ${comparison_runtime} seconds."
echo "CORE_COUNT=${core_count}" >> $GITHUB_ENV
echo "CLIENT_COUNT=${client_count}" >> $GITHUB_ENV
echo "COMPARISON_RUNTIME=${comparison_runtime}" >> $GITHUB_ENV
echo "core_count=${core_count}" >> $GITHUB_OUTPUT
echo "client_count=${client_count}" >> $GITHUB_OUTPUT
- name: Build release server and plugins
if: env.SKIP_HYRISE == 'false'
run: |
mkdir -p encoding_plugin/rel
pushd encoding_plugin/rel > /dev/null
# Erase all encoding types. Hurts performance but allows us to compile in release mode with GitHub runners.
# Further, we use the relaxed mode as there are several issues with newer compiler (fixed in Hyrise's master,
# but not in the project's code).
cmake -DCMAKE_BUILD_TYPE=Release -DHYRISE_RELAXED_BUILD=ON -DERASE_SEGMENT_TYPES=Dictionary,LZ4,RunLength,FSST,FrameOfReference,Unencoded,FixedStringDictionary ..
cmake --build . --target hyriseServer WorkloadStatisticsPlugin WorkloadHandlerPlugin CommandExecutorPlugin DataCharacteristicsPlugin
popd > /dev/null
- name: Run calibration - data collection phase (TPC-H only)
if: env.SKIP_HYRISE == 'false'
run: |
# We have a custom CMake target that might not trigger correctly. Since we don't use TPC-DS in this GitHub Action
# run here, creating an empty directory should be fine.
rm -rf encoding_plugin/rel/resources || true # mkdir -p does not work wity symlinks. Just get it done.
mkdir -p encoding_plugin/rel/resources/benchmark/tpcds/tpcds-result-reproduction/query_qualification
source ~/venv/bin/activate
pushd python > /dev/null
python3 runner.py --hyrise_server_path=../encoding_plugin/rel/ --base_benchmark_runs=1 --single_benchmark=TPC-H --execute=calibration --scale_factor ${{ env.SCALE_FACTOR }} --random_encoding_configs_count=3
popd > /dev/null
- name: Run calibration - learn runtime and size models
id: calibration
if: env.SKIP_HYRISE == 'false'
run: |
source ~/venv/bin/activate
pushd python > /dev/null
calibration_run=`ls -t calibration | grep -v 'results' | head -n1`
# Run pipeline without selection.
python3 encoding_selection_pipeline.py --calibration_dir=calibration/${calibration_run} --skip_phases selection
popd > /dev/null
echo "calibration_run=${calibration_run}" >> $GITHUB_OUTPUT
echo "CALIBRATION_RUN=${calibration_run}" >> $GITHUB_ENV
- name: Run encoding selection
if: env.SKIP_HYRISE == 'false'
run: |
source ~/venv/bin/activate
pushd python > /dev/null
# Run selection. For simplicity: use calibration workload as workload to optimize.
python3 encoding_selection_pipeline.py --calibration_dir=calibration/${{ env.CALIBRATION_RUN }} --use_calibration_as_workload --skip_phases load_csv prepare learn_runtime learn_size --budget_steps_stretch_factor 5.0
popd > /dev/null
- name: Benchmark encoding configurations
if: env.SKIP_HYRISE == 'false'
run: |
source ~/venv/bin/activate
pushd python
python3 runner.py --hyrise_server_path=../encoding_plugin/rel/ --execute=evaluation --configurations_dir "evaluation/${{ env.CALIBRATION_RUN }}/configurations__default/TPCH" --results_dir "evaluation/${{ env.CALIBRATION_RUN }}/results/TPCH" --scale_factor ${{ env.SCALE_FACTOR }} --single_benchmark=TPCH --port 5551
popd
- name: Benchmark non-constrained Hyrise (database comparison)
if: env.SKIP_HYRISE == 'false'
run: |
source ~/venv/bin/activate
pushd python
python3 db_comparison_runner.py hyrise --hyrise_server_path=../encoding_plugin/rel/ --cores=${{ env.CORE_COUNT }} --scale_factor=${{ env.SCALE_FACTOR }} --clients=${{ env.CLIENT_COUNT }} --time=${{ env.COMPARISON_RUNTIME }}
python3 db_comparison_runner.py hyrise --hyrise_server_path=../encoding_plugin/rel/ --cores=${{ env.CORE_COUNT }} --scale_factor=${{ env.SCALE_FACTOR }} --determine_size_only
popd
- name: Benchmark budget-constrained Hyrise (database comparison)
if: env.SKIP_HYRISE == 'false'
run: |
source ~/venv/bin/activate
pushd python
python3 runner.py --hyrise_server_path=../encoding_plugin/rel/ --execute=evaluation --configurations_dir "evaluation/${{ env.CALIBRATION_RUN }}/configurations__default/TPCH/LPCompressionSelection" --results_dir "evaluation/${{ env.CALIBRATION_RUN }}/results/TPCH/LPCompressionSelection" --scale_factor ${{ env.SCALE_FACTOR }} --single_benchmark=TPCH --port 5551 --cores=${{ env.CORE_COUNT }} --clients=${{ env.CLIENT_COUNT }}
popd
- name: Upload benchmark results (non-constrained)
uses: actions/upload-artifact@master
if: env.SKIP_HYRISE == 'false'
with:
name: comparison_results_hyrise_non-constrained
path: |
python/db_comparison_results/*.csv
- name: Upload benchmark results (budget-constrained)
uses: actions/upload-artifact@master
if: env.SKIP_HYRISE == 'false'
with:
name: comparison_results_hyrise_budget-constrained
path: |
python/evaluation/${{ env.CALIBRATION_RUN }}/results/TPCH/*.csv
python/evaluation/${{ env.CALIBRATION_RUN }}/results/TPCH/LPCompressionSelection/*.csv
database_comparison:
name: Database Comparison
runs-on: ubuntu-24.04
outputs:
core_count: ${{ steps.core_client_counts.outputs.core_count }}
client_count: ${{ steps.core_client_counts.outputs.client_count }}
steps:
- uses: actions/checkout@master
- uses: actions/checkout@master
if: env.SKIP_MONETDB == 'false'
with:
token: ${{ secrets.PAT }}
repository: MonetDB/MonetDB
ref: 'Dec2023_9'
path: ./MonetDB
- uses: actions/checkout@master
if: env.SKIP_MONETDB == 'false'
with:
token: ${{ secrets.PAT }}
repository: MonetDBSolutions/tpch-scripts
path: ./tpch-scripts
- uses: actions/checkout@master
if: env.SKIP_DUCKDB == 'false'
with:
token: ${{ secrets.PAT }}
repository: electrum/tpch-dbgen
path: ./tpch-dbgen
- uses: actions/checkout@master
if: env.SKIP_HYRISE_MASTER == 'false'
with:
token: ${{ secrets.PAT }}
repository: hyrise/hyrise
path: ./hyrise_master
submodules: recursive
- name: Determine client and core counts for database comparison
id: core_client_counts
run: |
core_count=`grep -Pc '^processor\t' /proc/cpuinfo`
client_count=$(python -c "import math; print(int(math.ceil(${core_count}*0.75)))")
comparison_runtime=$(python -c "print(min(1800, max(300, int(${{ env.SCALE_FACTOR }}*3500))))")
echo "Using ${core_count} cores and ${client_count} clients, comparison benchmarks running for ${comparison_runtime} seconds."
echo "CORE_COUNT=${core_count}" >> $GITHUB_ENV
echo "CLIENT_COUNT=${client_count}" >> $GITHUB_ENV
echo "COMPARISON_RUNTIME=${comparison_runtime}" >> $GITHUB_ENV
echo "core_count=${core_count}" >> $GITHUB_OUTPUT
echo "client_count=${client_count}" >> $GITHUB_OUTPUT
- name: Install dependencies for Act setup
if: ${{ env.ACT }}
run: |
sudo apt-get update -y -qq
sudo apt-get install -y -qq git build-essential cmake python3-pip
- name: Install dependencies
run: |
sudo apt-get update -y -qq
sudo apt-get install -y -qq ninja-build libsqlite3-dev postgresql-server-dev-16 numactl bison python3-venv libboost-all-dev libtbb-dev lld libreadline-dev
python3 -m venv ~/venv
source ~/venv/bin/activate
pip3 install -r python/requirements.txt # Not using --quiet to log the installed DuckDB version.
- name: Setup MonetDB
if: env.SKIP_MONETDB == 'false'
run: |
pushd MonetDB
mkdir rel
pushd rel
cmake -DCMAKE_INSTALL_PREFIX=~/monetdb_bin/ -DASSERT=OFF -DCMAKE_BUILD_TYPE=Release .. 1> /dev/null
cmake --build . --target install
echo "${HOME}/monetdb_bin/bin" >> $GITHUB_PATH
popd
popd
- name: Generate TPC-H data set (MonetDB)
if: env.SKIP_MONETDB == 'false'
run: |
mkdir -p monetdb_farm
pushd tpch-scripts
./tpch_build.sh -s ${{ env.SCALE_FACTOR }} -f ~/monetdb_farm
popd
- name: Benchmark MonetDB (database comparison)
if: env.SKIP_MONETDB == 'false'
run: |
pushd python
source ~/venv/bin/activate
python3 db_comparison_runner.py monetdb --cores=${{ env.CORE_COUNT }} --scale_factor=${{ env.SCALE_FACTOR }} --clients=${{ env.CLIENT_COUNT }} --time=${{ env.COMPARISON_RUNTIME }}
python3 db_comparison_runner.py monetdb --cores=${{ env.CORE_COUNT }} --scale_factor=${{ env.SCALE_FACTOR }} --determine_size_only
popd
- name: Generate TPC-H data set (for DuckDB and Umbra)
if: env.SKIP_DUCKDB == 'false'
run: |
pushd tpch-dbgen
make &> /dev/null
./dbgen -s ${{ env.SCALE_FACTOR }} -f
# Unsure what happens here, but without mode change, DuckDB cannot read nation.tbl as it has mode 111 (seen
# only with GH action on 24.04).
chmod 644 *.tbl
mkdir -p sf${{ env.SCALE_FACTOR }}
mv *.tbl sf${{ env.SCALE_FACTOR }}
popd
mv tpch-dbgen ~
- name: Benchmark DuckDB (database comparison)
if: env.SKIP_DUCKDB == 'false'
run: |
pushd python
source ~/venv/bin/activate
python3 db_comparison_runner.py duckdb --cores=${{ env.CORE_COUNT }} --scale_factor=${{ env.SCALE_FACTOR }} --clients=${{ env.CLIENT_COUNT }} --time=${{ env.COMPARISON_RUNTIME }}
python3 db_comparison_runner.py duckdb --cores=${{ env.CORE_COUNT }} --scale_factor=${{ env.SCALE_FACTOR }} --determine_size_only
popd
- name: Build Hyrise (master) release server
if: env.SKIP_HYRISE_MASTER == 'false'
run: |
mkdir -p hyrise_master/rel
pushd hyrise_master/rel > /dev/null
# Erase all encoding types. Hurts performance but allows us to compile in release mode with GitHub runners.
cmake -DCMAKE_BUILD_TYPE=Release -DHYRISE_RELAXED_BUILD=ON -DERASE_SEGMENT_TYPES=Dictionary,LZ4,RunLength,FSST,FrameOfReference,Unencoded,FixedStringDictionary ..
cmake --build . --target hyriseServer
popd > /dev/null
- name: Benchmark Hyrise (master, database comparison)
if: env.SKIP_HYRISE_MASTER == 'false'
run: |
pushd python
source ~/venv/bin/activate
python3 db_comparison_runner.py hyrise --hyrise_server_path=../hyrise_master/rel/ --cores=${{ env.CORE_COUNT }} --scale_factor=${{ env.SCALE_FACTOR }} --clients=${{ env.CLIENT_COUNT }} --time=301
python3 db_comparison_runner.py hyrise --hyrise_server_path=../hyrise_master/rel/ --cores=${{ env.CORE_COUNT }} --scale_factor=${{ env.SCALE_FACTOR }} --determine_size_only
mv db_comparison_results/database_comparison__TPC-H__hyrise.csv db_comparison_results/database_comparison__TPC-H__hyrise_master.csv
mv db_comparison_results/size_hyrise__SF${{env.SCALE_FACTOR}}.csv db_comparison_results/size_hyrise_master__SF${{env.SCALE_FACTOR}}.csv
popd
- name: Upload benchmark results
uses: actions/upload-artifact@master
if: env.SKIP_DUCKDB == 'false' || env.SKIP_MONETDB == 'false' || env.SKIP_HYRISE_MASTER == 'false'
with:
name: comparison_results
path: |
python/db_comparison_results/*.csv
plotting:
needs: [hyrise_full_pipeline, database_comparison]
name: Plotting
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@master
- uses: r-lib/actions/setup-r@v2
- uses: actions/download-artifact@master
with:
name: comparison_results
path: results_to_plot
- uses: actions/download-artifact@master
with:
name: comparison_results_hyrise_non-constrained
path: results_to_plot
- uses: actions/download-artifact@master
with:
name: comparison_results_hyrise_budget-constrained
path: results_to_plot
- name: Set environment variables
run: |
echo "HYRISE_CORE_COUNT=${{ needs.hyrise_full_pipeline.outputs.core_count }}" >> $GITHUB_ENV
echo "HYRISE_CLIENT_COUNT=${{ needs.hyrise_full_pipeline.outputs.client_count }}" >> $GITHUB_ENV
echo "CALIBRATION_RUN=${{ needs.hyrise_full_pipeline.outputs.calibration_run }}" >> $GITHUB_ENV
echo "COMPARISON_CORE_COUNT=${{ needs.database_comparison.outputs.core_count }}" >> $GITHUB_ENV
echo "COMPARISON_CLIENT_COUNT=${{ needs.database_comparison.outputs.client_count }}" >> $GITHUB_ENV
ls -lisa results_to_plot
# Install R packages (install action did not work with act)
- name: Install dependencies
run: |
install.packages(c(
"dplyr",
"ggplot2",
"ggrepel"
))
shell: Rscript {0}
- name: Plot
run: |
source("R/plot.R")
shell: Rscript {0}
- name: Upload database comparison plot
uses: actions/upload-artifact@master
with:
name: database_comparison
path: |
db_comparison.pdf