Merge branch 'main' into session_setup

cmu-db · May 13, 2024 · 3c00b6a · 3c00b6a
2 parents 7f508a6 + 56c3e24
commit 3c00b6a
Show file tree

Hide file tree

Showing 27 changed files with 1,093 additions and 192 deletions.
diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml
@@ -86,6 +86,21 @@ jobs:
         name: benchbase-${{matrix.profile}}
         path: target/benchbase-${{matrix.profile}}.tgz
 
+    # Needed for running a customized service containers using docker/*/up.sh scripts.
+    # See postgres example below.
+    # https://github.com/actions/runner/issues/2139
+    - name: Package docker-compose configs
+      if: ${{ matrix.profile == 'postgres' }}
+      run: |
+        tar czvpf docker-compose-${{matrix.profile}}.tar.gz docker/${{matrix.profile}}-latest
+
+    - name: Upload docker-compose configs
+      if: ${{ matrix.profile == 'postgres' }}
+      uses: actions/upload-artifact@v4
+      with:
+        name: docker-compose-${{matrix.profile}}
+        path: docker-compose-${{matrix.profile}}.tar.gz
+
   ## ----------------------------------------------------------------------------------
   ## SQLITE
   ## ----------------------------------------------------------------------------------
@@ -406,21 +421,25 @@ jobs:
       fail-fast: false
       matrix:
         benchmark: [ 'auctionmark', 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'templated', 'tpcc', 'tpcc-with-reconnects', 'tpch', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
-    services:
-      postgres: # https://hub.docker.com/_/postgres
-        image: postgres:latest
-        env:
-          POSTGRES_DB: benchbase
-          POSTGRES_USER: admin
-          POSTGRES_PASSWORD: password
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          - 5432:5432
     steps:
+      # Note: we download just the docker-compose scripts/configs rather than the
+      # whole source code repo for better testing.
+      - name: Download artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: docker-compose-postgres
+
+      - name: Extract docker-compose artifacts
+        run: |
+          tar xvzf docker-compose-postgres.tar.gz
+
+      # Use docker-compose to start the postgres service so we can modify the
+      # command line args to include extensions.
+      # https://github.com/actions/runner/issues/2139
+      - name: Start custom postgres service
+        run: |
+          ./docker/postgres-latest/up.sh --quiet-pull postgres
+
       - name: Download artifact
         uses: actions/download-artifact@v4
         with:
@@ -472,6 +491,10 @@ jobs:
           ./scripts/check_latest_benchmark_results.sh $results_benchmark
           ./scripts/check_histogram_results.sh results/histograms.json $ERRORS_THRESHOLD
 
+      - name: Stop custom postgres service
+        run: |
+          ./docker/postgres-latest/down.sh
+
   ## ----------------------------------------------------------------------------------
   ## COCKROACHDB
   ## ----------------------------------------------------------------------------------

diff --git a/.gitignore b/.gitignore
@@ -55,4 +55,6 @@ build/
 # vim swap files
 .*.swp
 
-.env
+.env
+
+docker-compose-*.tar.gz
diff --git a/README.md b/README.md
@@ -124,7 +124,8 @@ usage: benchbase
                                 supported: [tpcc, tpch, tatp, wikipedia,
                                 resourcestresser, twitter, epinions, ycsb,
                                 seats, auctionmark, chbenchmark, voter,
-                                sibench, noop, smallbank, hyadapt, otmetrics]
+                                sibench, noop, smallbank, hyadapt,
+                                otmetrics, templated]
  -c,--config <arg>              [required] Workload configuration file
     --clear <arg>               Clear all records in the database for this
                                 benchmark
@@ -136,6 +137,7 @@ usage: benchbase
  -h,--help                      Print this help
  -im,--interval-monitor <arg>   Throughput Monitoring Interval in
                                 milliseconds
+ -jh,--json-histograms <arg>    Export histograms to JSON file
     --load <arg>                Load data using the benchmark's data
                                 loader
  -s,--sample <arg>              Sampling window

diff --git a/data/templated/example.xml b/data/templated/example.xml
@@ -9,12 +9,12 @@
          <type>INTEGER</type>
       </types>
       <values>
-         <value>3</value>
-         <value>4</value>
+         <value dist="uniform" min="1" max="3"/>
+         <value dist="zipfian" min="1" max="2"/>
       </values>
        <values>
-         <value>5</value>
-         <value>6</value>
+         <value dist="scrambled" min="0" max="4" seed="999"/>
+         <value dist="normal" min="1" max="3" seed="1"/>
       </values>
    </template>
    <template name="GetCust">
@@ -33,8 +33,8 @@
          <type>INTEGER</type>
       </types>
       <values>
+        <value>1</value>
         <value />
-        <value>8</value>
       </values>
    </template>
    <template name="GetWarehouse">
@@ -53,8 +53,8 @@
          <type>FLOAT</type>
       </types>
       <values>
-         <value>10</value>
-         <value>10.49</value>
+         <value dist="normal" min="10" max="12.5"/>
+         <value dist="uniform" min="15.1" max="100.22"/>
       </values>
       <values>
          <value>10.50</value>
@@ -64,19 +64,28 @@
     <template name="UpdateItemPrice">
       <query><![CDATA[UPDATE item SET i_price = i_price + 1 WHERE i_price < ?]]></query>
       <types>
-         <type>FLOAT</type>
+         <type>INTEGER</type>
       </types>
       <values>
-         <value>2.1</value>
+         <value dist="zipfian" min="10" max="40">2</value>
       </values>
    </template>
    <template name="DeleteItem">
-      <query><![CDATA[DELETE FROM item WHERE i_price > ?]]></query>
+      <query><![CDATA[DELETE FROM oorder WHERE o_entry_d < ?]]></query>
       <types>
-         <type>FLOAT</type>
+         <type>TIMESTAMP</type>
       </types>
       <values>
-         <value>255.0</value>
+         <value dist="uniform" min="1000" max="300000"></value>
+      </values>
+       <values>
+         <value dist="normal" min="1000" max="300000"></value>
+      </values>
+       <values>
+         <value dist="zipfian" min="1000" max="300000"></value>
+      </values>
+       <values>
+         <value dist="scrambled" min="1000" max="300000"></value>
       </values>
    </template>
    <template name="InsertItem">
@@ -97,9 +106,9 @@
          <value>1</value>
          <value>1</value>
          <value>1</value>
-         <value>2022-10-10 11:30:30</value>
-         <value>1.0</value>
-         <value>Test</value>
+         <value dist="uniform" min="1999-12-12 01:01:55" max="2000-12-12 04:04:44"/>
+         <value dist="normal" min="0.5" max="12.5"/>
+         <value dist="uniform" min="0" max="23"/>
       </values>
    </template>
 </templates>
diff --git a/docker/postgres-latest/docker-compose.yml b/docker/postgres-latest/docker-compose.yml
@@ -6,13 +6,19 @@ services:
     container_name: postgres
     hostname: postgres
     image: postgres:alpine
-    command: postgres -N 500
+    command: postgres -N 500 -c shared_preload_libraries=pg_stat_statements
     environment:
       POSTGRES_USER: admin
       POSTGRES_PASSWORD: password
       POSTGRES_DB: benchbase
     ports:
       - "5432:5432"
+    healthcheck:
+      test: pg_isready
+      interval: 10s
+      timeout: 5s
+      retries: 5
+      start_period: 30s
 
   postgres-ui:
     container_name: postgres-ui

diff --git a/docker/postgres-latest/up.sh b/docker/postgres-latest/up.sh
@@ -4,4 +4,24 @@ set -eu
 scriptdir=$(dirname "$(readlink -f "$0")")
 cd "$scriptdir/"
 
-docker compose up -d
+services="$@"
+
+docker compose up -d $services
+
+# Wait until ready
+for i in {1..5}; do
+    if docker exec postgres pg_isready && sleep 2 && docker exec postgres pg_isready; then
+        break
+    else
+        sleep 5
+    fi
+done
+
+function run_psql_in_docker() {
+    set -x
+    docker exec --env PGPASSWORD=password postgres psql -h localhost -U admin benchbase --csv -c "$@"
+    set +x
+}
+
+run_psql_in_docker "CREATE EXTENSION IF NOT EXISTS pg_stat_statements"
+run_psql_in_docker "SELECT COUNT(*) FROM pg_stat_statements" | egrep '^[1-9][0-9]*$'
diff --git a/pom.xml b/pom.xml
@@ -64,7 +64,7 @@
                 <dependency>
                     <groupId>org.xerial</groupId>
                     <artifactId>sqlite-jdbc</artifactId>
-                    <version>3.42.0.0</version>
+                    <version>3.45.2.0</version>
                 </dependency>
             </dependencies>
         </profile>
@@ -83,7 +83,7 @@
                 <dependency>
                     <groupId>org.postgresql</groupId>
                     <artifactId>postgresql</artifactId>
-                    <version>42.7.2</version>
+                    <version>42.7.3</version>
                     <scope>runtime</scope>
                 </dependency>
             </dependencies>
@@ -125,7 +125,7 @@
                 <dependency>
                     <groupId>com.oracle.database.jdbc</groupId>
                     <artifactId>ojdbc11</artifactId>
-                    <version>23.3.0.23.09</version>
+                    <version>23.4.0.24.05</version>
                     <scope>runtime</scope>
                 </dependency>
             </dependencies>
@@ -165,7 +165,7 @@
                 <dependency>
                     <groupId>com.google.cloud</groupId>
                     <artifactId>google-cloud-spanner-jdbc</artifactId>
-                    <version>2.16.0</version>
+                    <version>2.16.1</version>
                     <scope>runtime</scope>
                 </dependency>
             </dependencies>
@@ -185,7 +185,7 @@
                 <dependency>
                     <groupId>org.postgresql</groupId>
                     <artifactId>postgresql</artifactId>
-                    <version>42.7.2</version>
+                    <version>42.7.3</version>
                     <scope>runtime</scope>
                 </dependency>
             </dependencies>
@@ -236,20 +236,20 @@
         <dependency>
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-api</artifactId>
-            <version>2.0.12</version>
+            <version>2.0.13</version>
         </dependency>
 
         <dependency>
             <!-- Drop-in replacement of log4j -->
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-reload4j</artifactId>
-            <version>2.0.12</version>
+            <version>2.0.13</version>
         </dependency>
 
         <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-configuration2</artifactId>
-            <version>2.9.0</version>
+            <version>2.10.1</version>
         </dependency>
 
         <dependency>
@@ -274,13 +274,13 @@
         <dependency>
             <groupId>commons-io</groupId>
             <artifactId>commons-io</artifactId>
-            <version>2.15.1</version>
+            <version>2.16.1</version>
         </dependency>
 
         <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-text</artifactId>
-            <version>1.11.0</version>
+            <version>1.12.0</version>
             <scope>compile</scope>
         </dependency>
 
@@ -305,7 +305,7 @@
         <dependency>
             <groupId>jakarta.xml.bind</groupId>
             <artifactId>jakarta.xml.bind-api</artifactId>
-            <version>4.0.1</version>
+            <version>4.0.2</version>
         </dependency>
 
         <dependency>
@@ -358,7 +358,7 @@
                 <!-- format/check code via google style format -->
                 <groupId>com.spotify.fmt</groupId>
                 <artifactId>fmt-maven-plugin</artifactId>
-                <version>2.22.1</version>
+                <version>2.23</version>
                 <executions>
                     <execution>
                         <goals>
@@ -371,7 +371,7 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-compiler-plugin</artifactId>
-                <version>3.12.1</version>
+                <version>3.13.0</version>
                 <configuration>
                     <source>${maven.compiler.source}</source>
                     <target>${maven.compiler.target}</target>
@@ -409,7 +409,7 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-jar-plugin</artifactId>
-                <version>3.3.0</version>
+                <version>3.4.1</version>
                 <configuration>
                     <classifier>${classifier}</classifier>
                     <archive>
@@ -426,7 +426,7 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-assembly-plugin</artifactId>
-                <version>3.7.0</version>
+                <version>3.7.1</version>
                 <configuration>
                     <appendAssemblyId>false</appendAssemblyId>
                     <finalName>${project.artifactId}-${classifier}</finalName>
-Original file line number
+Diff line change
@@ Expand Up / @@ -55,4 +55,6 @@ build/ @@
     # vim swap files
     .*.swp
-    .env
+    .env
+    docker-compose-*.tar.gz