Add DocBench benchmark

polypheny · Jun 13, 2022 · ddb4bfa · ddb4bfa
1 parent dd7f060
commit ddb4bfa
Show file tree

Hide file tree

Showing 43 changed files with 1,229 additions and 60 deletions.
diff --git a/README.md b/README.md
@@ -6,6 +6,7 @@ A simple benchmarking and testing client for Polypheny-DB. It includes support f
 * **kNN-Bench**: A benchmark tailored towards typical workloads of multimedia retrieval applications and especially k-Nearest-Neighbor search.
 * **Multimedia**: This benchmarks produces workload containing or requesting randomly generated BLOBs.
 * **Graph**: A simple synthetic graph benchmark that executes a mixture of DQL and DML Cypher queries against a pseudo-random network.
+* **DocBench**: A simple benchmark for document schemas generating MongoQL queries.
 * [**AuctionMark**](http://hstore.cs.brown.edu/projects/auctionmark/): A benchmark executing workload that simulates the activities found in a well-known auction site.
 * [**SmallBank**](http://ses.library.usyd.edu.au/bitstream/2123/5353/1/michael-cahill-2009-thesis.pdf): The SmallBank benchmark models a simple banking application.
 * [**TPC-C**](http://www.tpc.org/tpcc/): A well-known on-line transaction processing (OLTP) benchmark.
@@ -56,19 +57,19 @@ OPTIONS
 
 ### Stand-alone
 
-This client can be used by specifying the name of the benchmark as first parameter (`gavel`, `knnbench`, `multimedia`). Use the `help` command to get an overview on all available parameters for this benchmark. For example:
+This client can be used by specifying the name of the benchmark as first parameter and the task as second parameter. Optionally, it is also possible to specify a multiplier for the data and workload (integer > 0). The general syntax is identical for all benchmarks:
 
 ```
-java -jar polypheny-simple-client.jar help gavel  
+java -jar polypheny-simple-client.jar  BENCHMARK  TASK  [ MULTIPLIER ] 
 ```
 
-The general syntax is identical for all three benchmarks:
+Use the `help` command to get an overview on all available parameters for a benchmark. For example:
 
 ```
-java -jar polypheny-simple-client.jar  BENCHMARK  TASK  [ MULTIPLIER ] 
+java -jar polypheny-simple-client.jar help gavel  
 ```
 
-_BENCHMARK_: `{ gavel | knnbench | multimedia | auctionmark | smallbank | tpcc | tpch | ycsb }`
+_BENCHMARK_: `{ gavel | knnbench | multimedia | graph | docbench | auctionmark | smallbank | tpcc | tpch | ycsb }`
 
 _TASK_: `{ schema | data | workload | warmup }`
 

diff --git a/build.gradle b/build.gradle
@@ -19,10 +19,11 @@ buildscript {
         classpath group: "gradle.plugin.com.github.johnrengelman", name: "shadow", version: "7.1.2"
         // Lombok (https://plugins.gradle.org/plugin/io.freefair.lombok)
         classpath group: "io.freefair.gradle", name: "lombok-plugin", version: "6.4.3"
+        // https://plugins.gradle.org/plugin/org.jetbrains.gradle.plugin.idea-ext
+        classpath group: "gradle.plugin.org.jetbrains.gradle.plugin.idea-ext", name: "gradle-idea-ext", version: "1.1.4"
     }
 }
 
-
 repositories {
     mavenCentral()
     maven {
@@ -31,7 +32,7 @@ repositories {
 }
 
 apply plugin: "java-library"
-apply plugin: "idea"
+apply plugin: "org.jetbrains.gradle.plugin.idea-ext"
 apply plugin: "io.freefair.lombok"
 apply plugin: "com.github.johnrengelman.shadow"
 
@@ -168,4 +169,37 @@ idea {
         outputDir = file("${project.buildDir}/classes")
         testOutputDir = file("${project.buildDir}/test-classes")
     }
+    project {
+        settings {
+            copyright {
+                useDefault = "MITLicense"
+                profiles {
+                    MITLicense {
+                        notice = 'The MIT License (MIT)\n' +
+                                '\n' +
+                                'Copyright (c) 2019-$today The Polypheny Project\n' +
+                                '\n' +
+                                'Permission is hereby granted, free of charge, to any person obtaining a\n' +
+                                'copy of this software and associated documentation files (the "Software"), to deal\n' +
+                                'in the Software without restriction, including without limitation the rights\n' +
+                                'to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n' +
+                                'copies of the Software, and to permit persons to whom the Software is\n' +
+                                'furnished to do so, subject to the following conditions:\n' +
+                                '\n' +
+                                'The above copyright notice and this permission notice shall be included in all\n' +
+                                'copies or substantial portions of the Software.\n' +
+                                '\n' +
+                                'THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n' +
+                                'IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n' +
+                                'FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n' +
+                                'AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n' +
+                                'LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n' +
+                                'OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\n' +
+                                'SOFTWARE.'
+                        keyword = "Copyright"
+                    }
+                }
+            }
+        }
+    }
 }
diff --git a/src/main/java/org/polypheny/simpleclient/cli/AbstractOltpBenchCommand.java b/src/main/java/org/polypheny/simpleclient/cli/AbstractOltpBenchCommand.java
@@ -84,7 +84,7 @@ public int run() throws SQLException {
                 System.err.println( "Unknown task: " + args.get( 0 ) );
             }
         } catch ( Throwable t ) {
-            log.error( "Exception while executing KnnBench!", t );
+            log.error( "Exception while executing OltpBench!", t );
             System.exit( 1 );
         }
 

diff --git a/src/main/java/org/polypheny/simpleclient/cli/DocBenchCommand.java b/src/main/java/org/polypheny/simpleclient/cli/DocBenchCommand.java
@@ -0,0 +1,108 @@
+/*
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2019-2022 The Polypheny Project
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.polypheny.simpleclient.cli;
+
+import com.github.rvesse.airline.HelpOption;
+import com.github.rvesse.airline.annotations.Arguments;
+import com.github.rvesse.airline.annotations.Command;
+import com.github.rvesse.airline.annotations.Option;
+import java.sql.SQLException;
+import java.util.List;
+import javax.inject.Inject;
+import lombok.extern.slf4j.Slf4j;
+import org.polypheny.simpleclient.executor.Executor.ExecutorFactory;
+import org.polypheny.simpleclient.executor.PolyphenyDbMongoQlExecutor.PolyphenyDbMongoQlExecutorFactory;
+import org.polypheny.simpleclient.main.DocBenchScenario;
+
+
+@Slf4j
+@Command(name = "docbench", description = "Mode for quick testing of Polypheny-DB using the DocBench benchmark.")
+public class DocBenchCommand implements CliRunnable {
+
+    @SuppressWarnings("SpringAutowiredFieldsWarningInspection")
+    @Inject
+    private HelpOption<DocBenchCommand> help;
+
+    @Arguments(description = "Task { schema | data | workload } and multiplier.")
+    private List<String> args;
+
+
+    @Option(name = { "-pdb", "--polyphenydb" }, title = "IP or Hostname", arity = 1, description = "IP or Hostname of the Polypheny-DB server (default: 127.0.0.1).")
+    public static String polyphenyDbHost = "127.0.0.1";
+
+
+    @Option(name = { "--writeCSV" }, arity = 0, description = "Write a CSV file containing execution times for all executed queries (default: false).")
+    public boolean writeCsv = false;
+
+
+    @Option(name = { "--queryList" }, arity = 0, description = "Dump all queries into a file (default: false).")
+    public boolean dumpQueryList = false;
+
+
+    @Override
+    public int run() throws SQLException {
+        if ( args == null || args.size() < 1 ) {
+            System.err.println( "Missing task" );
+            System.exit( 1 );
+        }
+
+        int multiplier = 1;
+        if ( args.size() > 1 ) {
+            multiplier = Integer.parseInt( args.get( 1 ) );
+            if ( multiplier < 1 ) {
+                System.err.println( "Multiplier needs to be a integer > 0!" );
+                System.exit( 1 );
+            }
+        }
+
+        ExecutorFactory executorFactory = new PolyphenyDbMongoQlExecutorFactory( polyphenyDbHost );
+
+        try {
+            if ( args.get( 0 ).equalsIgnoreCase( "data" ) ) {
+                DocBenchScenario.data( executorFactory, multiplier, true );
+            } else if ( args.get( 0 ).equalsIgnoreCase( "workload" ) ) {
+                DocBenchScenario.workload( executorFactory, multiplier, true, writeCsv, dumpQueryList );
+            } else if ( args.get( 0 ).equalsIgnoreCase( "schema" ) ) {
+                DocBenchScenario.schema( executorFactory, true );
+            } else if ( args.get( 0 ).equalsIgnoreCase( "warmup" ) ) {
+                DocBenchScenario.warmup( executorFactory, multiplier, true, dumpQueryList );
+            } else {
+                System.err.println( "Unknown task: " + args.get( 0 ) );
+            }
+        } catch ( Throwable t ) {
+            log.error( "Exception while executing DocBench!", t );
+            System.exit( 1 );
+        }
+
+        try {
+            Thread.sleep( 2000 );
+        } catch ( InterruptedException e ) {
+            throw new RuntimeException( "Unexpected interrupt", e );
+        }
+
+        return 0;
+    }
+
+}
diff --git a/src/main/java/org/polypheny/simpleclient/cli/GraphCommand.java b/src/main/java/org/polypheny/simpleclient/cli/GraphCommand.java
@@ -92,7 +92,7 @@ public int run() throws SQLException {
                 System.err.println( "Unknown task: " + args.get( 0 ) );
             }
         } catch ( Throwable t ) {
-            log.error( "Exception while executing KnnBench!", t );
+            log.error( "Exception while executing GraphBench!", t );
             System.exit( 1 );
         }
 

diff --git a/src/main/java/org/polypheny/simpleclient/cli/Main.java b/src/main/java/org/polypheny/simpleclient/cli/Main.java
@@ -43,6 +43,7 @@ public static void main( String[] args ) throws SQLException {
         builder.withCommands( KnnCommand.class );
         builder.withCommands( MultimediaCommand.class );
         builder.withCommands( GraphCommand.class );
+        builder.withCommands( DocBenchCommand.class );
         builder.withCommands( AuctionMarkCommand.class );
         builder.withCommands( SmallBankCommand.class );
         builder.withCommands( TpccCommand.class );

diff --git a/src/main/java/org/polypheny/simpleclient/executor/Executor.java b/src/main/java/org/polypheny/simpleclient/executor/Executor.java
@@ -59,6 +59,12 @@ public Executor createExecutorInstance() {
 
         public abstract Executor createExecutorInstance( CsvWriter csvWriter );
 
+
+        public Executor createExecutorInstance( CsvWriter csvWriter, String namespace ) {
+            return createExecutorInstance( csvWriter );
+        }
+
+
         // Allows to limit number of concurrent executor threads, 0 means no limit
         public abstract int getMaxNumberOfThreads();
 

diff --git a/src/main/java/org/polypheny/simpleclient/executor/PolyphenyDbCypherExecutor.java b/src/main/java/org/polypheny/simpleclient/executor/PolyphenyDbCypherExecutor.java
@@ -49,10 +49,10 @@ public PolyphenyDbCypherExecutor( String host, CsvWriter csvWriter ) {
 
 
     @Override
-    protected HttpRequest<?> buildQuery( String query ) {
+    protected HttpRequest<?> buildQuery( String query, String namespace ) {
         JsonObject data = new JsonObject();
         data.addProperty( "query", query );
-        data.addProperty( "database", "test" );
+        data.addProperty( "database", namespace );
         return Unirest.post( "{protocol}://{host}:{port}/cypher" )
                 .header( "Content-Type", "application/json" )
                 .body( data );

diff --git a/src/main/java/org/polypheny/simpleclient/executor/PolyphenyDbHttpExecutor.java b/src/main/java/org/polypheny/simpleclient/executor/PolyphenyDbHttpExecutor.java
@@ -49,6 +49,9 @@ public abstract class PolyphenyDbHttpExecutor implements PolyphenyDbExecutor {
     @Getter
     public final String name;
 
+    @Getter
+    String namespace;
+
     @Getter
     public final Function<Query, String> queryAccessor;
 
@@ -140,7 +143,7 @@ public long executeQuery( Query query ) throws ExecutorException {
         }
         long time;
 
-        HttpRequest<?> request = getRequest( queryAccessor.apply( query ) );
+        HttpRequest<?> request = getRequest( queryAccessor.apply( query ), namespace );
         try {
             long start = System.nanoTime();
             @SuppressWarnings("rawtypes") HttpResponse result = request.asBytes();
@@ -159,19 +162,19 @@ public long executeQuery( Query query ) throws ExecutorException {
     }
 
 
-    HttpRequest<?> buildQuery( String mql ) {
+    HttpRequest<?> buildQuery( String mql, String namespace ) {
         JsonObject data = new JsonObject();
         data.addProperty( "query", mql );
-        data.addProperty( "database", "test" );
+        data.addProperty( "database", namespace );
 
         return Unirest.post( "{protocol}://{host}:{port}/" + name.toLowerCase( Locale.ROOT ) )
                 .header( "Content-Type", "application/json" )
                 .body( data );
     }
 
 
-    HttpRequest<?> getRequest( String query ) {
-        HttpRequest<?> request = buildQuery( query );
+    HttpRequest<?> getRequest( String query, String namespace ) {
+        HttpRequest<?> request = buildQuery( query, namespace );
         request.basicAuth( "pa", "" );
         request.routeParam( "protocol", "http" );
         request.routeParam( "host", "127.0.0.1" );
@@ -187,7 +190,7 @@ public long executeQueryAndGetNumber( Query query ) throws ExecutorException {
             throw new RuntimeException( "not supported" );
         }
 
-        HttpRequest<?> request = getRequest( queryAccessor.apply( query ) );
+        HttpRequest<?> request = getRequest( queryAccessor.apply( query ), namespace );
 
         try {
             long start = System.nanoTime();

diff --git a/src/main/java/org/polypheny/simpleclient/executor/PolyphenyDbMongoQlExecutor.java b/src/main/java/org/polypheny/simpleclient/executor/PolyphenyDbMongoQlExecutor.java
@@ -44,9 +44,12 @@
 @Slf4j
 public class PolyphenyDbMongoQlExecutor extends PolyphenyDbHttpExecutor {
 
+    public static final String DEFAULT_NAMESPACE = "test";
 
-    public PolyphenyDbMongoQlExecutor( String host, CsvWriter csvWriter ) {
+
+    public PolyphenyDbMongoQlExecutor( String host, CsvWriter csvWriter, String namespace ) {
         super( "Mongo", Query::getMongoQl, host, csvWriter );
+        this.namespace = namespace;
     }
 
 
@@ -60,7 +63,7 @@ public long executeQuery( Query query ) throws ExecutorException {
         }
         long time;
 
-        HttpRequest<?> request = getRequest( query.getMongoQl() );
+        HttpRequest<?> request = getRequest( query.getMongoQl(), namespace );
         log.debug( request.getUrl() );
         try {
             long start = System.nanoTime();
@@ -87,7 +90,7 @@ public long executeQueryAndGetNumber( Query query ) throws ExecutorException {
             throw new RuntimeException( "not supported" );
         }
 
-        HttpRequest<?> request = getRequest( query.getMongoQl() );
+        HttpRequest<?> request = getRequest( query.getMongoQl(), namespace );
         log.debug( request.getUrl() );
         try {
             long start = System.nanoTime();
@@ -131,25 +134,25 @@ public void closeConnection() throws ExecutorException {
 
     @Override
     public void executeInsertList( List<BatchableInsert> batchList, AbstractConfig config ) throws ExecutorException {
-        String currentTable = null;
-        List<String> rows = new ArrayList<>();
+        String currentDocument = null;
+        List<String> documents = new ArrayList<>();
         for ( BatchableInsert query : batchList ) {
             query.debug();
             if ( query instanceof MultipartInsert ) {
                 continue;
             }
-            if ( currentTable == null ) {
-                currentTable = query.getTable();
+            if ( currentDocument == null ) {
+                currentDocument = query.getEntity();
             }
 
-            if ( currentTable.equals( query.getTable() ) ) {
-                rows.add( Objects.requireNonNull( query.getMongoQlRowExpression() ) );
+            if ( currentDocument.equals( query.getEntity() ) ) {
+                documents.add( Objects.requireNonNull( query.getMongoQlRowExpression() ) );
             } else {
                 throw new RuntimeException( "Different tables in multi-inserts. This should not happen!" );
             }
         }
-        if ( rows.size() > 0 ) {
-            executeQuery( new RawQuery( null, null, Query.buildMongoQlManyInsert( currentTable, rows ), null, false ) );
+        if ( documents.size() > 0 ) {
+            executeQuery( new RawQuery( null, null, Query.buildMongoQlManyInsert( currentDocument, documents ), null, false ) );
         }
     }
 
@@ -166,7 +169,13 @@ public PolyphenyDbMongoQlExecutorFactory( String host ) {
 
         @Override
         public PolyphenyDbMongoQlExecutor createExecutorInstance( CsvWriter csvWriter ) {
-            return new PolyphenyDbMongoQlExecutor( host, csvWriter );
+            return new PolyphenyDbMongoQlExecutor( host, csvWriter, DEFAULT_NAMESPACE );
+        }
+
+
+        @Override
+        public Executor createExecutorInstance( CsvWriter csvWriter, String namespace ) {
+            return new PolyphenyDbMongoQlExecutor( host, csvWriter, namespace );
         }
 
 

diff --git a/src/main/java/org/polypheny/simpleclient/executor/PolyphenyDbRestExecutor.java b/src/main/java/org/polypheny/simpleclient/executor/PolyphenyDbRestExecutor.java
@@ -201,10 +201,10 @@ public void executeInsertList( List<BatchableInsert> batchList, AbstractConfig c
                 continue;
             }
             if ( currentTable == null ) {
-                currentTable = query.getTable();
+                currentTable = query.getEntity();
             }
 
-            if ( currentTable.equals( query.getTable() ) ) {
+            if ( currentTable.equals( query.getEntity() ) ) {
                 rows.add( Objects.requireNonNull( query.getRestRowExpression() ) );
             } else {
                 throw new RuntimeException( "Different tables in multi-inserts. This should not happen!" );