Merge branch 'mm'

src/main/java/org/polypheny/simpleclient/cli/MultimediaCommand.java

@@ -35,6 +35,7 @@
 import lombok.extern.slf4j.Slf4j;
 import org.polypheny.simpleclient.executor.Executor.ExecutorFactory;
 import org.polypheny.simpleclient.executor.PolyphenyDbJdbcExecutor.PolyphenyDbJdbcExecutorFactory;
+import org.polypheny.simpleclient.executor.PolyphenyDbRestExecutor.PolyphenyDbRestExecutorFactory;
 import org.polypheny.simpleclient.main.Multimedia;
 
 
@@ -52,8 +53,8 @@ public class MultimediaCommand implements CliRunnable {
     @Option(name = { "-pdb", "--polyphenydb" }, title = "IP or Hostname", arity = 1, description = "IP or Hostname of the Polypheny-DB server (default: 127.0.0.1).")
     public static String polyphenyDbHost = "127.0.0.1";
 
-    //@Option(name = { "--rest" }, arity = 0, description = "Use Polypheny-DB REST interface instead of the JDBC interface (default: false).")
-    //public static boolean restInterface = false;
+    @Option(name = { "--rest" }, arity = 0, description = "Use Polypheny-DB REST interface instead of the JDBC interface (default: false).")
+    public static boolean restInterface = false;
 
     @Option(name = { "--writeCSV" }, arity = 0, description = "Write a CSV file containing execution times for all executed queries (default: false).")
     public boolean writeCsv = false;
@@ -79,12 +80,11 @@ public int run() {
         }
 
         ExecutorFactory executorFactory;
-        /*if ( restInterface ) {
+        if ( restInterface ) {
             executorFactory = new PolyphenyDbRestExecutorFactory( polyphenyDbHost );
         } else {
-            executorFactory = new PolyphenyDbJdbcExecutorFactory( polyphenyDbHost );
-        }*/
-        executorFactory = new PolyphenyDbJdbcExecutorFactory( polyphenyDbHost, true );
+            executorFactory = new PolyphenyDbJdbcExecutorFactory( polyphenyDbHost, true );
+        }
 
         if ( args.get( 0 ).equalsIgnoreCase( "data" ) ) {
             Multimedia.data( executorFactory, multiplier, true );

src/main/java/org/polypheny/simpleclient/executor/JdbcExecutor.java

@@ -26,14 +26,19 @@
 package org.polypheny.simpleclient.executor;
 
 
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.nio.file.Files;
 import java.sql.Connection;
 import java.sql.Date;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
 import java.sql.Timestamp;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -74,8 +79,9 @@ public JdbcExecutor( CsvWriter csvWriter, boolean prepareStatements ) {
     @Override
     public long executeQuery( Query query ) throws ExecutorException {
         try {
-            log.debug( query.getSql() );
+            log.debug( query.getSql().substring( 0, Math.min( 500, query.getSql().length() ) ) );
 
+            ArrayList<File> files = new ArrayList<>();
             long start = System.nanoTime();
 
             if ( prepareStatements && query.getParameterizedSqlQuery() != null ) {
@@ -109,6 +115,11 @@ public long executeQuery( Query query ) throws ExecutorException {
                         case BYTE_ARRAY:
                             preparedStatement.setBytes( entry.getKey(), (byte[]) entry.getValue().right );
                             break;
+                        case FILE:
+                            File f = (File) entry.getValue().right;
+                            files.add( f );
+                            preparedStatement.setBinaryStream( entry.getKey(), new FileInputStream( f ) );
+                            break;
                     }
                 }
                 if ( query.isExpectResultSet() ) {
@@ -131,11 +142,12 @@ public long executeQuery( Query query ) throws ExecutorException {
             }
 
             long time = System.nanoTime() - start;
+            files.forEach( File::delete );
             if ( csvWriter != null ) {
                 csvWriter.appendToCsv( query.getSql(), time );
             }
             return time;
-        } catch ( SQLException e ) {
+        } catch ( SQLException | FileNotFoundException e ) {
             throw new ExecutorException( e );
         }
     }
@@ -144,7 +156,7 @@ public long executeQuery( Query query ) throws ExecutorException {
     @Override
     public long executeQueryAndGetNumber( Query query ) throws ExecutorException {
         try {
-            log.debug( query.getSql() );
+            //log.debug( query.getSql() );
 
             ResultSet resultSet = executeStatement.executeQuery( query.getSql() );
 
@@ -216,6 +228,7 @@ public void executeInsertList( List<BatchableInsert> queryList, AbstractConfig c
     protected void executeInsertListAsPreparedBatch( List<BatchableInsert> queryList ) throws ExecutorException {
         try {
             PreparedStatement preparedStatement = connection.prepareStatement( queryList.get( 0 ).getParameterizedSqlQuery() );
+            ArrayList<File> files = new ArrayList<>();
             for ( BatchableInsert insert : queryList ) {
                 Map<Integer, ImmutablePair<DataTypes, Object>> data = insert.getParameterValues();
                 for ( Map.Entry<Integer, ImmutablePair<DataTypes, Object>> entry : data.entrySet() ) {
@@ -241,13 +254,21 @@ protected void executeInsertListAsPreparedBatch( List<BatchableInsert> queryList
                         case BYTE_ARRAY:
                             preparedStatement.setBytes( entry.getKey(), (byte[]) entry.getValue().right );
                             break;
+                        case FILE:
+                            File f = (File) entry.getValue().right;
+                            byte[] b = Files.readAllBytes( f.toPath() );
+                            preparedStatement.setBytes( entry.getKey(), b );
+                            break;
                     }
                 }
                 preparedStatement.addBatch();
             }
             preparedStatement.executeBatch();
             preparedStatement.close();
-        } catch ( SQLException e ) {
+            files.forEach( File::delete );
+        } catch ( SQLException | FileNotFoundException e ) {
+            throw new ExecutorException( e );
+        } catch ( IOException e ) {
             throw new ExecutorException( e );
         }
     }

src/main/java/org/polypheny/simpleclient/executor/PolyphenyDbRestExecutor.java

@@ -1,5 +1,6 @@
 package org.polypheny.simpleclient.executor;
 
+
 import com.google.gson.JsonObject;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -15,6 +16,7 @@
 import org.polypheny.simpleclient.executor.PolyphenyDbJdbcExecutor.PolyphenyDbJdbcExecutorFactory;
 import org.polypheny.simpleclient.main.CsvWriter;
 import org.polypheny.simpleclient.query.BatchableInsert;
+import org.polypheny.simpleclient.query.MultipartInsert;
 import org.polypheny.simpleclient.query.Query;
 import org.polypheny.simpleclient.query.RawQuery;
 import org.polypheny.simpleclient.scenario.AbstractConfig;
@@ -45,6 +47,12 @@ public void reset() throws ExecutorException {
 
     @Override
     public long executeQuery( Query query ) throws ExecutorException {
+        //query.debug();
+        if ( query instanceof MultipartInsert ) {
+            long l = executeQuery( new RawQuery( null, ((MultipartInsert) query).buildMultipartInsert(), query.isExpectResultSet() ) );
+            ((MultipartInsert) query).cleanup();
+            return l;
+        }
         long time;
         if ( query.getRest() != null ) {
             HttpRequest<?> request = query.getRest();
@@ -55,7 +63,7 @@ public long executeQuery( Query query ) throws ExecutorException {
             log.debug( request.getUrl() );
             try {
                 long start = System.nanoTime();
-                HttpResponse<JsonNode> result = request.asJson();
+                @SuppressWarnings("rawtypes") HttpResponse result = request.asBytes();
                 if ( !result.isSuccess() ) {
                     throw new ExecutorException( "Error while executing REST query. Message: " + result.getStatusText() + "  |  URL: " + request.getUrl() );
                 }
@@ -68,7 +76,7 @@ public long executeQuery( Query query ) throws ExecutorException {
             }
         } else {
             // There is no REST expression available for this query. Executing SQL expression via JDBC.
-            log.warn( query.getSql() );
+            //log.warn( query.getSql() );
             JdbcExecutor executor = null;
             try {
                 executor = jdbcExecutorFactory.createExecutorInstance( csvWriter );
@@ -89,6 +97,12 @@ public long executeQuery( Query query ) throws ExecutorException {
 
     @Override
     public long executeQueryAndGetNumber( Query query ) throws ExecutorException {
+        query.debug();
+        if ( query instanceof MultipartInsert ) {
+            long l = executeQuery( new RawQuery( null, ((MultipartInsert) query).buildMultipartInsert(), query.isExpectResultSet() ) );
+            ((MultipartInsert) query).cleanup();
+            return l;
+        }
         if ( query.getRest() != null ) {
             HttpRequest<?> request = query.getRest();
             request.basicAuth( "pa", "" );
@@ -120,7 +134,7 @@ public long executeQueryAndGetNumber( Query query ) throws ExecutorException {
             }
         } else {
             // There is no REST expression available for this query. Executing SQL expression via JDBC.
-            log.warn( query.getSql() );
+            log.warn( query.getSql().substring( 0, Math.min( 500, query.getSql().length() ) ) );
             JdbcExecutor executor = null;
             try {
                 executor = jdbcExecutorFactory.createExecutorInstance( csvWriter );
@@ -157,6 +171,12 @@ public void executeInsertList( List<BatchableInsert> batchList, AbstractConfig c
         String currentTable = null;
         List<JsonObject> rows = new ArrayList<>();
         for ( BatchableInsert query : batchList ) {
+            query.debug();
+            if ( query instanceof MultipartInsert ) {
+                executeQuery( new RawQuery( null, ((MultipartInsert) query).buildMultipartInsert(), query.isExpectResultSet() ) );
+                ((MultipartInsert) query).cleanup();
+                continue;
+            }
             if ( currentTable == null ) {
                 currentTable = query.getTable();
             }

src/main/java/org/polypheny/simpleclient/query/MultipartInsert.java

@@ -0,0 +1,76 @@
+/*
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2020 Databases and Information Systems Research Group, University of Basel, Switzerland
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+package org.polypheny.simpleclient.query;
+
+
+import com.google.gson.JsonArray;
+import java.io.File;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import kong.unirest.HttpRequest;
+import kong.unirest.MultipartBody;
+import kong.unirest.Unirest;
+
+
+public abstract class MultipartInsert extends BatchableInsert {
+
+    Map<String, File> files = new HashMap<>();
+
+    public MultipartInsert( boolean expectResult ) {
+        super( expectResult );
+    }
+
+    public void setFile( String column, File f ) {
+        files.put( column, f );
+    }
+
+    public abstract Map<String, String> getRestParameters();
+
+    public HttpRequest<?> buildMultipartInsert() {
+        MultipartBody body = Unirest.post( "{protocol}://{host}:{port}/restapi/v1/multipart" ).multiPartContent();
+        body.field( "resName", getTable() );
+        if ( getRestRowExpression() != null ) {
+            JsonArray jsonArray = new JsonArray();
+            jsonArray.add( getRestRowExpression() );
+            body.field( "data", jsonArray.toString() );
+        }
+        if ( getRestParameters() != null ) {
+            for ( Entry<String, String> entry : getRestParameters().entrySet() ) {
+                body.field( entry.getKey(), entry.getValue() );
+            }
+        }
+        for ( Entry<String, File> entry : files.entrySet() ) {
+            body.field( entry.getKey(), entry.getValue() );
+        }
+        return body;
+    }
+
+    public void cleanup() {
+        files.forEach( ( key, file ) -> file.delete() );
+    }
+
+}

src/main/java/org/polypheny/simpleclient/query/Query.java

@@ -34,9 +34,11 @@
 import kong.unirest.RequestBodyEntity;
 import kong.unirest.Unirest;
 import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.tuple.ImmutablePair;
 
 
+@Slf4j
 public abstract class Query {
 
     @Getter
@@ -48,7 +50,7 @@ public Query( boolean expectResultSet ) {
     }
 
 
-    public enum DataTypes {INTEGER, VARCHAR, TIMESTAMP, DATE, ARRAY_INT, ARRAY_REAL, BYTE_ARRAY}
+    public enum DataTypes {INTEGER, VARCHAR, TIMESTAMP, DATE, ARRAY_INT, ARRAY_REAL, BYTE_ARRAY, FILE}
 
 
     public abstract String getSql();
@@ -94,4 +96,16 @@ public static HttpRequest<?> buildRestUpdate( String table, JsonObject set, Map<
         return request;
     }
 
+    public void debug() {
+        String parametrizedQuery = getParameterizedSqlQuery();
+        if ( parametrizedQuery != null ) {
+            log.debug( parametrizedQuery.substring( 0, Math.min( 500, parametrizedQuery.length() ) ) );
+        } else {
+            String sql = getSql();
+            if ( sql != null ) {
+                log.debug( sql.substring( 0, Math.min( 500, sql.length() ) ) );
+            }
+        }
+    }
+
 }

src/main/java/org/polypheny/simpleclient/scenario/multimedia/DataGenerator.java

@@ -26,9 +26,12 @@
 package org.polypheny.simpleclient.scenario.multimedia;
 
 
+import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.ThreadLocalRandom;
 import lombok.extern.slf4j.Slf4j;
 import org.polypheny.simpleclient.executor.Executor;
@@ -52,6 +55,7 @@ public class DataGenerator {
     private final ProgressReporter progressReporter;
 
     private final List<BatchableInsert> batchList;
+    Map<String, List<Long>> queryTimes = new HashMap<>();
 
     private boolean aborted;
 
@@ -66,7 +70,7 @@ public class DataGenerator {
     }
 
 
-    void generateUsers() throws ExecutorException {
+    Map<String, List<Long>> generateUsers() throws ExecutorException {
         int numberOfUsers = config.numberOfUsers;
         int mod = numberOfUsers / progressReporter.base;
         InsertUser insertUser = new InsertUser( config.imgSize );
@@ -116,6 +120,7 @@ void generateUsers() throws ExecutorException {
             }
             executeInsertList();
         }
+        return queryTimes;
     }
 
 
@@ -128,8 +133,28 @@ private void addToInsertList( BatchableInsert query ) throws ExecutorException {
 
 
     private void executeInsertList() throws ExecutorException {
+        if ( batchList.size() == 0 ) {
+            return;
+        }
+        long startTime = System.nanoTime();
         theExecutor.executeInsertList( batchList, config );
         theExecutor.executeCommit();
+        long executionTime = System.nanoTime() - startTime;
+        ArrayList<Long> executionTimes = new ArrayList<>();
+        //add execution n times to get the right average later on
+        for ( int i = 0; i < batchList.size(); i++ ) {
+            executionTimes.add( executionTime );
+        }
+        //the batchList contains only queries of one type
+        String sql = batchList.get( 0 ).getParameterizedSqlQuery();
+        if ( sql == null ) {
+            sql = batchList.get( 0 ).getSql();
+            sql = sql.substring( 0, Math.min( 500, sql.length() ) );
+        }
+        if ( !queryTimes.containsKey( sql ) ) {
+            queryTimes.put( batchList.get( 0 ).getParameterizedSqlQuery(), new ArrayList<>() );
+        }
+        queryTimes.get( batchList.get( 0 ).getParameterizedSqlQuery() ).addAll( executionTimes );
         batchList.clear();
     }