Skip to content

Commit

Permalink
Merge branch 'knn-benchmark'
Browse files Browse the repository at this point in the history
  • Loading branch information
vogti committed Dec 3, 2020
2 parents 06ae342 + d29c396 commit e8a4131
Show file tree
Hide file tree
Showing 6 changed files with 373 additions and 32 deletions.
19 changes: 19 additions & 0 deletions src/main/java/org/polypheny/simpleclient/scenario/Scenario.java
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,25 @@ protected double calculateMean( List<Long> times ) {
}


protected double calculateSampleStandardDeviation( List<Long> times, double mean ) {
double preVariance = times.stream().mapToDouble( it -> Math.pow( ( it - mean ), 2 ) ).sum();
double variance = preVariance / ( times.size() - 1.0 );
return Math.sqrt( variance );
}


protected double processDoubleValue( double value ) {
DecimalFormat df = new DecimalFormat( "0.000" );
double temp1 = value / 1_000_000;
String roundFormat = df.format( value );
try {
return df.parse( roundFormat ).doubleValue();
} catch ( ParseException e ) {
log.error( "Exception", e );
}
return -1;
}

public abstract int getNumberOfInsertThreads();

}
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,9 @@
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.LongSummaryStatistics;
import java.util.Map;
import java.util.OptionalDouble;
import java.util.Properties;
import java.util.Vector;
import java.util.concurrent.ConcurrentHashMap;
Expand All @@ -30,6 +32,8 @@
import org.polypheny.simpleclient.scenario.knnbench.queryBuilder.CreateRealFeature;
import org.polypheny.simpleclient.scenario.knnbench.queryBuilder.MetadataKnnIntFeature;
import org.polypheny.simpleclient.scenario.knnbench.queryBuilder.MetadataKnnRealFeature;
import org.polypheny.simpleclient.scenario.knnbench.queryBuilder.SimpleKnnIdIntFeature;
import org.polypheny.simpleclient.scenario.knnbench.queryBuilder.SimpleKnnIdRealFeature;
import org.polypheny.simpleclient.scenario.knnbench.queryBuilder.SimpleKnnIntFeature;
import org.polypheny.simpleclient.scenario.knnbench.queryBuilder.SimpleKnnRealFeature;

Expand Down Expand Up @@ -97,6 +101,8 @@ public long execute( ProgressReporter progressReporter, CsvWriter csvWriter, Fil
List<QueryListEntry> queryList = new Vector<>();
addNumberOfTimes( queryList, new SimpleKnnIntFeature( config.randomSeedQuery, config.dimensionFeatureVectors, config.limitKnnQueries, config.distanceNorm ), config.numberOfSimpleKnnIntFeatureQueries );
addNumberOfTimes( queryList, new SimpleKnnRealFeature( config.randomSeedQuery, config.dimensionFeatureVectors, config.limitKnnQueries, config.distanceNorm ), config.numberOfSimpleKnnRealFeatureQueries );
addNumberOfTimes( queryList, new SimpleKnnIdIntFeature( config.randomSeedQuery, config.dimensionFeatureVectors, config.limitKnnQueries, config.distanceNorm ), config.numberOfSimpleKnnIdIntFeatureQueries );
addNumberOfTimes( queryList, new SimpleKnnIdRealFeature( config.randomSeedQuery, config.dimensionFeatureVectors, config.limitKnnQueries, config.distanceNorm ), config.numberOfSimpleKnnIdRealFeatureQueries );
addNumberOfTimes( queryList, new MetadataKnnIntFeature( config.randomSeedQuery, config.dimensionFeatureVectors, config.limitKnnQueries, config.distanceNorm ), config.numberOfMetadataKnnIntFeatureQueries );
addNumberOfTimes( queryList, new MetadataKnnRealFeature( config.randomSeedQuery, config.dimensionFeatureVectors, config.limitKnnQueries, config.distanceNorm ), config.numberOfMetadataKnnRealFeatureQueries );

Expand Down Expand Up @@ -148,32 +154,6 @@ public long execute( ProgressReporter progressReporter, CsvWriter csvWriter, Fil
throw new RuntimeException( "Exception while executing benchmark", threadMonitor.exception );
}

/*Executor executor = executorFactory.createInstance( csvWriter );
SimpleKnnIntFeature simpleKnnIntFeatureBuilder = new SimpleKnnIntFeature( config.randomSeedQuery, config.dimensionFeatureVectors, config.limitKnnQueries, config.distanceNorm );
SimpleKnnRealFeature simpleKnnRealFeatureBuilder = new SimpleKnnRealFeature( config.randomSeedQuery, config.dimensionFeatureVectors, config.limitKnnQueries, config.distanceNorm );
MetadataKnnIntFeature metadataKnnIntFeature = new MetadataKnnIntFeature( config.randomSeedQuery, config.dimensionFeatureVectors, config.limitKnnQueries, config.distanceNorm );
MetadataKnnRealFeature metadataKnnRealFeature = new MetadataKnnRealFeature( config.randomSeedQuery, config.dimensionFeatureVectors, config.limitKnnQueries, config.distanceNorm );
try {
for ( int i = 0; i < config.numberOfPureKnnQueries; i++ ) {
executor.executeQuery( simpleKnnIntFeatureBuilder.getNewQuery() );
}
for ( int i = 0; i < config.numberOfPureKnnQueries; i++ ) {
executor.executeQuery( simpleKnnRealFeatureBuilder.getNewQuery() );
}
for ( int i = 0; i < config.numberOfPureKnnQueries; i++ ) {
executor.executeQuery( metadataKnnIntFeature.getNewQuery() );
}
for ( int i = 0; i < config.numberOfPureKnnQueries; i++ ) {
executor.executeQuery( metadataKnnRealFeature.getNewQuery() );
}
executor.flushCsvWriter();
} catch ( ExecutorException e ) {
throw new RuntimeException( "Error occured during workload.", e );
} finally {
commitAndCloseExecutor( executor );
}*/

long runTime = System.nanoTime() - startTime;
log.info( "run time: {} s", runTime / 1000000000 );

Expand All @@ -192,6 +172,8 @@ public void warmUp( ProgressReporter progressReporter, int iterations ) {
Executor executor = null;
SimpleKnnIntFeature simpleKnnIntFeatureBuilder = new SimpleKnnIntFeature( config.randomSeedQuery, config.dimensionFeatureVectors, config.limitKnnQueries, config.distanceNorm );
SimpleKnnRealFeature simpleKnnRealFeatureBuilder = new SimpleKnnRealFeature( config.randomSeedQuery, config.dimensionFeatureVectors, config.limitKnnQueries, config.distanceNorm );
SimpleKnnIdIntFeature simpleKnnIdIntFeatureBuilder = new SimpleKnnIdIntFeature( config.randomSeedQuery, config.dimensionFeatureVectors, config.limitKnnQueries, config.distanceNorm );
SimpleKnnIdRealFeature simpleKnnIdRealFeatureBuilder = new SimpleKnnIdRealFeature( config.randomSeedQuery, config.dimensionFeatureVectors, config.limitKnnQueries, config.distanceNorm );
MetadataKnnIntFeature metadataKnnIntFeature = new MetadataKnnIntFeature( config.randomSeedQuery, config.dimensionFeatureVectors, config.limitKnnQueries, config.distanceNorm );
MetadataKnnRealFeature metadataKnnRealFeature = new MetadataKnnRealFeature( config.randomSeedQuery, config.dimensionFeatureVectors, config.limitKnnQueries, config.distanceNorm );

Expand All @@ -204,6 +186,12 @@ public void warmUp( ProgressReporter progressReporter, int iterations ) {
if ( config.numberOfSimpleKnnRealFeatureQueries > 0 ) {
executor.executeQuery( simpleKnnRealFeatureBuilder.getNewQuery() );
}
if ( config.numberOfSimpleKnnIdIntFeatureQueries > 0 ) {
executor.executeQuery( simpleKnnIdIntFeatureBuilder.getNewQuery() );
}
if ( config.numberOfSimpleKnnIdRealFeatureQueries > 0 ) {
executor.executeQuery( simpleKnnIdRealFeatureBuilder.getNewQuery() );
}
if ( config.numberOfMetadataKnnIntFeatureQueries > 0 ) {
executor.executeQuery( metadataKnnIntFeature.getNewQuery() );
}
Expand Down Expand Up @@ -349,16 +337,33 @@ public void analyze( Properties properties ) {
properties.put( "measuredTime", calculateMean( measuredTimes ) );

measuredTimePerQueryType.forEach( ( templateId, time ) -> {
properties.put( "queryTypes_" + templateId + "_mean", calculateMean( time ) );
if ( ChronosAgent.STORE_INDIVIDUAL_QUERY_TIMES ) {
properties.put( "queryTypes_" + templateId + "_all", Joiner.on( ',' ).join( time ) );
}
properties.put( "queryTypes_" + templateId + "_example", queryTypes.get( templateId ) );
calculateResults( properties, templateId, time );
} );
properties.put( "queryTypes_maxId", queryTypes.size() );
}



private void calculateResults( Properties properties, int templateId, List<Long> time ) {
LongSummaryStatistics summaryStatistics = time.stream().mapToLong( Long::longValue ).summaryStatistics();
double mean = summaryStatistics.getAverage();
long max = summaryStatistics.getMax();
long min = summaryStatistics.getMin();
double stddev = calculateSampleStandardDeviation( time, mean );

properties.put( "queryTypes_" + templateId + "_mean", processDoubleValue( mean ) );
if ( ChronosAgent.STORE_INDIVIDUAL_QUERY_TIMES ) {
properties.put( "queryTypes_" + templateId + "_all", Joiner.on( ',' ).join( time ) );
}
properties.put( "queryTypes_" + templateId + "_stddev", processDoubleValue( stddev ) );
properties.put( "queryTypes_" + templateId + "_min", min / 1_000_000L );
properties.put( "queryTypes_" + templateId + "_max", max / 1_000_000L );
properties.put( "queryTypes_" + templateId + "_example", queryTypes.get( templateId ) );
}




@Override
public int getNumberOfInsertThreads() {
return 1;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ public class KnnBenchConfig extends AbstractConfig {
public final int numberOfEntries;
public final int numberOfSimpleKnnIntFeatureQueries;
public final int numberOfSimpleKnnRealFeatureQueries;
public final int numberOfSimpleKnnIdIntFeatureQueries;
public final int numberOfSimpleKnnIdRealFeatureQueries;
public final int numberOfMetadataKnnIntFeatureQueries;
public final int numberOfMetadataKnnRealFeatureQueries;
// public final int numberOfCombinedQueries;
Expand Down Expand Up @@ -67,9 +69,10 @@ public KnnBenchConfig( Properties properties, int multiplier ) {
batchSizeQueries = getIntProperty( properties, "batchSizeQueries" );
numberOfSimpleKnnIntFeatureQueries = getIntProperty( properties, "numberOfSimpleKnnIntFeatureQueries" ) * multiplier;
numberOfSimpleKnnRealFeatureQueries = getIntProperty( properties, "numberOfSimpleKnnRealFeatureQueries" ) * multiplier;
numberOfSimpleKnnIdIntFeatureQueries = getIntProperty( properties, "numberOfSimpleKnnIdIntFeatureQueries" ) * multiplier;
numberOfSimpleKnnIdRealFeatureQueries = getIntProperty( properties, "numberOfSimpleKnnIdRealFeatureQueries" ) * multiplier;
numberOfMetadataKnnIntFeatureQueries = getIntProperty( properties, "numberOfMetadataKnnIntFeatureQueries" ) * multiplier;
numberOfMetadataKnnRealFeatureQueries = getIntProperty( properties, "numberOfMetadataKnnRealFeatureQueries" ) * multiplier;
// numberOfCombinedQueries = getIntProperty( properties, "numberOfCombinedQueries" ) * multiplier;
limitKnnQueries = getIntProperty( properties, "limitKnnQueries" );
distanceNorm = getStringProperty( properties, "distanceNorm" );
}
Expand Down Expand Up @@ -115,6 +118,8 @@ public KnnBenchConfig( Map<String, String> cdl ) {
batchSizeQueries = Integer.parseInt( cdl.get( "batchSizeQueries" ) );
numberOfSimpleKnnIntFeatureQueries = Integer.parseInt( cdl.get( "numberOfSimpleKnnIntFeatureQueries" ) );
numberOfSimpleKnnRealFeatureQueries = Integer.parseInt( cdl.get( "numberOfSimpleKnnRealFeatureQueries" ) );
numberOfSimpleKnnIdIntFeatureQueries = Integer.parseInt( cdl.get( "numberOfSimpleKnnIdIntFeatureQueries" ) );
numberOfSimpleKnnIdRealFeatureQueries = Integer.parseInt( cdl.get( "numberOfSimpleKnnIdRealFeatureQueries" ) );
numberOfMetadataKnnIntFeatureQueries = Integer.parseInt( cdl.get( "numberOfMetadataKnnIntFeatureQueries" ) );
numberOfMetadataKnnRealFeatureQueries = Integer.parseInt( cdl.get( "numberOfMetadataKnnRealFeatureQueries" ) );
// numberOfCombinedQueries = getIntProperty( properties, "numberOfCombinedQueries" ) * multiplier;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
package org.polypheny.simpleclient.scenario.knnbench.queryBuilder;


import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.Random;
import kong.unirest.HttpRequest;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.polypheny.simpleclient.query.CottontailQuery;
import org.polypheny.simpleclient.query.CottontailQuery.QueryType;
import org.polypheny.simpleclient.query.Query;
import org.polypheny.simpleclient.query.QueryBuilder;
import org.vitrivr.cottontail.grpc.CottontailGrpc;
import org.vitrivr.cottontail.grpc.CottontailGrpc.Entity;
import org.vitrivr.cottontail.grpc.CottontailGrpc.From;
import org.vitrivr.cottontail.grpc.CottontailGrpc.IntVector;
import org.vitrivr.cottontail.grpc.CottontailGrpc.Knn;
import org.vitrivr.cottontail.grpc.CottontailGrpc.Knn.Distance;
import org.vitrivr.cottontail.grpc.CottontailGrpc.Projection;
import org.vitrivr.cottontail.grpc.CottontailGrpc.Schema;
import org.vitrivr.cottontail.grpc.CottontailGrpc.Vector;


public class SimpleKnnIdIntFeature extends QueryBuilder {

private static final boolean EXPECT_RESULT = true;

private final long randomSeed;
private final int dimension;
private final int limit;
private final String norm;

private final Random random;


public SimpleKnnIdIntFeature( long randomSeed, int dimension, int limit, String norm ) {
this.randomSeed = randomSeed;
this.dimension = dimension;

this.random = new Random( randomSeed );
this.limit = limit;
this.norm = norm;
}


private Integer[] getRandomVector() {
Integer[] integers = new Integer[this.dimension];
for ( int i = 0; i < this.dimension; i++ ) {
integers[i] = random.nextInt( 500 );
}

return integers;
}


@Override
public synchronized Query getNewQuery() {
return new SimpleKnnIdIntFeatureQuery(
getRandomVector(),
limit,
norm
);
}


private static class SimpleKnnIdIntFeatureQuery extends Query {

private static final String SQL_1 = "SELECT id FROM knn_intfeature ORDER BY distance(feature, ";
private static final String SQL_2 = ", ";
private static final String SQL_3 = ") ASC LIMIT ";

private final Integer[] target;
private final int limit;
private final String norm;


public SimpleKnnIdIntFeatureQuery( Integer[] target, int limit, String norm ) {
super( EXPECT_RESULT );
this.target = target;
this.limit = limit;
this.norm = norm;
}


@Override
public String getSql() {
return SQL_1 + "ARRAY" + Arrays.toString( target ) + SQL_2 + "'" + norm + "'" + SQL_3 + limit;
}


@Override
public String getParameterizedSqlQuery() {
return SQL_1 + "?" + SQL_2 + "'" + norm + "'" + SQL_3 + limit;
}


@Override
public Map<Integer, ImmutablePair<DataTypes, Object>> getParameterValues() {
Map<Integer, ImmutablePair<DataTypes, Object>> map = new HashMap<>();
map.put( 1, new ImmutablePair<>( DataTypes.ARRAY_INT, target ) );
return map;
}


@Override
public HttpRequest<?> getRest() {
return null;
}


@Override
public CottontailQuery getCottontail() {
Map<String, String> projection = new HashMap<>();
projection.put( "id", "id" );
CottontailGrpc.Query query = CottontailGrpc.Query.newBuilder()
.setFrom( From.newBuilder().setEntity( Entity.newBuilder().setSchema( Schema.newBuilder().setName( "public" ).build() ).setName( "knn_intfeature" ).build() ) )
.setLimit( limit )
.setKnn( Knn.newBuilder()
.setAttribute( "feature" )
.setK( limit )
.addQuery( Vector.newBuilder().setIntVector( IntVector.newBuilder().addAllVector( Arrays.asList( target ) ).build() ).build() )
.setDistance( getDistance( norm ) )
.build() )
.setProjection( Projection.newBuilder().putAllAttributes( projection ).build() )
.build();
return new CottontailQuery(
QueryType.QUERY,
query
);
}


private static Distance getDistance( String norm ) {
if ( "L2".equalsIgnoreCase( norm ) ) {
return Distance.L2;
}
if ( "L1".equalsIgnoreCase( norm ) ) {
return Distance.L1;
}
if ( "L2SQUARED".equalsIgnoreCase( norm ) ) {
return Distance.L2SQUARED;
}
if ( "CHISQUARED".equalsIgnoreCase( norm ) ) {
return Distance.CHISQUARED;
}
if ( "COSINE".equalsIgnoreCase( norm ) ) {
return Distance.COSINE;
}

throw new RuntimeException( "Unsupported norm: " + norm );
}
}

}
Loading

0 comments on commit e8a4131

Please sign in to comment.