Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/main' into esql_points_from_sour…
Browse files Browse the repository at this point in the history 
…ce_wkb
  • Loading branch information
@craigtaverner
craigtaverner committed Dec 28, 2023
2 parents 4cbac42 + d16ea55 commit 3b0c89d
Show file tree
Hide file tree
Showing 416 changed files with 7,553 additions and 3,542 deletions.
25 changes: 13 additions & 12 deletions ...marks/src/main/java/org/elasticsearch/benchmark/compute/operator/AggregatorBenchmark.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,10 @@ public class AggregatorBenchmark {
private static final int GROUPS = 5;

private static final BigArrays BIG_ARRAYS = BigArrays.NON_RECYCLING_INSTANCE; // TODO real big arrays?
private static final BlockFactory blockFactory = BlockFactory.getInstance(
new NoopCircuitBreaker("noop"),
BigArrays.NON_RECYCLING_INSTANCE
);

private static final String LONGS = "longs";
private static final String INTS = "ints";
Expand Down Expand Up @@ -446,7 +450,7 @@ private static Block dataBlock(BlockFactory blockFactory, String blockType) {
BLOCK_LENGTH
).asBlock();
case MULTIVALUED_LONGS -> {
var builder = LongBlock.newBlockBuilder(BLOCK_LENGTH);
var builder = blockFactory.newLongBlockBuilder(BLOCK_LENGTH);
builder.beginPositionEntry();
for (int i = 0; i < BLOCK_LENGTH; i++) {
builder.appendLong(i);
Expand All @@ -459,15 +463,15 @@ private static Block dataBlock(BlockFactory blockFactory, String blockType) {
yield builder.build();
}
case HALF_NULL_LONGS -> {
var builder = LongBlock.newBlockBuilder(BLOCK_LENGTH);
var builder = blockFactory.newLongBlockBuilder(BLOCK_LENGTH);
for (int i = 0; i < BLOCK_LENGTH; i++) {
builder.appendLong(i);
builder.appendNull();
}
yield builder.build();
}
case HALF_NULL_DOUBLES -> {
var builder = DoubleBlock.newBlockBuilder(BLOCK_LENGTH);
var builder = blockFactory.newDoubleBlockBuilder(BLOCK_LENGTH);
for (int i = 0; i < BLOCK_LENGTH; i++) {
builder.appendDouble(i);
builder.appendNull();
Expand Down Expand Up @@ -499,7 +503,7 @@ private static Block groupingBlock(String grouping, String blockType) {
};
return switch (grouping) {
case LONGS -> {
var builder = LongBlock.newBlockBuilder(BLOCK_LENGTH);
var builder = blockFactory.newLongBlockBuilder(BLOCK_LENGTH);
for (int i = 0; i < BLOCK_LENGTH; i++) {
for (int v = 0; v < valuesPerGroup; v++) {
builder.appendLong(i % GROUPS);
Expand All @@ -508,7 +512,7 @@ private static Block groupingBlock(String grouping, String blockType) {
yield builder.build();
}
case INTS -> {
var builder = IntBlock.newBlockBuilder(BLOCK_LENGTH);
var builder = blockFactory.newIntBlockBuilder(BLOCK_LENGTH);
for (int i = 0; i < BLOCK_LENGTH; i++) {
for (int v = 0; v < valuesPerGroup; v++) {
builder.appendInt(i % GROUPS);
Expand All @@ -517,7 +521,7 @@ private static Block groupingBlock(String grouping, String blockType) {
yield builder.build();
}
case DOUBLES -> {
var builder = DoubleBlock.newBlockBuilder(BLOCK_LENGTH);
var builder = blockFactory.newDoubleBlockBuilder(BLOCK_LENGTH);
for (int i = 0; i < BLOCK_LENGTH; i++) {
for (int v = 0; v < valuesPerGroup; v++) {
builder.appendDouble(i % GROUPS);
Expand All @@ -526,7 +530,7 @@ private static Block groupingBlock(String grouping, String blockType) {
yield builder.build();
}
case BOOLEANS -> {
BooleanBlock.Builder builder = BooleanBlock.newBlockBuilder(BLOCK_LENGTH);
BooleanBlock.Builder builder = blockFactory.newBooleanBlockBuilder(BLOCK_LENGTH);
for (int i = 0; i < BLOCK_LENGTH; i++) {
for (int v = 0; v < valuesPerGroup; v++) {
builder.appendBoolean(i % 2 == 1);
Expand All @@ -535,7 +539,7 @@ private static Block groupingBlock(String grouping, String blockType) {
yield builder.build();
}
case BYTES_REFS -> {
BytesRefBlock.Builder builder = BytesRefBlock.newBlockBuilder(BLOCK_LENGTH);
BytesRefBlock.Builder builder = blockFactory.newBytesRefBlockBuilder(BLOCK_LENGTH);
for (int i = 0; i < BLOCK_LENGTH; i++) {
for (int v = 0; v < valuesPerGroup; v++) {
builder.appendBytesRef(bytesGroup(i % GROUPS));
Expand Down Expand Up @@ -582,9 +586,6 @@ private static void run(String grouping, String op, String blockType, int opCoun
}

static DriverContext driverContext() {
return new DriverContext(
BigArrays.NON_RECYCLING_INSTANCE,
BlockFactory.getInstance(new NoopCircuitBreaker("noop"), BigArrays.NON_RECYCLING_INSTANCE)
);
return new DriverContext(BigArrays.NON_RECYCLING_INSTANCE, blockFactory);
}
}
105 changes: 86 additions & 19 deletions benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/BlockBenchmark.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
package org.elasticsearch.benchmark.compute.operator;

import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.breaker.NoopCircuitBreaker;
import org.elasticsearch.common.util.BigArrays;
import org.elasticsearch.common.util.BitArray;
import org.elasticsearch.common.util.BytesRefArray;
Expand Down Expand Up @@ -75,6 +76,7 @@ public class BlockBenchmark {
public static final String[] RELEVANT_TYPE_BLOCK_COMBINATIONS = {
"boolean/array",
"boolean/array-multivalue-null",
"boolean/big-array",
"boolean/big-array-multivalue-null",
"boolean/vector",
"boolean/vector-big-array",
Expand All @@ -85,18 +87,21 @@ public class BlockBenchmark {
"BytesRef/vector-const",
"double/array",
"double/array-multivalue-null",
"double/big-array",
"double/big-array-multivalue-null",
"double/vector",
"double/vector-big-array",
"double/vector-const",
"int/array",
"int/array-multivalue-null",
"int/big-array",
"int/big-array-multivalue-null",
"int/vector",
"int/vector-big-array",
"int/vector-const",
"long/array",
"long/array-multivalue-null",
"long/big-array",
"long/big-array-multivalue-null",
"long/vector",
"long/vector-big-array",
Expand All @@ -111,6 +116,11 @@ public class BlockBenchmark {

private static final Random random = new Random();

private static final BlockFactory blockFactory = BlockFactory.getInstance(
new NoopCircuitBreaker("noop"),
BigArrays.NON_RECYCLING_INSTANCE
);

static {
// Smoke test all the expected values and force loading subclasses more like prod
int totalPositions = 10;
Expand All @@ -131,7 +141,6 @@ public class BlockBenchmark {
private record BenchmarkBlocks(Block[] blocks, long[] checkSums) {};

private static BenchmarkBlocks buildBlocks(String dataType, String blockKind, int totalPositions) {
BlockFactory blockFactory = BlockFactory.getNonBreakingInstance();
Block[] blocks = new Block[NUM_BLOCKS_PER_ITERATION];
long[] checkSums = new long[NUM_BLOCKS_PER_ITERATION];

Expand Down Expand Up @@ -172,6 +181,23 @@ private static BenchmarkBlocks buildBlocks(String dataType, String blockKind, in
Block.MvOrdering.UNORDERED
);
}
case "big-array" -> {
BitArray valuesBigArray = new BitArray(totalPositions, BigArrays.NON_RECYCLING_INSTANCE);
for (int i = 0; i < values.length; i++) {
if (values[i]) {
valuesBigArray.set(i);
}
}

blocks[blockIndex] = new BooleanBigArrayBlock(
valuesBigArray,
totalPositions,
null,
null,
Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING,
blockFactory
);
}
case "big-array-multivalue-null" -> {
int[] firstValueIndexes = randomFirstValueIndexes(totalPositions);
int positionCount = firstValueIndexes.length - 1;
Expand All @@ -189,7 +215,7 @@ private static BenchmarkBlocks buildBlocks(String dataType, String blockKind, in
firstValueIndexes,
nulls,
Block.MvOrdering.UNORDERED,
BlockFactory.getNonBreakingInstance()
blockFactory
);
}
case "vector" -> {
Expand Down Expand Up @@ -310,13 +336,26 @@ private static BenchmarkBlocks buildBlocks(String dataType, String blockKind, in
Block.MvOrdering.UNORDERED
);
}
case "big-array" -> {
DoubleArray valuesBigArray = blockFactory.bigArrays().newDoubleArray(totalPositions, false);
for (int i = 0; i < values.length; i++) {
valuesBigArray.set(i, values[i]);
}

blocks[blockIndex] = new DoubleBigArrayBlock(
valuesBigArray,
totalPositions,
null,
null,
Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING,
blockFactory
);
}
case "big-array-multivalue-null" -> {
int[] firstValueIndexes = randomFirstValueIndexes(totalPositions);
int positionCount = firstValueIndexes.length - 1;
BitSet nulls = randomNulls(positionCount);
DoubleArray valuesBigArray = BlockFactory.getNonBreakingInstance()
.bigArrays()
.newDoubleArray(totalPositions, false);
DoubleArray valuesBigArray = blockFactory.bigArrays().newDoubleArray(totalPositions, false);
for (int i = 0; i < values.length; i++) {
valuesBigArray.set(i, values[i]);
}
Expand All @@ -327,17 +366,15 @@ private static BenchmarkBlocks buildBlocks(String dataType, String blockKind, in
firstValueIndexes,
nulls,
Block.MvOrdering.UNORDERED,
BlockFactory.getNonBreakingInstance()
blockFactory
);
}
case "vector" -> {
DoubleVector vector = blockFactory.newDoubleArrayVector(values, totalPositions);
blocks[blockIndex] = vector.asBlock();
}
case "vector-big-array" -> {
DoubleArray valuesBigArray = BlockFactory.getNonBreakingInstance()
.bigArrays()
.newDoubleArray(totalPositions, false);
DoubleArray valuesBigArray = blockFactory.bigArrays().newDoubleArray(totalPositions, false);
for (int i = 0; i < values.length; i++) {
valuesBigArray.set(i, values[i]);
}
Expand Down Expand Up @@ -391,11 +428,26 @@ private static BenchmarkBlocks buildBlocks(String dataType, String blockKind, in
Block.MvOrdering.UNORDERED
);
}
case "big-array" -> {
IntArray valuesBigArray = blockFactory.bigArrays().newIntArray(totalPositions, false);
for (int i = 0; i < values.length; i++) {
valuesBigArray.set(i, values[i]);
}

blocks[blockIndex] = new IntBigArrayBlock(
valuesBigArray,
totalPositions,
null,
null,
Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING,
blockFactory
);
}
case "big-array-multivalue-null" -> {
int[] firstValueIndexes = randomFirstValueIndexes(totalPositions);
int positionCount = firstValueIndexes.length - 1;
BitSet nulls = randomNulls(positionCount);
IntArray valuesBigArray = BlockFactory.getNonBreakingInstance().bigArrays().newIntArray(totalPositions, false);
IntArray valuesBigArray = blockFactory.bigArrays().newIntArray(totalPositions, false);
for (int i = 0; i < values.length; i++) {
valuesBigArray.set(i, values[i]);
}
Expand All @@ -406,15 +458,15 @@ private static BenchmarkBlocks buildBlocks(String dataType, String blockKind, in
firstValueIndexes,
nulls,
Block.MvOrdering.UNORDERED,
BlockFactory.getNonBreakingInstance()
blockFactory
);
}
case "vector" -> {
IntVector vector = blockFactory.newIntArrayVector(values, totalPositions);
blocks[blockIndex] = vector.asBlock();
}
case "vector-big-array" -> {
IntArray valuesBigArray = BlockFactory.getNonBreakingInstance().bigArrays().newIntArray(totalPositions, false);
IntArray valuesBigArray = blockFactory.bigArrays().newIntArray(totalPositions, false);
for (int i = 0; i < values.length; i++) {
valuesBigArray.set(i, values[i]);
}
Expand Down Expand Up @@ -468,13 +520,26 @@ private static BenchmarkBlocks buildBlocks(String dataType, String blockKind, in
Block.MvOrdering.UNORDERED
);
}
case "big-array" -> {
LongArray valuesBigArray = blockFactory.bigArrays().newLongArray(totalPositions, false);
for (int i = 0; i < values.length; i++) {
valuesBigArray.set(i, values[i]);
}

blocks[blockIndex] = new LongBigArrayBlock(
valuesBigArray,
totalPositions,
null,
null,
Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING,
blockFactory
);
}
case "big-array-multivalue-null" -> {
int[] firstValueIndexes = randomFirstValueIndexes(totalPositions);
int positionCount = firstValueIndexes.length - 1;
BitSet nulls = randomNulls(positionCount);
LongArray valuesBigArray = BlockFactory.getNonBreakingInstance()
.bigArrays()
.newLongArray(totalPositions, false);
LongArray valuesBigArray = blockFactory.bigArrays().newLongArray(totalPositions, false);
for (int i = 0; i < values.length; i++) {
valuesBigArray.set(i, values[i]);
}
Expand All @@ -485,17 +550,15 @@ private static BenchmarkBlocks buildBlocks(String dataType, String blockKind, in
firstValueIndexes,
nulls,
Block.MvOrdering.UNORDERED,
BlockFactory.getNonBreakingInstance()
blockFactory
);
}
case "vector" -> {
LongVector vector = blockFactory.newLongArrayVector(values, totalPositions);
blocks[blockIndex] = vector.asBlock();
}
case "vector-big-array" -> {
LongArray valuesBigArray = BlockFactory.getNonBreakingInstance()
.bigArrays()
.newLongArray(totalPositions, false);
LongArray valuesBigArray = blockFactory.bigArrays().newLongArray(totalPositions, false);
for (int i = 0; i < values.length; i++) {
valuesBigArray.set(i, values[i]);
}
Expand Down Expand Up @@ -718,6 +781,7 @@ private static boolean isRandom(String accessType) {
{
"boolean/array",
"boolean/array-multivalue-null",
"boolean/big-array",
"boolean/big-array-multivalue-null",
"boolean/vector",
"boolean/vector-big-array",
Expand All @@ -728,18 +792,21 @@ private static boolean isRandom(String accessType) {
"BytesRef/vector-const",
"double/array",
"double/array-multivalue-null",
"double/big-array",
"double/big-array-multivalue-null",
"double/vector",
"double/vector-big-array",
"double/vector-const",
"int/array",
"int/array-multivalue-null",
"int/big-array",
"int/big-array-multivalue-null",
"int/vector",
"int/vector-big-array",
"int/vector-const",
"long/array",
"long/array-multivalue-null",
"long/big-array",
"long/big-array-multivalue-null",
"long/vector",
"long/vector-big-array",
Expand Down
Loading

0 comments on commit 3b0c89d

Please sign in to comment.