mirrors
/
elasticsearch
mirror of https://mirror.osredm.com/root/elasticsearch.git
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

ESQL: Keep ordinals in conversion functions (#125357) 

Make the conversion functions that process `BytesRef`s into `BytesRefs`
keep the `OrdinalBytesRefVector`s when processing. Let's use `TO_LOWER`
as an example. First, the performance numbers:
```
  (operation)  Mode   Score   Error ->  Score    Error Units
     to_lower  30.662 ± 6.163 -> 30.048 ±  0.479 ns/op
to_lower_ords  30.773 ± 0.370 ->  0.025 ±  0.001 ns/op
     to_upper  33.552 ± 0.529 -> 35.775 ±  1.799 ns/op
to_upper_ords  35.791 ± 0.658 ->  0.027 ±  0.001 ns/op
```
The test has a 8192 positions containing alternating `foo` and `bar`.
Running `TO_LOWER` via ordinals is super duper faster. No longer
`O(positions)` and now `O(unique_values)`.

Let's paint some pictures! `OrdinalBytesRefVector` is a lookup table.
Like this:
```
+-------+----------+
| bytes | ordinals |
| ----- | -------- |
|  FOO  | 0        |
|  BAR  | 1        |
|  BAZ  | 2        |
+-------+ 1        |
        | 1        |
        | 0        |
        +----------+
```

That lookup table is one block. When you read it you look up the
`ordinal` and match it to the `bytes`. Previously `TO_LOWER` would
process each value one at a time and make:
```
bytes
-----
 foo
 bar
 baz
 bar
 bar
 foo
```

So it'd run `TO_LOWER` once per `ordinal` and it'd make an ordinal
non-lookup table. With this change `TO_LOWER` will now make:
```
+-------+----------+
| bytes | ordinals |
| ----- | -------- |
|  foo  | 0        |
|  bar  | 1        |
|  baz  | 2        |
+-------+ 1        |
        | 1        |
        | 0        |
        +----------+
```
We don't even have to copy the `ordinals` - we can reuse those from the
input and just bump the reference count. That's why this goes from
`O(positions)` to `O(unique_values)`.
This commit is contained in:
Nik Everett 2025-03-21 14:00:15 -04:00 committed by GitHub
parent 0930a75642
commit c5e76847ad
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
17 changed files with 370 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java
View File

@ -24,6 +24,7 @@ import org.elasticsearch.compute.data.DoubleBlock;
import org.elasticsearch.compute.data.DoubleVector;
import org.elasticsearch.compute.data.LongBlock;
import org.elasticsearch.compute.data.LongVector;
import org.elasticsearch.compute.data.OrdinalBytesRefVector;
import org.elasticsearch.compute.data.Page;
import org.elasticsearch.compute.operator.DriverContext;
import org.elasticsearch.compute.operator.EvalOperator;
@ -127,7 +128,9 @@ public class EvalBenchmark {
"mv_min_ascending",
"rlike",
"to_lower",
"to_upper" }
"to_lower_ords",
"to_upper",
"to_upper_ords" }
)
public String operation;
@ -235,12 +238,12 @@ public class EvalBenchmark {
RLike rlike = new RLike(Source.EMPTY, keywordField, new RLikePattern(".ar"));
yield EvalMapper.toEvaluator(FOLD_CONTEXT, rlike, layout(keywordField)).get(driverContext);
}
case "to_lower" -> {
case "to_lower", "to_lower_ords" -> {
FieldAttribute keywordField = keywordField();
ToLower toLower = new ToLower(Source.EMPTY, keywordField, configuration());
yield EvalMapper.toEvaluator(FOLD_CONTEXT, toLower, layout(keywordField)).get(driverContext);
}
case "to_upper" -> {
case "to_upper", "to_upper_ords" -> {
FieldAttribute keywordField = keywordField();
ToUpper toUpper = new ToUpper(Source.EMPTY, keywordField, configuration());
yield EvalMapper.toEvaluator(FOLD_CONTEXT, toUpper, layout(keywordField)).get(driverContext);
@ -414,13 +417,15 @@ public class EvalBenchmark {
}
}
}
case "to_lower" -> checkBytes(operation, actual, new BytesRef[] { new BytesRef("foo"), new BytesRef("bar") });
case "to_upper" -> checkBytes(operation, actual, new BytesRef[] { new BytesRef("FOO"), new BytesRef("BAR") });
case "to_lower" -> checkBytes(operation, actual, false, new BytesRef[] { new BytesRef("foo"), new BytesRef("bar") });
case "to_lower_ords" -> checkBytes(operation, actual, true, new BytesRef[] { new BytesRef("foo"), new BytesRef("bar") });
case "to_upper" -> checkBytes(operation, actual, false, new BytesRef[] { new BytesRef("FOO"), new BytesRef("BAR") });
case "to_upper_ords" -> checkBytes(operation, actual, true, new BytesRef[] { new BytesRef("FOO"), new BytesRef("BAR") });
default -> throw new UnsupportedOperationException(operation);
}
}
private static void checkBytes(String operation, Page actual, BytesRef[] expectedVals) {
private static void checkBytes(String operation, Page actual, boolean expectOrds, BytesRef[] expectedVals) {
BytesRef scratch = new BytesRef();
BytesRefVector v = actual.<BytesRefBlock>getBlock(1).asVector();
for (int i = 0; i < BLOCK_LENGTH; i++) {
@ -430,6 +435,15 @@ public class EvalBenchmark {
throw new AssertionError("[" + operation + "] expected [" + expected + "] but was [" + b + "]");
}
}
if (expectOrds) {
if (v.asOrdinals() == null) {
throw new IllegalArgumentException("expected ords but got " + v);
}
} else {
if (v.asOrdinals() != null) {
throw new IllegalArgumentException("expected non-ords but got " + v);
}
}
}
private static Page page(String operation) {
@ -510,6 +524,16 @@ public class EvalBenchmark {
}
yield new Page(builder.build().asBlock());
}
case "to_lower_ords", "to_upper_ords" -> {
var bytes = blockFactory.newBytesRefVectorBuilder(BLOCK_LENGTH);
bytes.appendBytesRef(new BytesRef("foo"));
bytes.appendBytesRef(new BytesRef("bar"));
var ordinals = blockFactory.newIntVectorFixedBuilder(BLOCK_LENGTH);
for (int i = 0; i < BLOCK_LENGTH; i++) {
ordinals.appendInt(i % 2);
}
yield new Page(new OrdinalBytesRefVector(ordinals.build(), bytes.build()).asBlock());
}
default -> throw new UnsupportedOperationException();
};
}

5
docs/changelog/125357.yaml Normal file
View File

@ -0,0 +1,5 @@
pr: 125357
summary: Keep ordinals in conversion functions
area: ES|QL
type: enhancement
issues: []

47
x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/ConvertEvaluatorImplementer.java
View File

@ -28,9 +28,12 @@ import static org.elasticsearch.compute.gen.Methods.getMethod;
import static org.elasticsearch.compute.gen.Types.ABSTRACT_CONVERT_FUNCTION_EVALUATOR;
import static org.elasticsearch.compute.gen.Types.BLOCK;
import static org.elasticsearch.compute.gen.Types.BYTES_REF;
import static org.elasticsearch.compute.gen.Types.BYTES_REF_VECTOR_BUILDER;
import static org.elasticsearch.compute.gen.Types.DRIVER_CONTEXT;
import static org.elasticsearch.compute.gen.Types.EXPRESSION_EVALUATOR;
import static org.elasticsearch.compute.gen.Types.EXPRESSION_EVALUATOR_FACTORY;
import static org.elasticsearch.compute.gen.Types.INT_VECTOR;
import static org.elasticsearch.compute.gen.Types.ORDINALS_BYTES_REF_VECTOR;
import static org.elasticsearch.compute.gen.Types.SOURCE;
import static org.elasticsearch.compute.gen.Types.VECTOR;
import static org.elasticsearch.compute.gen.Types.blockType;
@ -41,7 +44,7 @@ public class ConvertEvaluatorImplementer {
private final TypeElement declarationType;
private final EvaluatorImplementer.ProcessFunction processFunction;
private final String extraName;
private final boolean canProcessOrdinals;
private final ClassName implementation;
private final TypeName argumentType;
private final List<TypeMirror> warnExceptions;
@ -55,6 +58,10 @@ public class ConvertEvaluatorImplementer {
) {
this.declarationType = (TypeElement) processFunction.getEnclosingElement();
this.processFunction = new EvaluatorImplementer.ProcessFunction(types, processFunction, warnExceptions);
this.canProcessOrdinals = warnExceptions.isEmpty()
&& this.processFunction.returnType().equals(BYTES_REF)
&& this.processFunction.args.getFirst() instanceof EvaluatorImplementer.StandardProcessFunctionArg s
&& s.type().equals(BYTES_REF);
if (this.processFunction.args.getFirst() instanceof EvaluatorImplementer.StandardProcessFunctionArg == false) {
throw new IllegalArgumentException("first argument must be the field to process");
@ -66,7 +73,6 @@ public class ConvertEvaluatorImplementer {
}
}
this.extraName = extraName;
this.argumentType = TypeName.get(processFunction.getParameters().get(0).asType());
this.warnExceptions = warnExceptions;
@ -102,6 +108,9 @@ public class ConvertEvaluatorImplementer {
builder.addMethod(evalValue(true));
builder.addMethod(evalBlock());
builder.addMethod(evalValue(false));
if (canProcessOrdinals) {
builder.addMethod(evalOrdinals());
}
builder.addMethod(processFunction.toStringMethod(implementation));
builder.addMethod(processFunction.close());
builder.addType(factory());
@ -132,6 +141,15 @@ public class ConvertEvaluatorImplementer {
TypeName vectorType = vectorType(argumentType);
builder.addStatement("$T vector = ($T) v", vectorType, vectorType);
if (canProcessOrdinals) {
builder.addStatement("$T ordinals = vector.asOrdinals()", ORDINALS_BYTES_REF_VECTOR);
builder.beginControlFlow("if (ordinals != null)");
{
builder.addStatement("return evalOrdinals(ordinals)");
}
builder.endControlFlow();
}
builder.addStatement("int positionCount = v.getPositionCount()");
String scratchPadName = argumentType.equals(BYTES_REF) ? "scratchPad" : null;
@ -299,6 +317,31 @@ public class ConvertEvaluatorImplementer {
return builder.build();
}
private MethodSpec evalOrdinals() {
MethodSpec.Builder builder = MethodSpec.methodBuilder("evalOrdinals").addModifiers(Modifier.PRIVATE);
builder.addParameter(ORDINALS_BYTES_REF_VECTOR, "v").returns(BLOCK);
builder.addStatement("int positionCount = v.getDictionaryVector().getPositionCount()");
builder.addStatement("BytesRef scratchPad = new BytesRef()");
builder.beginControlFlow(
"try ($T builder = driverContext.blockFactory().newBytesRefVectorBuilder(positionCount))",
BYTES_REF_VECTOR_BUILDER
);
{
builder.beginControlFlow("for (int p = 0; p < positionCount; p++)");
{
builder.addStatement("builder.appendBytesRef($N)", evalValueCall("v.getDictionaryVector()", "p", "scratchPad"));
}
builder.endControlFlow();
builder.addStatement("$T ordinals = v.getOrdinalsVector()", INT_VECTOR);
builder.addStatement("ordinals.incRef()");
builder.addStatement("return new $T(ordinals, builder.build()).asBlock()", ORDINALS_BYTES_REF_VECTOR);
}
builder.endControlFlow();
return builder.build();
}
private TypeSpec factory() {
TypeSpec.Builder builder = TypeSpec.classBuilder("Factory");
builder.addSuperinterface(EXPRESSION_EVALUATOR_FACTORY);

1
x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Types.java
View File

@ -61,6 +61,7 @@ public class Types {
static final ClassName BOOLEAN_VECTOR = ClassName.get(DATA_PACKAGE, "BooleanVector");
static final ClassName BYTES_REF_VECTOR = ClassName.get(DATA_PACKAGE, "BytesRefVector");
static final ClassName ORDINALS_BYTES_REF_VECTOR = ClassName.get(DATA_PACKAGE, "OrdinalBytesRefVector");
static final ClassName INT_VECTOR = ClassName.get(DATA_PACKAGE, "IntVector");
static final ClassName LONG_VECTOR = ClassName.get(DATA_PACKAGE, "LongVector");
static final ClassName DOUBLE_VECTOR = ClassName.get(DATA_PACKAGE, "DoubleVector");

69
x-pack/plugin/esql/compute/test/src/main/java/org/elasticsearch/compute/test/BlockTestUtils.java
View File

@ -13,17 +13,23 @@ import org.elasticsearch.compute.data.BlockFactory;
import org.elasticsearch.compute.data.BlockUtils;
import org.elasticsearch.compute.data.BooleanBlock;
import org.elasticsearch.compute.data.BytesRefBlock;
import org.elasticsearch.compute.data.BytesRefVector;
import org.elasticsearch.compute.data.DocBlock;
import org.elasticsearch.compute.data.DoubleBlock;
import org.elasticsearch.compute.data.ElementType;
import org.elasticsearch.compute.data.FloatBlock;
import org.elasticsearch.compute.data.IntBlock;
import org.elasticsearch.compute.data.LongBlock;
import org.elasticsearch.compute.data.OrdinalBytesRefBlock;
import org.elasticsearch.compute.data.Page;
import org.elasticsearch.core.Releasables;
import org.hamcrest.Matcher;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.elasticsearch.compute.data.BlockUtils.toJavaObject;
import static org.elasticsearch.test.ESTestCase.between;
@ -267,4 +273,67 @@ public class BlockTestUtils {
}
return result;
}
/**
* Convert all of the {@link Block}s in a page that contain {@link BytesRef}s into
* {@link OrdinalBytesRefBlock}s.
*/
public static Page convertBytesRefsToOrdinals(Page page) {
Block[] blocks = new Block[page.getBlockCount()];
try {
for (int b = 0; b < page.getBlockCount(); b++) {
Block block = page.getBlock(b);
if (block.elementType() != ElementType.BYTES_REF) {
blocks[b] = block;
continue;
}
Map<BytesRef, Integer> dedupe = new HashMap<>();
BytesRefBlock bytesRefBlock = (BytesRefBlock) block;
try (
IntBlock.Builder ordinals = block.blockFactory().newIntBlockBuilder(block.getPositionCount());
BytesRefVector.Builder bytes = block.blockFactory().newBytesRefVectorBuilder(block.getPositionCount())
) {
BytesRef scratch = new BytesRef();
for (int p = 0; p < block.getPositionCount(); p++) {
int first = block.getFirstValueIndex(p);
int count = block.getValueCount(p);
if (count == 0) {
ordinals.appendNull();
continue;
}
if (count == 1) {
BytesRef v = bytesRefBlock.getBytesRef(first, scratch);
ordinals.appendInt(dedupe(dedupe, bytes, v));
continue;
}
int end = first + count;
ordinals.beginPositionEntry();
for (int i = first; i < end; i++) {
BytesRef v = bytesRefBlock.getBytesRef(i, scratch);
ordinals.appendInt(dedupe(dedupe, bytes, v));
}
ordinals.endPositionEntry();
}
blocks[b] = new OrdinalBytesRefBlock(ordinals.build(), bytes.build());
bytesRefBlock.decRef();
}
}
Page p = new Page(blocks);
Arrays.fill(blocks, null);
return p;
} finally {
Releasables.close(blocks);
}
}
private static int dedupe(Map<BytesRef, Integer> dedupe, BytesRefVector.Builder bytes, BytesRef v) {
Integer current = dedupe.get(v);
if (current != null) {
return current;
}
bytes.appendBytesRef(v);
int o = dedupe.size();
dedupe.put(v, o);
return o;
}
}

19
x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromCartesianPointEvaluator.java generated
View File

@ -10,6 +10,8 @@ import org.apache.lucene.util.BytesRef;
import org.elasticsearch.compute.data.Block;
import org.elasticsearch.compute.data.BytesRefBlock;
import org.elasticsearch.compute.data.BytesRefVector;
import org.elasticsearch.compute.data.IntVector;
import org.elasticsearch.compute.data.OrdinalBytesRefVector;
import org.elasticsearch.compute.data.Vector;
import org.elasticsearch.compute.operator.DriverContext;
import org.elasticsearch.compute.operator.EvalOperator;
@ -37,6 +39,10 @@ public final class ToStringFromCartesianPointEvaluator extends AbstractConvertFu
@Override
public Block evalVector(Vector v) {
BytesRefVector vector = (BytesRefVector) v;
OrdinalBytesRefVector ordinals = vector.asOrdinals();
if (ordinals != null) {
return evalOrdinals(ordinals);
}
int positionCount = v.getPositionCount();
BytesRef scratchPad = new BytesRef();
if (vector.isConstant()) {
@ -91,6 +97,19 @@ public final class ToStringFromCartesianPointEvaluator extends AbstractConvertFu
return ToString.fromCartesianPoint(value);
}
private Block evalOrdinals(OrdinalBytesRefVector v) {
int positionCount = v.getDictionaryVector().getPositionCount();
BytesRef scratchPad = new BytesRef();
try (BytesRefVector.Builder builder = driverContext.blockFactory().newBytesRefVectorBuilder(positionCount)) {
for (int p = 0; p < positionCount; p++) {
builder.appendBytesRef(evalValue(v.getDictionaryVector(), p, scratchPad));
}
IntVector ordinals = v.getOrdinalsVector();
ordinals.incRef();
return new OrdinalBytesRefVector(ordinals, builder.build()).asBlock();
}
}
@Override
public String toString() {
return "ToStringFromCartesianPointEvaluator[" + "wkb=" + wkb + "]";

19
x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromCartesianShapeEvaluator.java generated
View File

@ -10,6 +10,8 @@ import org.apache.lucene.util.BytesRef;
import org.elasticsearch.compute.data.Block;
import org.elasticsearch.compute.data.BytesRefBlock;
import org.elasticsearch.compute.data.BytesRefVector;
import org.elasticsearch.compute.data.IntVector;
import org.elasticsearch.compute.data.OrdinalBytesRefVector;
import org.elasticsearch.compute.data.Vector;
import org.elasticsearch.compute.operator.DriverContext;
import org.elasticsearch.compute.operator.EvalOperator;
@ -37,6 +39,10 @@ public final class ToStringFromCartesianShapeEvaluator extends AbstractConvertFu
@Override
public Block evalVector(Vector v) {
BytesRefVector vector = (BytesRefVector) v;
OrdinalBytesRefVector ordinals = vector.asOrdinals();
if (ordinals != null) {
return evalOrdinals(ordinals);
}
int positionCount = v.getPositionCount();
BytesRef scratchPad = new BytesRef();
if (vector.isConstant()) {
@ -91,6 +97,19 @@ public final class ToStringFromCartesianShapeEvaluator extends AbstractConvertFu
return ToString.fromCartesianShape(value);
}
private Block evalOrdinals(OrdinalBytesRefVector v) {
int positionCount = v.getDictionaryVector().getPositionCount();
BytesRef scratchPad = new BytesRef();
try (BytesRefVector.Builder builder = driverContext.blockFactory().newBytesRefVectorBuilder(positionCount)) {
for (int p = 0; p < positionCount; p++) {
builder.appendBytesRef(evalValue(v.getDictionaryVector(), p, scratchPad));
}
IntVector ordinals = v.getOrdinalsVector();
ordinals.incRef();
return new OrdinalBytesRefVector(ordinals, builder.build()).asBlock();
}
}
@Override
public String toString() {
return "ToStringFromCartesianShapeEvaluator[" + "wkb=" + wkb + "]";

19
x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromGeoPointEvaluator.java generated
View File

@ -10,6 +10,8 @@ import org.apache.lucene.util.BytesRef;
import org.elasticsearch.compute.data.Block;
import org.elasticsearch.compute.data.BytesRefBlock;
import org.elasticsearch.compute.data.BytesRefVector;
import org.elasticsearch.compute.data.IntVector;
import org.elasticsearch.compute.data.OrdinalBytesRefVector;
import org.elasticsearch.compute.data.Vector;
import org.elasticsearch.compute.operator.DriverContext;
import org.elasticsearch.compute.operator.EvalOperator;
@ -37,6 +39,10 @@ public final class ToStringFromGeoPointEvaluator extends AbstractConvertFunction
@Override
public Block evalVector(Vector v) {
BytesRefVector vector = (BytesRefVector) v;
OrdinalBytesRefVector ordinals = vector.asOrdinals();
if (ordinals != null) {
return evalOrdinals(ordinals);
}
int positionCount = v.getPositionCount();
BytesRef scratchPad = new BytesRef();
if (vector.isConstant()) {
@ -91,6 +97,19 @@ public final class ToStringFromGeoPointEvaluator extends AbstractConvertFunction
return ToString.fromGeoPoint(value);
}
private Block evalOrdinals(OrdinalBytesRefVector v) {
int positionCount = v.getDictionaryVector().getPositionCount();
BytesRef scratchPad = new BytesRef();
try (BytesRefVector.Builder builder = driverContext.blockFactory().newBytesRefVectorBuilder(positionCount)) {
for (int p = 0; p < positionCount; p++) {
builder.appendBytesRef(evalValue(v.getDictionaryVector(), p, scratchPad));
}
IntVector ordinals = v.getOrdinalsVector();
ordinals.incRef();
return new OrdinalBytesRefVector(ordinals, builder.build()).asBlock();
}
}
@Override
public String toString() {
return "ToStringFromGeoPointEvaluator[" + "wkb=" + wkb + "]";

19
x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromGeoShapeEvaluator.java generated
View File

@ -10,6 +10,8 @@ import org.apache.lucene.util.BytesRef;
import org.elasticsearch.compute.data.Block;
import org.elasticsearch.compute.data.BytesRefBlock;
import org.elasticsearch.compute.data.BytesRefVector;
import org.elasticsearch.compute.data.IntVector;
import org.elasticsearch.compute.data.OrdinalBytesRefVector;
import org.elasticsearch.compute.data.Vector;
import org.elasticsearch.compute.operator.DriverContext;
import org.elasticsearch.compute.operator.EvalOperator;
@ -37,6 +39,10 @@ public final class ToStringFromGeoShapeEvaluator extends AbstractConvertFunction
@Override
public Block evalVector(Vector v) {
BytesRefVector vector = (BytesRefVector) v;
OrdinalBytesRefVector ordinals = vector.asOrdinals();
if (ordinals != null) {
return evalOrdinals(ordinals);
}
int positionCount = v.getPositionCount();
BytesRef scratchPad = new BytesRef();
if (vector.isConstant()) {
@ -91,6 +97,19 @@ public final class ToStringFromGeoShapeEvaluator extends AbstractConvertFunction
return ToString.fromGeoShape(value);
}
private Block evalOrdinals(OrdinalBytesRefVector v) {
int positionCount = v.getDictionaryVector().getPositionCount();
BytesRef scratchPad = new BytesRef();
try (BytesRefVector.Builder builder = driverContext.blockFactory().newBytesRefVectorBuilder(positionCount)) {
for (int p = 0; p < positionCount; p++) {
builder.appendBytesRef(evalValue(v.getDictionaryVector(), p, scratchPad));
}
IntVector ordinals = v.getOrdinalsVector();
ordinals.incRef();
return new OrdinalBytesRefVector(ordinals, builder.build()).asBlock();
}
}
@Override
public String toString() {
return "ToStringFromGeoShapeEvaluator[" + "wkb=" + wkb + "]";

19
x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromIPEvaluator.java generated
View File

@ -10,6 +10,8 @@ import org.apache.lucene.util.BytesRef;
import org.elasticsearch.compute.data.Block;
import org.elasticsearch.compute.data.BytesRefBlock;
import org.elasticsearch.compute.data.BytesRefVector;
import org.elasticsearch.compute.data.IntVector;
import org.elasticsearch.compute.data.OrdinalBytesRefVector;
import org.elasticsearch.compute.data.Vector;
import org.elasticsearch.compute.operator.DriverContext;
import org.elasticsearch.compute.operator.EvalOperator;
@ -37,6 +39,10 @@ public final class ToStringFromIPEvaluator extends AbstractConvertFunction.Abstr
@Override
public Block evalVector(Vector v) {
BytesRefVector vector = (BytesRefVector) v;
OrdinalBytesRefVector ordinals = vector.asOrdinals();
if (ordinals != null) {
return evalOrdinals(ordinals);
}
int positionCount = v.getPositionCount();
BytesRef scratchPad = new BytesRef();
if (vector.isConstant()) {
@ -91,6 +97,19 @@ public final class ToStringFromIPEvaluator extends AbstractConvertFunction.Abstr
return ToString.fromIP(value);
}
private Block evalOrdinals(OrdinalBytesRefVector v) {
int positionCount = v.getDictionaryVector().getPositionCount();
BytesRef scratchPad = new BytesRef();
try (BytesRefVector.Builder builder = driverContext.blockFactory().newBytesRefVectorBuilder(positionCount)) {
for (int p = 0; p < positionCount; p++) {
builder.appendBytesRef(evalValue(v.getDictionaryVector(), p, scratchPad));
}
IntVector ordinals = v.getOrdinalsVector();
ordinals.incRef();
return new OrdinalBytesRefVector(ordinals, builder.build()).asBlock();
}
}
@Override
public String toString() {
return "ToStringFromIPEvaluator[" + "ip=" + ip + "]";

19
x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromVersionEvaluator.java generated
View File

@ -10,6 +10,8 @@ import org.apache.lucene.util.BytesRef;
import org.elasticsearch.compute.data.Block;
import org.elasticsearch.compute.data.BytesRefBlock;
import org.elasticsearch.compute.data.BytesRefVector;
import org.elasticsearch.compute.data.IntVector;
import org.elasticsearch.compute.data.OrdinalBytesRefVector;
import org.elasticsearch.compute.data.Vector;
import org.elasticsearch.compute.operator.DriverContext;
import org.elasticsearch.compute.operator.EvalOperator;
@ -37,6 +39,10 @@ public final class ToStringFromVersionEvaluator extends AbstractConvertFunction.
@Override
public Block evalVector(Vector v) {
BytesRefVector vector = (BytesRefVector) v;
OrdinalBytesRefVector ordinals = vector.asOrdinals();
if (ordinals != null) {
return evalOrdinals(ordinals);
}
int positionCount = v.getPositionCount();
BytesRef scratchPad = new BytesRef();
if (vector.isConstant()) {
@ -91,6 +97,19 @@ public final class ToStringFromVersionEvaluator extends AbstractConvertFunction.
return ToString.fromVersion(value);
}
private Block evalOrdinals(OrdinalBytesRefVector v) {
int positionCount = v.getDictionaryVector().getPositionCount();
BytesRef scratchPad = new BytesRef();
try (BytesRefVector.Builder builder = driverContext.blockFactory().newBytesRefVectorBuilder(positionCount)) {
for (int p = 0; p < positionCount; p++) {
builder.appendBytesRef(evalValue(v.getDictionaryVector(), p, scratchPad));
}
IntVector ordinals = v.getOrdinalsVector();
ordinals.incRef();
return new OrdinalBytesRefVector(ordinals, builder.build()).asBlock();
}
}
@Override
public String toString() {
return "ToStringFromVersionEvaluator[" + "version=" + version + "]";

19
x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionFromStringEvaluator.java generated
View File

@ -10,6 +10,8 @@ import org.apache.lucene.util.BytesRef;
import org.elasticsearch.compute.data.Block;
import org.elasticsearch.compute.data.BytesRefBlock;
import org.elasticsearch.compute.data.BytesRefVector;
import org.elasticsearch.compute.data.IntVector;
import org.elasticsearch.compute.data.OrdinalBytesRefVector;
import org.elasticsearch.compute.data.Vector;
import org.elasticsearch.compute.operator.DriverContext;
import org.elasticsearch.compute.operator.EvalOperator;
@ -37,6 +39,10 @@ public final class ToVersionFromStringEvaluator extends AbstractConvertFunction.
@Override
public Block evalVector(Vector v) {
BytesRefVector vector = (BytesRefVector) v;
OrdinalBytesRefVector ordinals = vector.asOrdinals();
if (ordinals != null) {
return evalOrdinals(ordinals);
}
int positionCount = v.getPositionCount();
BytesRef scratchPad = new BytesRef();
if (vector.isConstant()) {
@ -91,6 +97,19 @@ public final class ToVersionFromStringEvaluator extends AbstractConvertFunction.
return ToVersion.fromKeyword(value);
}
private Block evalOrdinals(OrdinalBytesRefVector v) {
int positionCount = v.getDictionaryVector().getPositionCount();
BytesRef scratchPad = new BytesRef();
try (BytesRefVector.Builder builder = driverContext.blockFactory().newBytesRefVectorBuilder(positionCount)) {
for (int p = 0; p < positionCount; p++) {
builder.appendBytesRef(evalValue(v.getDictionaryVector(), p, scratchPad));
}
IntVector ordinals = v.getOrdinalsVector();
ordinals.incRef();
return new OrdinalBytesRefVector(ordinals, builder.build()).asBlock();
}
}
@Override
public String toString() {
return "ToVersionFromStringEvaluator[" + "asString=" + asString + "]";

19
x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ChangeCaseEvaluator.java generated
View File

@ -11,6 +11,8 @@ import org.apache.lucene.util.BytesRef;
import org.elasticsearch.compute.data.Block;
import org.elasticsearch.compute.data.BytesRefBlock;
import org.elasticsearch.compute.data.BytesRefVector;
import org.elasticsearch.compute.data.IntVector;
import org.elasticsearch.compute.data.OrdinalBytesRefVector;
import org.elasticsearch.compute.data.Vector;
import org.elasticsearch.compute.operator.DriverContext;
import org.elasticsearch.compute.operator.EvalOperator;
@ -45,6 +47,10 @@ public final class ChangeCaseEvaluator extends AbstractConvertFunction.AbstractE
@Override
public Block evalVector(Vector v) {
BytesRefVector vector = (BytesRefVector) v;
OrdinalBytesRefVector ordinals = vector.asOrdinals();
if (ordinals != null) {
return evalOrdinals(ordinals);
}
int positionCount = v.getPositionCount();
BytesRef scratchPad = new BytesRef();
if (vector.isConstant()) {
@ -99,6 +105,19 @@ public final class ChangeCaseEvaluator extends AbstractConvertFunction.AbstractE
return ChangeCase.process(value, this.locale, this.caseType);
}
private Block evalOrdinals(OrdinalBytesRefVector v) {
int positionCount = v.getDictionaryVector().getPositionCount();
BytesRef scratchPad = new BytesRef();
try (BytesRefVector.Builder builder = driverContext.blockFactory().newBytesRefVectorBuilder(positionCount)) {
for (int p = 0; p < positionCount; p++) {
builder.appendBytesRef(evalValue(v.getDictionaryVector(), p, scratchPad));
}
IntVector ordinals = v.getOrdinalsVector();
ordinals.incRef();
return new OrdinalBytesRefVector(ordinals, builder.build()).asBlock();
}
}
@Override
public String toString() {
return "ChangeCaseEvaluator[" + "val=" + val + ", locale=" + locale + ", caseType=" + caseType + "]";

17
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java
View File

@ -19,9 +19,11 @@ import org.elasticsearch.common.util.PageCacheRecycler;
import org.elasticsearch.compute.data.Block;
import org.elasticsearch.compute.data.BlockFactory;
import org.elasticsearch.compute.data.BlockUtils;
import org.elasticsearch.compute.data.ElementType;
import org.elasticsearch.compute.data.Page;
import org.elasticsearch.compute.operator.DriverContext;
import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator;
import org.elasticsearch.compute.test.BlockTestUtils;
import org.elasticsearch.compute.test.TestBlockFactory;
import org.elasticsearch.indices.CrankyCircuitBreakerService;
import org.elasticsearch.logging.LogManager;
@ -553,7 +555,18 @@ public abstract class AbstractFunctionTestCase extends ESTestCase {
}
protected final Page row(List<Object> values) {
return new Page(1, BlockUtils.fromListRow(TestBlockFactory.getNonBreakingInstance(), values));
return maybeConvertBytesRefsToOrdinals(new Page(1, BlockUtils.fromListRow(TestBlockFactory.getNonBreakingInstance(), values)));
}
private Page maybeConvertBytesRefsToOrdinals(Page page) {
boolean anyBytesRef = false;
for (int b = 0; b < page.getBlockCount(); b++) {
if (page.getBlock(b).elementType() == ElementType.BYTES_REF) {
anyBytesRef = true;
break;
}
}
return anyBytesRef && randomBoolean() ? BlockTestUtils.convertBytesRefsToOrdinals(page) : page;
}
/**
@ -605,7 +618,7 @@ public abstract class AbstractFunctionTestCase extends ESTestCase {
}
}
pages.add(new Page(pageSize, blocks));
pages.add(maybeConvertBytesRefsToOrdinals(new Page(pageSize, blocks)));
initialRow += pageSize;
pageSize = randomIntBetween(1, 100);
}

44
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java
View File

@ -11,6 +11,8 @@ import org.elasticsearch.common.breaker.CircuitBreakingException;
import org.elasticsearch.common.util.MockBigArrays;
import org.elasticsearch.compute.data.Block;
import org.elasticsearch.compute.data.BlockFactory;
import org.elasticsearch.compute.data.BytesRefBlock;
import org.elasticsearch.compute.data.BytesRefVector;
import org.elasticsearch.compute.data.ElementType;
import org.elasticsearch.compute.data.Page;
import org.elasticsearch.compute.data.Vector;
@ -43,6 +45,7 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizer
import static org.hamcrest.Matchers.either;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.not;
import static org.hamcrest.Matchers.nullValue;
import static org.hamcrest.Matchers.sameInstance;
@ -110,14 +113,46 @@ public abstract class AbstractScalarFunctionTestCase extends AbstractFunctionTes
if (testCase.getExpectedBuildEvaluatorWarnings() != null) {
assertWarnings(testCase.getExpectedBuildEvaluatorWarnings());
}
try (Block block = evaluator.eval(row(testCase.getDataValues()))) {
Page row = row(testCase.getDataValues());
try (Block block = evaluator.eval(row)) {
assertThat(block.getPositionCount(), is(1));
result = toJavaObjectUnsignedLongAware(block, 0);
extraBlockTests(row, block);
} finally {
row.releaseBlocks();
}
}
assertTestCaseResultAndWarnings(result);
}
/**
* Extra assertions on the output block.
*/
protected void extraBlockTests(Page in, Block out) {}
protected final void assertIsOrdIfInIsOrd(Page in, Block out) {
BytesRefBlock inBytes = in.getBlock(0);
BytesRefBlock outBytes = (BytesRefBlock) out;
BytesRefVector inVec = inBytes.asVector();
if (inVec == null) {
assertThat(outBytes.asVector(), nullValue());
return;
}
BytesRefVector outVec = outBytes.asVector();
if (inVec.isConstant()) {
assertTrue(outVec.isConstant());
return;
}
if (inVec.asOrdinals() != null) {
assertThat(outBytes.asOrdinals(), not(nullValue()));
return;
}
assertThat(outBytes.asOrdinals(), nullValue());
}
/**
* Evaluates a {@link Block} of values, all copied from the input pattern..
* <p>
@ -227,10 +262,8 @@ public abstract class AbstractScalarFunctionTestCase extends AbstractFunctionTes
}
b++;
}
try (
ExpressionEvaluator eval = evaluator(expression).get(context);
Block block = eval.eval(new Page(positions, manyPositionsBlocks))
) {
Page in = new Page(positions, manyPositionsBlocks);
try (ExpressionEvaluator eval = evaluator(expression).get(context); Block block = eval.eval(in)) {
if (testCase.getExpectedBuildEvaluatorWarnings() != null) {
assertWarnings(testCase.getExpectedBuildEvaluatorWarnings());
}
@ -247,6 +280,7 @@ public abstract class AbstractScalarFunctionTestCase extends AbstractFunctionTes
block.blockFactory(),
either(sameInstance(context.blockFactory())).or(sameInstance(inputBlockFactory))
);
extraBlockTests(in, block);
}
} finally {
Releasables.close(onePositionPage::releaseBlocks, Releasables.wrap(manyPositionsBlocks));

7
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java
View File

@ -13,6 +13,8 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.lucene.BytesRefs;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.compute.data.Block;
import org.elasticsearch.compute.data.Page;
import org.elasticsearch.xpack.esql.EsqlTestUtils;
import org.elasticsearch.xpack.esql.core.expression.Expression;
import org.elasticsearch.xpack.esql.core.expression.FoldContext;
@ -102,4 +104,9 @@ public class ToLowerTests extends AbstractConfigurationFunctionTestCase {
return new TestCaseSupplier.TestCase(values, expectedToString, type, equalTo(expectedValue));
}));
}
@Override
protected void extraBlockTests(Page in, Block out) {
assertIsOrdIfInIsOrd(in, out);
}
}

7
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java
View File

@ -13,6 +13,8 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.lucene.BytesRefs;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.compute.data.Block;
import org.elasticsearch.compute.data.Page;
import org.elasticsearch.xpack.esql.EsqlTestUtils;
import org.elasticsearch.xpack.esql.core.expression.Expression;
import org.elasticsearch.xpack.esql.core.expression.FoldContext;
@ -102,4 +104,9 @@ public class ToUpperTests extends AbstractConfigurationFunctionTestCase {
return new TestCaseSupplier.TestCase(values, expectedToString, type, equalTo(expectedValue));
}));
}
@Override
protected void extraBlockTests(Page in, Block out) {
assertIsOrdIfInIsOrd(in, out);
}
}