mirrors
/
elasticsearch
mirror of https://mirror.osredm.com/root/elasticsearch.git
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

ESQL: `text ==` and `text !=` pushdown (#127355) 

Reenables `text ==` pushdown and adds support for `text !=` pushdown.

It does so by making `TranslationAware#translatable` return something
we can turn into a tri-valued function. It has these values:
* `YES`
* `NO`
* `RECHECK`

`YES` means the `Expression` is entirely pushable into Lucene. They will
be pushed into Lucene and removed from the plan.

`NO` means the `Expression` can't be pushed to Lucene at all and will stay
in the plan.

`RECHECK` mean the `Expression` can push a query that makes *candidate*
matches but must be rechecked. Documents that don't match the query won't
match the expression, but documents that match the query might not match
the expression. These are pushed to Lucene *and* left in the plan.

This is required because `txt != "b"` can build a *candidate* query
against the `txt.keyword` subfield but it can't be sure of the match
without loading the `_source` - which we do in the compute engine.

I haven't plugged rally into this, but here's some basic
performance tests:
```
Before:
not text eq {"took":460,"documents_found":1000000}
    text eq {"took":432,"documents_found":1000000}

After:
    text eq {"took":5,"documents_found":1}
not text eq {"took":351,"documents_found":800000}    
```

This comes from:
```
rm -f /tmp/bulk*
for a in {1..1000}; do
    echo '{"index":{}}' >> /tmp/bulk
    echo '{"text":"text '$(printf $(($a % 5)))'"}' >> /tmp/bulk
done
ls -l /tmp/bulk*

passwd="redacted"
curl -sk -uelastic:$passwd -HContent-Type:application/json -XDELETE https://localhost:9200/test
curl -sk -uelastic:$passwd -HContent-Type:application/json -XPUT https://localhost:9200/test -d'{
    "settings": {
        "index.codec": "best_compression",
        "index.refresh_interval": -1
    },
    "mappings": {
        "properties": {
            "many": {
                "enabled": false
            }
        }
    }
}'
for a in {1..1000}; do
    printf %04d: $a
    curl -sk -uelastic:$passwd -HContent-Type:application/json -XPOST https://localhost:9200/test/_bulk?pretty --data-binary @/tmp/bulk | grep errors
done
curl -sk -uelastic:$passwd -HContent-Type:application/json -XPOST https://localhost:9200/test/_forcemerge?max_num_segments=1
curl -sk -uelastic:$passwd -HContent-Type:application/json -XPOST https://localhost:9200/test/_refresh
echo
curl -sk -uelastic:$passwd https://localhost:9200/_cat/indices?v

text_eq() {
    echo -n "    text eq "
    curl -sk -uelastic:$passwd -HContent-Type:application/json -XPOST 'https://localhost:9200/_query?pretty' -d'{
        "query": "FROM test | WHERE text == \"text 1\" | STATS COUNT(*)",
        "pragma": {
            "data_partitioning": "shard"
        }
    }' | jq -c '{took, documents_found}'
}

not_text_eq() {
    echo -n "not text eq "
    curl -sk -uelastic:$passwd -HContent-Type:application/json -XPOST 'https://localhost:9200/_query?pretty' -d'{
        "query": "FROM test | WHERE NOT text == \"text 1\" | STATS COUNT(*)",
        "pragma": {
            "data_partitioning": "shard"
        }
    }' | jq -c '{took, documents_found}'
}


for a in {1..100}; do
    text_eq
    not_text_eq
done
```
This commit is contained in:
Nik Everett 2025-05-08 10:00:05 -04:00 committed by GitHub
parent 3f5f8994ef
commit 3551494b9a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
30 changed files with 562 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
docs/changelog/127355.yaml Normal file
View File

@ -0,0 +1,5 @@
pr: 127355
summary: '`text ==` and `text !=` pushdown'
area: ES|QL
type: enhancement
issues: []

194
x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java
View File

@ -7,10 +7,12 @@
package org.elasticsearch.xpack.esql.qa.single_node;
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.ResponseException;
import org.elasticsearch.test.ListMatcher;
import org.elasticsearch.test.MapMatcher;
import org.elasticsearch.test.TestClustersThreadFilter;
@ -27,6 +29,7 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.regex.Pattern;
import java.util.stream.Stream;
import static org.elasticsearch.test.ListMatcher.matchesList;
import static org.elasticsearch.test.MapMatcher.assertMap;
@ -48,50 +51,161 @@ public class PushQueriesIT extends ESRestTestCase {
@ClassRule
public static ElasticsearchCluster cluster = Clusters.testCluster();
public void testPushEqualityOnDefaults() throws IOException {
String value = "v".repeat(between(0, 256));
testPushQuery(value, """
FROM test
| WHERE test == "%value"
""", "*:*", true, true);
@ParametersFactory(argumentFormatting = "%1s")
public static List<Object[]> args() {
return Stream.of("auto", "text", "match_only_text", "semantic_text").map(s -> new Object[] { s }).toList();
}
public void testPushEqualityOnDefaultsTooBigToPush() throws IOException {
private final String type;
public PushQueriesIT(String type) {
this.type = type;
}
public void testEquality() throws IOException {
String value = "v".repeat(between(0, 256));
String esqlQuery = """
FROM test
| WHERE test == "%value"
""";
String luceneQuery = switch (type) {
case "text", "auto" -> "#test.keyword:%value -_ignored:test.keyword";
case "match_only_text" -> "*:*";
case "semantic_text" -> "FieldExistsQuery [field=_primary_term]";
default -> throw new UnsupportedOperationException("unknown type [" + type + "]");
};
boolean filterInCompute = switch (type) {
case "text", "auto" -> false;
case "match_only_text", "semantic_text" -> true;
default -> throw new UnsupportedOperationException("unknown type [" + type + "]");
};
testPushQuery(value, esqlQuery, luceneQuery, filterInCompute, true);
}
public void testEqualityTooBigToPush() throws IOException {
String value = "a".repeat(between(257, 1000));
testPushQuery(value, """
String esqlQuery = """
FROM test
| WHERE test == "%value"
""", "*:*", true, true);
""";
String luceneQuery = switch (type) {
case "text", "auto", "match_only_text" -> "*:*";
case "semantic_text" -> "FieldExistsQuery [field=_primary_term]";
default -> throw new UnsupportedOperationException("unknown type [" + type + "]");
};
testPushQuery(value, esqlQuery, luceneQuery, true, true);
}
public void testPushInequalityOnDefaults() throws IOException {
/**
* Turns into an {@code IN} which isn't currently pushed.
*/
public void testEqualityOrTooBig() throws IOException {
String value = "v".repeat(between(0, 256));
testPushQuery(value, """
String tooBig = "a".repeat(between(257, 1000));
String esqlQuery = """
FROM test
| WHERE test == "%value" OR test == "%tooBig"
""".replace("%tooBig", tooBig);
String luceneQuery = switch (type) {
case "text", "auto", "match_only_text" -> "*:*";
case "semantic_text" -> "FieldExistsQuery [field=_primary_term]";
default -> throw new UnsupportedOperationException("unknown type [" + type + "]");
};
testPushQuery(value, esqlQuery, luceneQuery, true, true);
}
public void testEqualityOrOther() throws IOException {
String value = "v".repeat(between(0, 256));
String esqlQuery = """
FROM test
| WHERE test == "%value" OR foo == 2
""";
String luceneQuery = switch (type) {
case "text", "auto" -> "(#test.keyword:%value -_ignored:test.keyword) foo:[2 TO 2]";
case "match_only_text" -> "*:*";
case "semantic_text" -> "FieldExistsQuery [field=_primary_term]";
default -> throw new UnsupportedOperationException("unknown type [" + type + "]");
};
boolean filterInCompute = switch (type) {
case "text", "auto" -> false;
case "match_only_text", "semantic_text" -> true;
default -> throw new UnsupportedOperationException("unknown type [" + type + "]");
};
testPushQuery(value, esqlQuery, luceneQuery, filterInCompute, true);
}
public void testEqualityAndOther() throws IOException {
String value = "v".repeat(between(0, 256));
String esqlQuery = """
FROM test
| WHERE test == "%value" AND foo == 1
""";
String luceneQuery = switch (type) {
case "text", "auto" -> "#test.keyword:%value -_ignored:test.keyword #foo:[1 TO 1]";
case "match_only_text" -> "foo:[1 TO 1]";
case "semantic_text" ->
/*
* single_value_match is here because there are extra documents hiding in the index
* that don't have the `foo` field.
*/
"#foo:[1 TO 1] #single_value_match(foo)";
default -> throw new UnsupportedOperationException("unknown type [" + type + "]");
};
boolean filterInCompute = switch (type) {
case "text", "auto" -> false;
case "match_only_text", "semantic_text" -> true;
default -> throw new UnsupportedOperationException("unknown type [" + type + "]");
};
testPushQuery(value, esqlQuery, luceneQuery, filterInCompute, true);
}
public void testInequality() throws IOException {
String value = "v".repeat(between(0, 256));
String esqlQuery = """
FROM test
| WHERE test != "%different_value"
""", "*:*", true, true);
""";
String luceneQuery = switch (type) {
case "text", "auto" -> "(-test.keyword:%different_value #*:*) _ignored:test.keyword";
case "match_only_text" -> "*:*";
case "semantic_text" -> "FieldExistsQuery [field=_primary_term]";
default -> throw new UnsupportedOperationException("unknown type [" + type + "]");
};
testPushQuery(value, esqlQuery, luceneQuery, true, true);
}
public void testPushInequalityOnDefaultsTooBigToPush() throws IOException {
public void testInequalityTooBigToPush() throws IOException {
String value = "a".repeat(between(257, 1000));
testPushQuery(value, """
String esqlQuery = """
FROM test
| WHERE test != "%value"
""", "*:*", true, false);
""";
String luceneQuery = switch (type) {
case "text", "auto", "match_only_text" -> "*:*";
case "semantic_text" -> "FieldExistsQuery [field=_primary_term]";
default -> throw new UnsupportedOperationException("unknown type [" + type + "]");
};
testPushQuery(value, esqlQuery, luceneQuery, true, false);
}
public void testPushCaseInsensitiveEqualityOnDefaults() throws IOException {
public void testCaseInsensitiveEquality() throws IOException {
String value = "a".repeat(between(0, 256));
testPushQuery(value, """
String esqlQuery = """
FROM test
| WHERE TO_LOWER(test) == "%value"
""", "*:*", true, true);
""";
String luceneQuery = switch (type) {
case "text", "auto", "match_only_text" -> "*:*";
case "semantic_text" -> "FieldExistsQuery [field=_primary_term]";
default -> throw new UnsupportedOperationException("unknown type [" + type + "]");
};
testPushQuery(value, esqlQuery, luceneQuery, true, true);
}
private void testPushQuery(String value, String esqlQuery, String luceneQuery, boolean filterInCompute, boolean found)
throws IOException {
indexValue(value);
String differentValue = randomValueOtherThan(value, () -> randomAlphaOfLength(value.length() == 0 ? 1 : value.length()));
String differentValue = randomValueOtherThan(value, () -> randomAlphaOfLength(value.isEmpty() ? 1 : value.length()));
String replacedQuery = esqlQuery.replaceAll("%value", value).replaceAll("%different_value", differentValue);
RestEsqlTestCase.RequestObjectBuilder builder = requestObjectBuilder().query(replacedQuery + "\n| KEEP test");
@ -148,15 +262,43 @@ public class PushQueriesIT extends ESRestTestCase {
}
private void indexValue(String value) throws IOException {
try {
// Delete the index if it has already been created.
client().performRequest(new Request("DELETE", "test"));
} catch (ResponseException e) {
if (e.getResponse().getStatusLine().getStatusCode() != 404) {
throw e;
}
}
Request createIndex = new Request("PUT", "test");
createIndex.setJsonEntity("""
String json = """
{
"settings": {
"index": {
"number_of_shards": 1
}
}
}""");
}""";
if (false == "auto".equals(type)) {
json += """
,
"mappings": {
"properties": {
"test": {
"type": "%type",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}""".replace("%type", type);
}
json += "}";
createIndex.setJsonEntity(json);
Response createResponse = client().performRequest(createIndex);
assertThat(
entityToMap(createResponse.getEntity(), XContentType.JSON),
@ -167,7 +309,7 @@ public class PushQueriesIT extends ESRestTestCase {
bulk.addParameter("refresh", "");
bulk.setJsonEntity(String.format(Locale.ROOT, """
{"create":{"_index":"test"}}
{"test":"%s"}
{"test":"%s","foo":1}
""", value));
Response bulkResponse = client().performRequest(bulk);
assertThat(entityToMap(bulkResponse.getEntity(), XContentType.JSON), matchesMap().entry("errors", false).extraOk());
@ -190,4 +332,10 @@ public class PushQueriesIT extends ESRestTestCase {
protected String getTestRestCluster() {
return cluster.getHttpAddresses();
}
@Override
protected boolean preserveClusterUponCompletion() {
// Preserve the cluser to speed up the semantic_text tests
return true;
}
}

1
x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/mv_text.csv
View File

@ -3,3 +3,4 @@
2023-10-23T13:55:01.544Z,Connected to 10.1.0.1
2023-10-23T13:55:01.545Z,[Connected to 10.1.0.1, More than one hundred characters long so it isn't indexed by the sub keyword field with ignore_above:100]
2023-10-23T13:55:01.546Z,More than one hundred characters long so it isn't indexed by the sub keyword field with ignore_above:100
2023-10-23T13:55:01.547Z,[More than one hundred characters long so it isn't indexed by the sub keyword field with ignore_above:100,Second than one hundred characters long so it isn't indexed by the sub keyword field with ignore_above:100]

1 @timestamp:date ,message:text
3 2023-10-23T13:55:01.544Z,Connected to 10.1.0.1
4 2023-10-23T13:55:01.545Z,[Connected to 10.1.0.1, More than one hundred characters long so it isn't indexed by the sub keyword field with ignore_above:100]
5 2023-10-23T13:55:01.546Z,More than one hundred characters long so it isn't indexed by the sub keyword field with ignore_above:100
6 2023-10-23T13:55:01.547Z,[More than one hundred characters long so it isn't indexed by the sub keyword field with ignore_above:100,Second than one hundred characters long so it isn't indexed by the sub keyword field with ignore_above:100]

130
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/TranslationAware.java
View File

@ -7,20 +7,34 @@
package org.elasticsearch.xpack.esql.capabilities;
import org.elasticsearch.compute.lucene.LuceneTopNSourceOperator;
import org.elasticsearch.compute.operator.FilterOperator;
import org.elasticsearch.xpack.esql.core.expression.Expression;
import org.elasticsearch.xpack.esql.core.querydsl.query.Query;
import org.elasticsearch.xpack.esql.optimizer.rules.physical.local.LucenePushdownPredicates;
import org.elasticsearch.xpack.esql.planner.TranslatorHandler;
/**
* Expressions implementing this interface can get called on data nodes to provide an Elasticsearch/Lucene query.
* Expressions implementing this interface are asked provide an
* Elasticsearch/Lucene query as part of the data node optimizations.
*/
public interface TranslationAware {
/**
* Indicates whether the expression can be translated or not.
* Usually checks whether the expression arguments are actual fields that exist in Lucene.
* Can this instance be translated or not? Usually checks whether the
* expression arguments are actual fields that exist in Lucene. See {@link Translatable}
* for precisely what can be signaled from this method.
*/
boolean translatable(LucenePushdownPredicates pushdownPredicates);
Translatable translatable(LucenePushdownPredicates pushdownPredicates);
/**
* Is an {@link Expression} translatable?
*/
static TranslationAware.Translatable translatable(Expression exp, LucenePushdownPredicates lucenePushdownPredicates) {
if (exp instanceof TranslationAware aware) {
return aware.translatable(lucenePushdownPredicates);
}
return TranslationAware.Translatable.NO;
}
/**
* Translates the implementing expression into a Query.
@ -42,4 +56,112 @@ public interface TranslationAware {
*/
Expression singleValueField();
}
/**
* How is this expression translatable?
*/
enum Translatable {
/**
* Not translatable at all. Calling {@link TranslationAware#asQuery} is an error.
* The expression will stay in the query plan and be filtered via a {@link FilterOperator}.
* Imagine {@code kwd == "a"} when {@code kwd} is configured without a search index.
*/
NO(FinishedTranslatable.NO),
/**
* Entirely translatable into a lucene query. Calling {@link TranslationAware#asQuery}
* will produce a query that matches all documents matching this expression and
* <strong>only</strong> documents matching this expression. Imagine {@code kwd == "a"}
* when {@code kwd} has a search index and doc values - which is the
* default configuration. This will entirely remove the clause from the
* {@code WHERE}, removing the entire {@link FilterOperator} if it's empty. Sometimes
* this allows us to push the entire top-n operation to lucene with
* a {@link LuceneTopNSourceOperator}.
*/
YES(FinishedTranslatable.YES),
/**
* Translation requires a recheck. Calling {@link TranslationAware#asQuery} will
* produce a query that matches all documents matching this expression but might
* match more documents that do not match the expression. This will cause us to
* push a query to lucene <strong>and</strong> keep the query in the query plan,
* rechecking it via a {@link FilterOperator}. This can never push the entire
* top-n to Lucene, but it's still quite a lot better than the full scan from
* {@link #NO}.
* <p>
* Imagine {@code kwd == "a"} where {@code kwd} has a search index but doesn't
* have doc values. In that case we can find candidate matches in lucene but
* can't tell if those docs are single-valued. If they are multivalued they'll
* still match the query but won't match the expression. Thus, the double-checking.
* <strong>Technically</strong> we could just check for single-valued-ness in
* this case, but it's simpler to
* </p>
*/
RECHECK(FinishedTranslatable.RECHECK),
/**
* The same as {@link #YES}, but if this expression is negated it turns into {@link #RECHECK}.
* This comes up when pushing {@code NOT(text == "a")} to {@code text.keyword} which can
* have ignored fields.
*/
YES_BUT_RECHECK_NEGATED(FinishedTranslatable.YES);
private final FinishedTranslatable finish;
Translatable(FinishedTranslatable finish) {
this.finish = finish;
}
/**
* Translate into a {@link FinishedTranslatable} which never
* includes {@link #YES_BUT_RECHECK_NEGATED}.
*/
public FinishedTranslatable finish() {
return finish;
}
public Translatable negate() {
if (this == YES_BUT_RECHECK_NEGATED) {
return RECHECK;
}
return this;
}
/**
* Merge two {@link TranslationAware#translatable} results.
*/
public Translatable merge(Translatable rhs) {
return switch (this) {
case NO -> NO;
case YES -> switch (rhs) {
case NO -> NO;
case YES -> YES;
case RECHECK -> RECHECK;
case YES_BUT_RECHECK_NEGATED -> YES_BUT_RECHECK_NEGATED;
};
case RECHECK -> switch (rhs) {
case NO -> NO;
case YES, RECHECK, YES_BUT_RECHECK_NEGATED -> RECHECK;
};
case YES_BUT_RECHECK_NEGATED -> switch (rhs) {
case NO -> NO;
case YES, YES_BUT_RECHECK_NEGATED -> YES_BUT_RECHECK_NEGATED;
case RECHECK -> RECHECK;
};
};
}
}
enum FinishedTranslatable {
/**
* See {@link Translatable#YES}.
*/
YES,
/**
* See {@link Translatable#NO}.
*/
NO,
/**
* See {@link Translatable#RECHECK}.
*/
RECHECK;
}
}

4
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java
View File

@ -156,9 +156,9 @@ public abstract class FullTextFunction extends Function
}
@Override
public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
public Translatable translatable(LucenePushdownPredicates pushdownPredicates) {
// In isolation, full text functions are pushable to source. We check if there are no disjunctions in Or conditions
return true;
return Translatable.YES;
}
@Override

4
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatch.java
View File

@ -179,8 +179,8 @@ public class CIDRMatch extends EsqlScalarFunction implements TranslationAware.Si
}
@Override
public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
return pushdownPredicates.isPushableFieldAttribute(ipField) && Expressions.foldable(matches);
public Translatable translatable(LucenePushdownPredicates pushdownPredicates) {
return pushdownPredicates.isPushableFieldAttribute(ipField) && Expressions.foldable(matches) ? Translatable.YES : Translatable.NO;
}
@Override

7
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunction.java
View File

@ -13,6 +13,7 @@ import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.geometry.Geometry;
import org.elasticsearch.lucene.spatial.CoordinateEncoder;
import org.elasticsearch.xpack.esql.capabilities.TranslationAware;
import org.elasticsearch.xpack.esql.core.expression.Expression;
import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
import org.elasticsearch.xpack.esql.core.expression.TypeResolutions;
@ -273,12 +274,14 @@ public abstract class BinarySpatialFunction extends BinaryScalarFunction impleme
/**
* Push-down to Lucene is only possible if one field is an indexed spatial field, and the other is a constant spatial or string column.
*/
public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
public TranslationAware.Translatable translatable(LucenePushdownPredicates pushdownPredicates) {
// The use of foldable here instead of SpatialEvaluatorFieldKey.isConstant is intentional to match the behavior of the
// Lucene pushdown code in EsqlTranslationHandler::SpatialRelatesTranslator
// We could enhance both places to support ReferenceAttributes that refer to constants, but that is a larger change
return isPushableSpatialAttribute(left(), pushdownPredicates) && right().foldable()
|| isPushableSpatialAttribute(right(), pushdownPredicates) && left().foldable();
|| isPushableSpatialAttribute(right(), pushdownPredicates) && left().foldable()
? TranslationAware.Translatable.YES
: TranslationAware.Translatable.NO;
}

2
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java
View File

@ -181,7 +181,7 @@ public abstract class SpatialRelatesFunction extends BinarySpatialFunction
}
@Override
public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
public Translatable translatable(LucenePushdownPredicates pushdownPredicates) {
return super.translatable(pushdownPredicates); // only for the explicit Override, as only this subclass implements TranslationAware
}

4
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWith.java
View File

@ -139,8 +139,8 @@ public class EndsWith extends EsqlScalarFunction implements TranslationAware.Sin
}
@Override
public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
return pushdownPredicates.isPushableAttribute(str) && suffix.foldable();
public Translatable translatable(LucenePushdownPredicates pushdownPredicates) {
return pushdownPredicates.isPushableAttribute(str) && suffix.foldable() ? Translatable.YES : Translatable.NO;
}
@Override

4
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLike.java
View File

@ -107,8 +107,8 @@ public class RLike extends org.elasticsearch.xpack.esql.core.expression.predicat
}
@Override
public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
return pushdownPredicates.isPushableFieldAttribute(field());
public Translatable translatable(LucenePushdownPredicates pushdownPredicates) {
return pushdownPredicates.isPushableFieldAttribute(field()) ? Translatable.YES : Translatable.NO;
}
@Override

4
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWith.java
View File

@ -136,8 +136,8 @@ public class StartsWith extends EsqlScalarFunction implements TranslationAware.S
}
@Override
public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
return pushdownPredicates.isPushableAttribute(str) && prefix.foldable();
public Translatable translatable(LucenePushdownPredicates pushdownPredicates) {
return pushdownPredicates.isPushableAttribute(str) && prefix.foldable() ? Translatable.YES : Translatable.NO;
}
@Override

4
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java
View File

@ -119,8 +119,8 @@ public class WildcardLike extends org.elasticsearch.xpack.esql.core.expression.p
}
@Override
public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
return pushdownPredicates.isPushableAttribute(field());
public Translatable translatable(LucenePushdownPredicates pushdownPredicates) {
return pushdownPredicates.isPushableAttribute(field()) ? Translatable.YES : Translatable.NO;
}
@Override

4
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/Range.java
View File

@ -215,8 +215,8 @@ public class Range extends ScalarFunction implements TranslationAware.SingleValu
}
@Override
public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
return pushdownPredicates.isPushableAttribute(value) && lower.foldable() && upper.foldable();
public Translatable translatable(LucenePushdownPredicates pushdownPredicates) {
return pushdownPredicates.isPushableAttribute(value) && lower.foldable() && upper.foldable() ? Translatable.YES : Translatable.NO;
}
@Override

7
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/logical/BinaryLogic.java
View File

@ -82,11 +82,8 @@ public abstract class BinaryLogic extends BinaryOperator<Boolean, Boolean, Boole
}
@Override
public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
return left() instanceof TranslationAware leftAware
&& leftAware.translatable(pushdownPredicates)
&& right() instanceof TranslationAware rightAware
&& rightAware.translatable(pushdownPredicates);
public Translatable translatable(LucenePushdownPredicates pushdownPredicates) {
return TranslationAware.translatable(left(), pushdownPredicates).merge(TranslationAware.translatable(right(), pushdownPredicates));
}
@Override

4
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/logical/Not.java
View File

@ -100,8 +100,8 @@ public class Not extends UnaryScalarFunction implements Negatable<Expression>, T
}
@Override
public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
return field() instanceof TranslationAware aware && aware.translatable(pushdownPredicates);
public Translatable translatable(LucenePushdownPredicates pushdownPredicates) {
return TranslationAware.translatable(field(), pushdownPredicates).negate();
}
@Override

2
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNotNull.java
View File

@ -75,7 +75,7 @@ public class IsNotNull extends UnaryScalarFunction implements Negatable<UnarySca
}
@Override
public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
public Translatable translatable(LucenePushdownPredicates pushdownPredicates) {
return IsNull.isTranslatable(field(), pushdownPredicates);
}

8
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNull.java
View File

@ -72,12 +72,14 @@ public class IsNull extends UnaryScalarFunction implements Negatable<UnaryScalar
}
@Override
public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
public Translatable translatable(LucenePushdownPredicates pushdownPredicates) {
return isTranslatable(field(), pushdownPredicates);
}
protected static boolean isTranslatable(Expression field, LucenePushdownPredicates pushdownPredicates) {
return LucenePushdownPredicates.isPushableTextFieldAttribute(field) || pushdownPredicates.isPushableFieldAttribute(field);
protected static Translatable isTranslatable(Expression field, LucenePushdownPredicates pushdownPredicates) {
return LucenePushdownPredicates.isPushableTextFieldAttribute(field) || pushdownPredicates.isPushableFieldAttribute(field)
? Translatable.YES
: Translatable.NO;
}
@Override

9
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java
View File

@ -128,11 +128,11 @@ public class Equals extends EsqlBinaryComparison implements Negatable<EsqlBinary
}
@Override
public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
public Translatable translatable(LucenePushdownPredicates pushdownPredicates) {
if (right() instanceof Literal lit) {
if (false && left().dataType() == DataType.TEXT && left() instanceof FieldAttribute fa) {
if (left().dataType() == DataType.TEXT && left() instanceof FieldAttribute fa) {
if (pushdownPredicates.canUseEqualityOnSyntheticSourceDelegate(fa, ((BytesRef) lit.value()).utf8ToString())) {
return true;
return Translatable.YES_BUT_RECHECK_NEGATED;
}
}
}
@ -142,8 +142,7 @@ public class Equals extends EsqlBinaryComparison implements Negatable<EsqlBinary
@Override
public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) {
if (right() instanceof Literal lit) {
// Disabled because it cased a bug with !=. Fix incoming shortly.
if (false && left().dataType() == DataType.TEXT && left() instanceof FieldAttribute fa) {
if (left().dataType() == DataType.TEXT && left() instanceof FieldAttribute fa) {
String value = ((BytesRef) lit.value()).utf8ToString();
if (pushdownPredicates.canUseEqualityOnSyntheticSourceDelegate(fa, value)) {
String name = handler.nameOf(fa);

8
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java
View File

@ -328,16 +328,16 @@ public abstract class EsqlBinaryComparison extends BinaryComparison
}
@Override
public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
public Translatable translatable(LucenePushdownPredicates pushdownPredicates) {
if (right().foldable()) {
if (pushdownPredicates.isPushableFieldAttribute(left())) {
return true;
return Translatable.YES;
}
if (LucenePushdownPredicates.isPushableMetadataAttribute(left())) {
return this instanceof Equals || this instanceof NotEquals;
return this instanceof Equals || this instanceof NotEquals ? Translatable.YES : Translatable.NO;
}
}
return false;
return Translatable.NO;
}
/**

4
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java
View File

@ -461,8 +461,8 @@ public class In extends EsqlScalarFunction implements TranslationAware.SingleVal
}
@Override
public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
return pushdownPredicates.isPushableAttribute(value) && Expressions.foldable(list());
public Translatable translatable(LucenePushdownPredicates pushdownPredicates) {
return pushdownPredicates.isPushableAttribute(value) && Expressions.foldable(list()) ? Translatable.YES : Translatable.NO;
}
@Override

4
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEquals.java
View File

@ -102,8 +102,8 @@ public class InsensitiveEquals extends InsensitiveBinaryComparison {
}
@Override
public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
return pushdownPredicates.isPushableFieldAttribute(left()) && right().foldable();
public Translatable translatable(LucenePushdownPredicates pushdownPredicates) {
return pushdownPredicates.isPushableFieldAttribute(left()) && right().foldable() ? Translatable.YES : Translatable.NO;
}
@Override

12
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/EnableSpatialDistancePushdown.java
View File

@ -12,6 +12,7 @@ import org.elasticsearch.geometry.Circle;
import org.elasticsearch.geometry.Geometry;
import org.elasticsearch.geometry.Point;
import org.elasticsearch.geometry.utils.WellKnownBinary;
import org.elasticsearch.xpack.esql.capabilities.TranslationAware;
import org.elasticsearch.xpack.esql.core.expression.Alias;
import org.elasticsearch.xpack.esql.core.expression.Attribute;
import org.elasticsearch.xpack.esql.core.expression.AttributeMap;
@ -42,8 +43,8 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import static org.elasticsearch.xpack.esql.capabilities.TranslationAware.translatable;
import static org.elasticsearch.xpack.esql.expression.predicate.Predicates.splitAnd;
import static org.elasticsearch.xpack.esql.optimizer.rules.physical.local.PushFiltersToSource.canPushToSource;
import static org.elasticsearch.xpack.esql.optimizer.rules.physical.local.PushFiltersToSource.getAliasReplacedBy;
/**
@ -106,7 +107,8 @@ public class EnableSpatialDistancePushdown extends PhysicalOptimizerRules.Parame
}
return comparison;
});
if (rewritten.equals(filterExec.condition()) == false && canPushToSource(rewritten, lucenePushdownPredicates)) {
if (rewritten.equals(filterExec.condition()) == false
&& translatable(rewritten, lucenePushdownPredicates).finish() == TranslationAware.FinishedTranslatable.YES) {
return new FilterExec(filterExec.source(), esQueryExec, rewritten);
}
return filterExec;
@ -156,7 +158,8 @@ public class EnableSpatialDistancePushdown extends PhysicalOptimizerRules.Parame
// Find and rewrite any binary comparisons that involve a distance function and a literal
var rewritten = rewriteDistanceFilters(ctx, resExp, distances);
// If all pushable StDistance functions were found and re-written, we need to re-write the FILTER/EVAL combination
if (rewritten.equals(resExp) == false && canPushToSource(rewritten, lucenePushdownPredicates)) {
if (rewritten.equals(resExp) == false
&& translatable(rewritten, lucenePushdownPredicates).finish() == TranslationAware.FinishedTranslatable.YES) {
pushable.add(rewritten);
} else {
nonPushable.add(exp);
@ -183,7 +186,8 @@ public class EnableSpatialDistancePushdown extends PhysicalOptimizerRules.Parame
private Map<NameId, StDistance> getPushableDistances(List<Alias> aliases, LucenePushdownPredicates lucenePushdownPredicates) {
Map<NameId, StDistance> distances = new LinkedHashMap<>();
aliases.forEach(alias -> {
if (alias.child() instanceof StDistance distance && distance.translatable(lucenePushdownPredicates)) {
if (alias.child() instanceof StDistance distance
&& distance.translatable(lucenePushdownPredicates).finish() == TranslationAware.FinishedTranslatable.YES) {
distances.put(alias.id(), distance);
} else if (alias.child() instanceof ReferenceAttribute ref && distances.containsKey(ref.id())) {
StDistance distance = distances.get(ref.id());

34
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java
View File

@ -8,7 +8,6 @@
package org.elasticsearch.xpack.esql.optimizer.rules.physical.local;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.xpack.esql.capabilities.TranslationAware;
import org.elasticsearch.xpack.esql.core.expression.Alias;
import org.elasticsearch.xpack.esql.core.expression.Attribute;
import org.elasticsearch.xpack.esql.core.expression.AttributeMap;
@ -36,6 +35,7 @@ import java.util.ArrayList;
import java.util.List;
import static java.util.Arrays.asList;
import static org.elasticsearch.xpack.esql.capabilities.TranslationAware.translatable;
import static org.elasticsearch.xpack.esql.expression.predicate.Predicates.splitAnd;
import static org.elasticsearch.xpack.esql.planner.TranslatorHandler.TRANSLATOR_HANDLER;
@ -57,7 +57,14 @@ public class PushFiltersToSource extends PhysicalOptimizerRules.ParameterizedOpt
List<Expression> pushable = new ArrayList<>();
List<Expression> nonPushable = new ArrayList<>();
for (Expression exp : splitAnd(filterExec.condition())) {
(canPushToSource(exp, pushdownPredicates) ? pushable : nonPushable).add(exp);
switch (translatable(exp, pushdownPredicates).finish()) {
case NO -> nonPushable.add(exp);
case YES -> pushable.add(exp);
case RECHECK -> {
pushable.add(exp);
nonPushable.add(exp);
}
}
}
return rewrite(pushdownPredicates, filterExec, queryExec, pushable, nonPushable, List.of());
}
@ -74,7 +81,14 @@ public class PushFiltersToSource extends PhysicalOptimizerRules.ParameterizedOpt
List<Expression> nonPushable = new ArrayList<>();
for (Expression exp : splitAnd(filterExec.condition())) {
Expression resExp = exp.transformUp(ReferenceAttribute.class, r -> aliasReplacedBy.resolve(r, r));
(canPushToSource(resExp, pushdownPredicates) ? pushable : nonPushable).add(exp);
switch (translatable(resExp, pushdownPredicates).finish()) {
case NO -> nonPushable.add(exp);
case YES -> pushable.add(exp);
case RECHECK -> {
nonPushable.add(exp);
nonPushable.add(exp);
}
}
}
// Replace field references with their actual field attributes
pushable.replaceAll(e -> e.transformDown(ReferenceAttribute.class, r -> aliasReplacedBy.resolve(r, r)));
@ -202,18 +216,4 @@ public class PushFiltersToSource extends PhysicalOptimizerRules.ParameterizedOpt
}
return changed ? CollectionUtils.combine(others, bcs, ranges) : pushable;
}
/**
* Check if the given expression can be pushed down to the source.
* This version of the check is called when we do not have SearchStats available. It assumes no exact subfields for TEXT fields,
* and makes the indexed/doc-values check using the isAggregatable flag only, which comes from field-caps, represents the field state
* over the entire cluster (is not node specific), and has risks for indexed=false/doc_values=true fields.
*/
public static boolean canPushToSource(Expression exp) {
return canPushToSource(exp, LucenePushdownPredicates.DEFAULT);
}
static boolean canPushToSource(Expression exp, LucenePushdownPredicates lucenePushdownPredicates) {
return exp instanceof TranslationAware aware && aware.translatable(lucenePushdownPredicates);
}
}

8
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushStatsToSource.java
View File

@ -10,6 +10,7 @@ package org.elasticsearch.xpack.esql.optimizer.rules.physical.local;
import org.elasticsearch.core.Tuple;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.xpack.esql.capabilities.TranslationAware;
import org.elasticsearch.xpack.esql.core.expression.Alias;
import org.elasticsearch.xpack.esql.core.expression.Attribute;
import org.elasticsearch.xpack.esql.core.expression.AttributeMap;
@ -33,7 +34,7 @@ import java.util.List;
import static java.util.Arrays.asList;
import static java.util.Collections.emptyList;
import static java.util.Collections.singletonList;
import static org.elasticsearch.xpack.esql.optimizer.rules.physical.local.PushFiltersToSource.canPushToSource;
import static org.elasticsearch.xpack.esql.capabilities.TranslationAware.translatable;
import static org.elasticsearch.xpack.esql.plan.physical.EsStatsQueryExec.StatsType.COUNT;
import static org.elasticsearch.xpack.esql.planner.TranslatorHandler.TRANSLATOR_HANDLER;
@ -107,7 +108,10 @@ public class PushStatsToSource extends PhysicalOptimizerRules.ParameterizedOptim
// That's because stats pushdown only works for 1 agg function (without BY); but in that case, filters
// are extracted into a separate filter node upstream from the aggregation (and hopefully pushed into
// the EsQueryExec separately).
if (canPushToSource(count.filter()) == false) {
if (translatable(
count.filter(),
LucenePushdownPredicates.DEFAULT
) != TranslationAware.Translatable.YES) {
return null; // can't push down
}
var countFilter = TRANSLATOR_HANDLER.asQuery(LucenePushdownPredicates.DEFAULT, count.filter());

7
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java
View File

@ -20,6 +20,7 @@ import org.elasticsearch.index.query.CoordinatorRewriteContext;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.SearchExecutionContext;
import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException;
import org.elasticsearch.xpack.esql.capabilities.TranslationAware;
import org.elasticsearch.xpack.esql.core.expression.AttributeSet;
import org.elasticsearch.xpack.esql.core.expression.Expression;
import org.elasticsearch.xpack.esql.core.expression.FoldContext;
@ -64,8 +65,8 @@ import static java.util.Arrays.asList;
import static org.elasticsearch.index.mapper.MappedFieldType.FieldExtractPreference.DOC_VALUES;
import static org.elasticsearch.index.mapper.MappedFieldType.FieldExtractPreference.EXTRACT_SPATIAL_BOUNDS;
import static org.elasticsearch.index.mapper.MappedFieldType.FieldExtractPreference.NONE;
import static org.elasticsearch.xpack.esql.capabilities.TranslationAware.translatable;
import static org.elasticsearch.xpack.esql.core.util.Queries.Clause.FILTER;
import static org.elasticsearch.xpack.esql.optimizer.rules.physical.local.PushFiltersToSource.canPushToSource;
import static org.elasticsearch.xpack.esql.planner.TranslatorHandler.TRANSLATOR_HANDLER;
public class PlannerUtils {
@ -245,7 +246,9 @@ public class PlannerUtils {
boolean matchesField = refsBuilder.removeIf(e -> fieldName.test(e.name()));
// the expression only contains the target reference
// and the expression is pushable (functions can be fully translated)
if (matchesField && refsBuilder.isEmpty() && canPushToSource(exp)) {
if (matchesField
&& refsBuilder.isEmpty()
&& translatable(exp, LucenePushdownPredicates.DEFAULT).finish() == TranslationAware.FinishedTranslatable.YES) {
matches.add(exp);
}
}

161
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java
View File

@ -17,7 +17,9 @@ import org.elasticsearch.TransportVersions;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.compute.lucene.LuceneSourceOperator;
import org.elasticsearch.compute.operator.DriverContext;
import org.elasticsearch.compute.operator.FilterOperator;
import org.elasticsearch.compute.operator.Warnings;
import org.elasticsearch.compute.querydsl.query.SingleValueMatchQuery;
import org.elasticsearch.index.mapper.IgnoredFieldMapper;
@ -50,6 +52,9 @@ import java.util.Objects;
* for now we're going to always wrap so we can always push. When we find cases
* where double checking is better we'll try that.
* </p>
* <p>
* NOTE: This will only work with {@code text} fields.
* </p>
*/
public class SingleValueQuery extends Query {
public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
@ -60,7 +65,7 @@ public class SingleValueQuery extends Query {
private final Query next;
private final String field;
private final boolean useSyntheticSourceDelegate;
private final UseSyntheticSourceDelegate useSyntheticSourceDelegate;
/**
* Build.
@ -71,6 +76,10 @@ public class SingleValueQuery extends Query {
* we often want to use its delegate.
*/
public SingleValueQuery(Query next, String field, boolean useSyntheticSourceDelegate) {
this(next, field, useSyntheticSourceDelegate ? UseSyntheticSourceDelegate.YES : UseSyntheticSourceDelegate.NO);
}
public SingleValueQuery(Query next, String field, UseSyntheticSourceDelegate useSyntheticSourceDelegate) {
super(next.source());
this.next = next;
this.field = field;
@ -79,9 +88,11 @@ public class SingleValueQuery extends Query {
@Override
protected AbstractBuilder asBuilder() {
return useSyntheticSourceDelegate
? new SyntheticSourceDelegateBuilder(next.toQueryBuilder(), field, next.source())
: new Builder(next.toQueryBuilder(), field, next.source());
return switch (useSyntheticSourceDelegate) {
case NO -> new Builder(next.toQueryBuilder(), field, next.source());
case YES -> new SyntheticSourceDelegateBuilder(next.toQueryBuilder(), field, next.source());
case YES_NEGATED -> new NegatedSyntheticSourceDelegateBuilder(next.toQueryBuilder(), field, next.source());
};
}
@Override
@ -91,7 +102,11 @@ public class SingleValueQuery extends Query {
@Override
public SingleValueQuery negate(Source source) {
return new SingleValueQuery(next.negate(source), field, useSyntheticSourceDelegate);
return new SingleValueQuery(next.negate(source), field, switch (useSyntheticSourceDelegate) {
case NO -> UseSyntheticSourceDelegate.NO;
case YES -> UseSyntheticSourceDelegate.YES_NEGATED;
case YES_NEGATED -> UseSyntheticSourceDelegate.YES;
});
}
@Override
@ -188,6 +203,28 @@ public class SingleValueQuery extends Query {
protected final int doHashCode() {
return Objects.hash(next, field);
}
protected final org.apache.lucene.search.Query simple(MappedFieldType ft, SearchExecutionContext context) throws IOException {
SingleValueMatchQuery singleValueQuery = new SingleValueMatchQuery(
context.getForField(ft, MappedFieldType.FielddataOperation.SEARCH),
Warnings.createWarnings(
DriverContext.WarningsMode.COLLECT,
source().source().getLineNumber(),
source().source().getColumnNumber(),
source().text()
),
"single-value function encountered multi-value"
);
org.apache.lucene.search.Query rewrite = singleValueQuery.rewrite(context.searcher());
if (rewrite instanceof MatchAllDocsQuery) {
// nothing to filter
return next().toQuery(context);
}
BooleanQuery.Builder builder = new BooleanQuery.Builder();
builder.add(next().toQuery(context), BooleanClause.Occur.FILTER);
builder.add(rewrite, BooleanClause.Occur.FILTER);
return builder.build();
}
}
/**
@ -227,25 +264,7 @@ public class SingleValueQuery extends Query {
if (ft == null) {
return new MatchNoDocsQuery("missing field [" + field() + "]");
}
SingleValueMatchQuery singleValueQuery = new SingleValueMatchQuery(
context.getForField(ft, MappedFieldType.FielddataOperation.SEARCH),
Warnings.createWarnings(
DriverContext.WarningsMode.COLLECT,
source().source().getLineNumber(),
source().source().getColumnNumber(),
source().text()
),
"single-value function encountered multi-value"
);
org.apache.lucene.search.Query rewrite = singleValueQuery.rewrite(context.searcher());
if (rewrite instanceof MatchAllDocsQuery) {
// nothing to filter
return next().toQuery(context);
}
BooleanQuery.Builder builder = new BooleanQuery.Builder();
builder.add(next().toQuery(context), BooleanClause.Occur.FILTER);
builder.add(rewrite, BooleanClause.Occur.FILTER);
return builder.build();
return simple(ft, context);
}
@Override
@ -255,7 +274,7 @@ public class SingleValueQuery extends Query {
}
/**
* Builds a {@code bool} query combining the "next" query, a {@link SingleValueMatchQuery},
* Builds a {@code bool} query ANDing the "next" query, a {@link SingleValueMatchQuery},
* and a {@link TermQuery} making sure we didn't ignore any values. Three total queries.
* This is only used if the "next" query matches fields that would not be ignored. Read all
* the paragraphs below to understand it. It's tricky!
@ -344,6 +363,92 @@ public class SingleValueQuery extends Query {
}
}
/**
* Builds a query matching either ignored values OR the union of {@code next} query
* and {@link SingleValueMatchQuery}. Three total queries. This is used to generate
* candidate matches for queries like {@code NOT(a == "b")} where some values of {@code a}
* are not indexed. In fact, let's use that as an example.
* <p>
* In that case you use a query for {@code a != "b"} as the "next" query. Then
* this query will find all documents where {@code a} is single valued and
* {@code == "b"} AND all documents that have ignored some values of {@code a}.
* This produces <strong>candidate</strong> matches for {@code NOT(a == "b")}.
* It'll find documents like:
* </p>
* <ul>
* <li>"a"</li>
* <li>ignored_value</li>
* <li>["a", ignored_value]</li>
* <li>[ignored_value1, ignored_value2]</li>
* <li>["b", ignored_field]</li>
* </ul>
* <p>
* The first and second of those <strong>should</strong> match {@code NOT(a == "b")}.
* The last three should be rejected. So! When using this query you <strong>must</strong>
* push this query to the {@link LuceneSourceOperator} <strong>and</strong>
* retain it in the {@link FilterOperator}.
* </p>
* <p>
* This will not find:
* </p>
* <ul>
* <li>"b"</li>
* </ul>
* <p>
* And that's also great! These can't match {@code NOT(a == "b")}
* </p>
*/
public static class NegatedSyntheticSourceDelegateBuilder extends AbstractBuilder {
NegatedSyntheticSourceDelegateBuilder(QueryBuilder next, String field, Source source) {
super(next, field, source);
}
@Override
public String getWriteableName() {
throw new UnsupportedOperationException("Not serialized");
}
@Override
protected void doXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject("negated_" + ENTRY.name);
builder.field("field", field() + ":synthetic_source_delegate");
builder.field("next", next(), params);
builder.field("source", source().toString());
builder.endObject();
}
@Override
public TransportVersion getMinimalSupportedVersion() {
throw new UnsupportedOperationException("Not serialized");
}
@Override
protected final org.apache.lucene.search.Query doToQuery(SearchExecutionContext context) throws IOException {
MappedFieldType ft = context.getFieldType(field());
if (ft == null) {
return new MatchNoDocsQuery("missing field [" + field() + "]");
}
ft = ((TextFieldMapper.TextFieldType) ft).syntheticSourceDelegate();
org.apache.lucene.search.Query svNext = simple(ft, context);
org.apache.lucene.search.Query ignored = new TermQuery(new org.apache.lucene.index.Term(IgnoredFieldMapper.NAME, ft.name()));
ignored = ignored.rewrite(context.searcher());
if (ignored instanceof MatchNoDocsQuery) {
return svNext;
}
BooleanQuery.Builder builder = new BooleanQuery.Builder();
builder.add(svNext, BooleanClause.Occur.SHOULD);
builder.add(ignored, BooleanClause.Occur.SHOULD);
return builder.build();
}
@Override
protected AbstractBuilder rewrite(QueryBuilder next) {
return new Builder(next, field(), source());
}
}
/**
* Write a {@link Source} including the text in it.
*/
@ -364,4 +469,10 @@ public class SingleValueQuery extends Query {
String text = in.readString();
return new Source(new Location(line, charPositionInLine), text);
}
public enum UseSyntheticSourceDelegate {
NO,
YES,
YES_NEGATED;
}
}

7
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java
View File

@ -11,6 +11,7 @@ import com.carrotsearch.randomizedtesting.annotations.Name;
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.xpack.esql.capabilities.TranslationAware;
import org.elasticsearch.xpack.esql.core.expression.Expression;
import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
import org.elasticsearch.xpack.esql.core.expression.Literal;
@ -113,7 +114,7 @@ public class EndsWithTests extends AbstractScalarFunctionTestCase {
new Literal(Source.EMPTY, "test", DataType.KEYWORD)
);
assertThat(function.translatable(LucenePushdownPredicates.DEFAULT), equalTo(false));
assertThat(function.translatable(LucenePushdownPredicates.DEFAULT), equalTo(TranslationAware.Translatable.NO));
}
public void testLuceneQuery_NonFoldableSuffix_NonTranslatable() {
@ -123,7 +124,7 @@ public class EndsWithTests extends AbstractScalarFunctionTestCase {
new FieldAttribute(Source.EMPTY, "field", new EsField("suffix", DataType.KEYWORD, Map.of(), true))
);
assertThat(function.translatable(LucenePushdownPredicates.DEFAULT), equalTo(false));
assertThat(function.translatable(LucenePushdownPredicates.DEFAULT), equalTo(TranslationAware.Translatable.NO));
}
public void testLuceneQuery_NonFoldableSuffix_Translatable() {
@ -133,7 +134,7 @@ public class EndsWithTests extends AbstractScalarFunctionTestCase {
new Literal(Source.EMPTY, "a*b?c\\", DataType.KEYWORD)
);
assertThat(function.translatable(LucenePushdownPredicates.DEFAULT), equalTo(true));
assertThat(function.translatable(LucenePushdownPredicates.DEFAULT), equalTo(TranslationAware.Translatable.YES));
var query = function.asQuery(LucenePushdownPredicates.DEFAULT, TranslatorHandler.TRANSLATOR_HANDLER);

7
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java
View File

@ -11,6 +11,7 @@ import com.carrotsearch.randomizedtesting.annotations.Name;
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.xpack.esql.capabilities.TranslationAware;
import org.elasticsearch.xpack.esql.core.expression.Expression;
import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
import org.elasticsearch.xpack.esql.core.expression.Literal;
@ -73,7 +74,7 @@ public class StartsWithTests extends AbstractScalarFunctionTestCase {
new Literal(Source.EMPTY, "test", DataType.KEYWORD)
);
assertThat(function.translatable(LucenePushdownPredicates.DEFAULT), equalTo(false));
assertThat(function.translatable(LucenePushdownPredicates.DEFAULT), equalTo(TranslationAware.Translatable.NO));
}
public void testLuceneQuery_NonFoldablePrefix_NonTranslatable() {
@ -83,7 +84,7 @@ public class StartsWithTests extends AbstractScalarFunctionTestCase {
new FieldAttribute(Source.EMPTY, "field", new EsField("prefix", DataType.KEYWORD, Map.of(), true))
);
assertThat(function.translatable(LucenePushdownPredicates.DEFAULT), equalTo(false));
assertThat(function.translatable(LucenePushdownPredicates.DEFAULT), equalTo(TranslationAware.Translatable.NO));
}
public void testLuceneQuery_NonFoldablePrefix_Translatable() {
@ -93,7 +94,7 @@ public class StartsWithTests extends AbstractScalarFunctionTestCase {
new Literal(Source.EMPTY, "a*b?c\\", DataType.KEYWORD)
);
assertThat(function.translatable(LucenePushdownPredicates.DEFAULT), equalTo(true));
assertThat(function.translatable(LucenePushdownPredicates.DEFAULT), equalTo(TranslationAware.Translatable.YES));
var query = function.asQuery(LucenePushdownPredicates.DEFAULT, TranslatorHandler.TRANSLATOR_HANDLER);

29
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java
View File

@ -59,6 +59,7 @@ import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute;
import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute;
import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison;
import org.elasticsearch.xpack.esql.core.querydsl.query.NotQuery;
import org.elasticsearch.xpack.esql.core.tree.Node;
import org.elasticsearch.xpack.esql.core.tree.Source;
import org.elasticsearch.xpack.esql.core.type.DataType;
@ -7795,7 +7796,6 @@ public class PhysicalPlanOptimizerTests extends ESTestCase {
}
public void testEqualsPushdownToDelegate() {
assumeFalse("disabled from bug", true);
var optimized = optimizedPlan(physicalPlan("""
FROM test
| WHERE job == "v"
@ -7824,6 +7824,33 @@ public class PhysicalPlanOptimizerTests extends ESTestCase {
as(limit2.child(), FilterExec.class);
}
public void testNotEqualsPushdownToDelegate() {
var optimized = optimizedPlan(physicalPlan("""
FROM test
| WHERE job != "v"
""", testDataLimitedRaw), SEARCH_STATS_SHORT_DELEGATES);
var limit = as(optimized, LimitExec.class);
var exchange = as(limit.child(), ExchangeExec.class);
var project = as(exchange.child(), ProjectExec.class);
var extract = as(project.child(), FieldExtractExec.class);
var limit2 = as(extract.child(), LimitExec.class);
var filter = as(limit2.child(), FilterExec.class);
var extract2 = as(filter.child(), FieldExtractExec.class);
var query = as(extract2.child(), EsQueryExec.class);
assertThat(
query.query(),
equalTo(
new BoolQueryBuilder().filter(
new SingleValueQuery(
new NotQuery(Source.EMPTY, new EqualsSyntheticSourceDelegate(Source.EMPTY, "job", "v")),
"job",
SingleValueQuery.UseSyntheticSourceDelegate.YES_NEGATED
).toQueryBuilder()
)
)
);
}
/*
* LimitExec[1000[INTEGER]]
* \_ExchangeExec[[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gender{f}#4, hire_date{f}#9, job{f}#10, job.raw{f}#11, langua

10
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQueryNegateTests.java
View File

@ -25,7 +25,15 @@ public class SingleValueQueryNegateTests extends ESTestCase {
var sv = new SingleValueQuery(new MatchAll(Source.EMPTY), "foo", useSyntheticSourceDelegate);
assertThat(
sv.negate(Source.EMPTY),
equalTo(new SingleValueQuery(new NotQuery(Source.EMPTY, new MatchAll(Source.EMPTY)), "foo", useSyntheticSourceDelegate))
equalTo(
new SingleValueQuery(
new NotQuery(Source.EMPTY, new MatchAll(Source.EMPTY)),
"foo",
useSyntheticSourceDelegate
? SingleValueQuery.UseSyntheticSourceDelegate.YES_NEGATED
: SingleValueQuery.UseSyntheticSourceDelegate.NO
)
)
);
}