ESQL: Revert "Allow partial results by default in ES|QL (#125060)" (#126286)

This reverts commit 81555cc9d9 from
https://github.com/elastic/elasticsearch/pull/125060.

Fix https://github.com/elastic/elasticsearch/issues/126275

@idegtiarenko and I investigated and believe this needs reverting:
silently dropping results from the query response in case any index is
missing can lead to real problems if users don't spot their mistake. I'm
also not sure if all the results will get dropped, or only from some
nodes/shards/clusters, meaning that this might be hard to spot by users
if only some results get dropped.

The main PR has no transport version bump, no new ESQL capability, and
was merged 15h ago - so it should be safe to just revert it. I noticed
there was a linked Serverless PR on the original PR, but it merely
disabled some obsolete tests on Serverless and doesn't require reverting
itself.

docs/changelog/125060.yaml

@@ -1,16 +0,0 @@
-pr: 125060
-summary: Allow partial results by default in ES|QL
-area: ES|QL
-type: breaking
-issues: [122802]
-
-breaking:
-  title: Allow partial results by default in ES|QL
-  area: ES|QL
-  details: >-
-    In earlier versions of {es}, ES|QL would fail the entire query if it encountered any error. ES|QL now returns partial results instead of failing when encountering errors.
-
-  impact: >-
-    Callers should check the `is_partial` flag returned in the response to determine if the result is partial or complete. If returning partial results is not desired, this option can be overridden per request via an `allow_partial_results` parameter in the query URL or globally via the cluster setting `esql.query.allow_partial_results`.
-
-  notable: true

docs/changelog/126286.yaml

@@ -0,0 +1,6 @@
+pr: 126286
+summary: Revert "Allow partial results by default in ES|QL"
+area: ES|QL
+type: bug
+issues:
+ - 126275

docs/release-notes/breaking-changes.md

@@ -14,11 +14,6 @@ To learn how to upgrade, check out <uprade docs>.
 
 % ## Next version [elasticsearch-nextversion-breaking-changes]
 
-## 9.1.0 [elasticsearch-910-breaking-changes]
-
-ES|QL
-:   * Allow partial results by default in ES|QL [#125060](https://github.com/elastic/elasticsearch/pull/125060)
-
 ## 9.0.0 [elasticsearch-900-breaking-changes]
 
 Allocation

test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/Clusters.java

@@ -21,7 +21,6 @@ public class Clusters {
             .module("test-esql-heap-attack")
             .setting("xpack.security.enabled", "false")
             .setting("xpack.license.self_generated.type", "trial")
-            .setting("esql.query.allow_partial_results", "false")
             .jvmArg("-Xmx512m");
         String javaVersion = JvmInfo.jvmInfo().version();
         if (javaVersion.equals("20") || javaVersion.equals("21")) {

x-pack/plugin/build.gradle

@@ -124,7 +124,6 @@ tasks.named("yamlRestCompatTestTransform").configure({ task ->
   task.skipTest("ml/post_data/Test POST data job api, flush, close and verify DataCounts doc", "Flush API is deprecated")
   task.replaceValueInMatch("Size", 49, "Test flamegraph from profiling-events")
   task.replaceValueInMatch("Size", 49, "Test flamegraph from test-events")
-  task.skipTest("esql/63_enrich_int_range/Invalid age as double", "TODO: require disable allow_partial_results")
 })
 
 tasks.named('yamlRestCompatTest').configure {

x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java

@@ -312,10 +312,7 @@ public class EsqlSecurityIT extends ESRestTestCase {
         searchRequest.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("es-security-runas-user", "metadata1_read2"));
 
         // ES|QL query on the same index pattern
-        var esqlResp = expectThrows(
+        var esqlResp = expectThrows(ResponseException.class, () -> runESQLCommand("metadata1_read2", "FROM index-user1,index-user2"));
-            ResponseException.class,
-            () -> runESQLCommand("metadata1_read2", "FROM index-user1,index-user2", false)
-        );
         var srchResp = expectThrows(ResponseException.class, () -> client().performRequest(searchRequest));
 
         for (ResponseException r : List.of(esqlResp, srchResp)) {
@@ -334,8 +331,7 @@ public class EsqlSecurityIT extends ESRestTestCase {
             ResponseException.class,
             () -> runESQLCommand(
                 "metadata1_read2",
-                "FROM index-user1,index-user2 METADATA _index | STATS sum=sum(value), index=VALUES(_index)",
+                "FROM index-user1,index-user2 METADATA _index | STATS sum=sum(value), index=VALUES(_index)"
-                false
             )
         );
         assertThat(
@@ -348,7 +344,7 @@ public class EsqlSecurityIT extends ESRestTestCase {
 
         resp = expectThrows(
             ResponseException.class,
-            () -> runESQLCommand("metadata1_read2", "FROM index-user1,index-user2 METADATA _index | STATS index=VALUES(_index)", false)
+            () -> runESQLCommand("metadata1_read2", "FROM index-user1,index-user2 METADATA _index | STATS index=VALUES(_index)")
         );
         assertThat(
             EntityUtils.toString(resp.getResponse().getEntity()),
@@ -360,7 +356,7 @@ public class EsqlSecurityIT extends ESRestTestCase {
 
         resp = expectThrows(
             ResponseException.class,
-            () -> runESQLCommand("metadata1_read2", "FROM index-user1,index-user2 | STATS sum=sum(value)", false)
+            () -> runESQLCommand("metadata1_read2", "FROM index-user1,index-user2 | STATS sum=sum(value)")
         );
         assertThat(
             EntityUtils.toString(resp.getResponse().getEntity()),
@@ -372,7 +368,7 @@ public class EsqlSecurityIT extends ESRestTestCase {
 
         resp = expectThrows(
             ResponseException.class,
-            () -> runESQLCommand("alias_user1", "FROM first-alias,index-user1 METADATA _index | KEEP _index, org, value | LIMIT 10", false)
+            () -> runESQLCommand("alias_user1", "FROM first-alias,index-user1 METADATA _index | KEEP _index, org, value | LIMIT 10")
         );
         assertThat(
             EntityUtils.toString(resp.getResponse().getEntity()),
@@ -386,8 +382,7 @@ public class EsqlSecurityIT extends ESRestTestCase {
             ResponseException.class,
             () -> runESQLCommand(
                 "alias_user2",
-                "from second-alias,index-user2 METADATA _index | stats sum=sum(value), index=VALUES(_index)",
+                "from second-alias,index-user2 METADATA _index | stats sum=sum(value), index=VALUES(_index)"
-                false
             )
         );
         assertThat(
@@ -831,10 +826,6 @@ public class EsqlSecurityIT extends ESRestTestCase {
     }
 
     protected Response runESQLCommand(String user, String command) throws IOException {
-        return runESQLCommand(user, command, null);
-    }
-
-    protected Response runESQLCommand(String user, String command, Boolean allowPartialResults) throws IOException {
         if (command.toLowerCase(Locale.ROOT).contains("limit") == false) {
             // add a (high) limit to avoid warnings on default limit
             command += " | limit 10000000";
@@ -848,9 +839,6 @@ public class EsqlSecurityIT extends ESRestTestCase {
         request.setJsonEntity(Strings.toString(json));
         request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("es-security-runas-user", user));
         request.addParameter("error_trace", "true");
-        if (allowPartialResults != null) {
-            request.addParameter("allow_partial_results", Boolean.toString(allowPartialResults));
-        }
         return client().performRequest(request);
     }
 

x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/EsqlRestValidationIT.java

@@ -83,6 +83,6 @@ public class EsqlRestValidationIT extends EsqlRestValidationTestCase {
 
     @Before
     public void skipTestOnOldVersions() {
-        assumeTrue("skip on old versions", Clusters.localClusterVersion().equals(Version.V_8_19_0));
+        assumeTrue("skip on old versions", Clusters.localClusterVersion().equals(Version.V_8_16_0));
     }
 }

x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/RequestIndexFilteringIT.java

@@ -28,7 +28,6 @@ import org.junit.rules.RuleChain;
 import org.junit.rules.TestRule;
 
 import java.io.IOException;
-import java.util.List;
 import java.util.Map;
 
 import static org.elasticsearch.test.MapMatcher.assertMap;
@@ -88,12 +87,6 @@ public class RequestIndexFilteringIT extends RequestIndexFilteringTestCase {
 
     @Override
     public Map<String, Object> runEsql(RestEsqlTestCase.RequestObjectBuilder requestObject) throws IOException {
-        if (requestObject.allowPartialResults() != null) {
-            assumeTrue(
-                "require allow_partial_results on local cluster",
-                clusterHasCapability("POST", "/_query", List.of(), List.of("support_partial_results")).orElse(false)
-            );
-        }
         requestObject.includeCCSMetadata(true);
         return super.runEsql(requestObject);
     }

x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java

@@ -111,7 +111,7 @@ public class RestEsqlIT extends RestEsqlTestCase {
             request.setJsonEntity("{\"f\":" + i + "}");
             assertOK(client().performRequest(request));
         }
-        RequestObjectBuilder builder = requestObjectBuilder().query("from test-index | limit 1 | keep f").allowPartialResults(false);
+        RequestObjectBuilder builder = requestObjectBuilder().query("from test-index | limit 1 | keep f");
         builder.pragmas(Settings.builder().put("data_partitioning", "invalid-option").build());
         ResponseException re = expectThrows(ResponseException.class, () -> runEsqlSync(builder));
         assertThat(EntityUtils.toString(re.getResponse().getEntity()), containsString("No enum constant"));

x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlRestValidationTestCase.java

@@ -129,7 +129,6 @@ public abstract class EsqlRestValidationTestCase extends ESRestTestCase {
         final var request = new Request("POST", "/_query");
         request.addParameter("error_trace", "true");
         request.addParameter("pretty", "true");
-        request.addParameter("allow_partial_results", Boolean.toString(false));
         request.setJsonEntity(
             Strings.toString(JsonXContent.contentBuilder().startObject().field("query", "from " + indexName).endObject())
         );

x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java

@@ -198,26 +198,17 @@ public abstract class RequestIndexFilteringTestCase extends ESRestTestCase {
         int docsTest1 = 0; // we are interested only in the created index, not necessarily that it has data
         indexTimestampData(docsTest1, "test1", "2024-11-26", "id1");
 
-        ResponseException e = expectThrows(
+        ResponseException e = expectThrows(ResponseException.class, () -> runEsql(timestampFilter("gte", "2020-01-01").query(from("foo"))));
-            ResponseException.class,
-            () -> runEsql(timestampFilter("gte", "2020-01-01").query(from("foo")).allowPartialResults(false))
-        );
         assertEquals(400, e.getResponse().getStatusLine().getStatusCode());
         assertThat(e.getMessage(), containsString("verification_exception"));
         assertThat(e.getMessage(), anyOf(containsString("Unknown index [foo]"), containsString("Unknown index [remote_cluster:foo]")));
 
-        e = expectThrows(
+        e = expectThrows(ResponseException.class, () -> runEsql(timestampFilter("gte", "2020-01-01").query(from("foo*"))));
-            ResponseException.class,
-            () -> runEsql(timestampFilter("gte", "2020-01-01").query(from("foo*")).allowPartialResults(false))
-        );
         assertEquals(400, e.getResponse().getStatusLine().getStatusCode());
         assertThat(e.getMessage(), containsString("verification_exception"));
         assertThat(e.getMessage(), anyOf(containsString("Unknown index [foo*]"), containsString("Unknown index [remote_cluster:foo*]")));
 
-        e = expectThrows(
+        e = expectThrows(ResponseException.class, () -> runEsql(timestampFilter("gte", "2020-01-01").query(from("foo", "test1"))));
-            ResponseException.class,
-            () -> runEsql(timestampFilter("gte", "2020-01-01").query(from("foo", "test1")).allowPartialResults(false))
-        );
         assertEquals(404, e.getResponse().getStatusLine().getStatusCode());
         assertThat(e.getMessage(), containsString("index_not_found_exception"));
         assertThat(e.getMessage(), anyOf(containsString("no such index [foo]"), containsString("no such index [remote_cluster:foo]")));
@@ -226,7 +217,7 @@ public abstract class RequestIndexFilteringTestCase extends ESRestTestCase {
             var pattern = from("test1");
             e = expectThrows(
                 ResponseException.class,
-                () -> runEsql(timestampFilter("gte", "2020-01-01").query(pattern + " | LOOKUP JOIN foo ON id1").allowPartialResults(false))
+                () -> runEsql(timestampFilter("gte", "2020-01-01").query(pattern + " | LOOKUP JOIN foo ON id1"))
             );
             assertEquals(400, e.getResponse().getStatusLine().getStatusCode());
             assertThat(

x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java

@@ -132,7 +132,7 @@ public abstract class RestEsqlTestCase extends ESRestTestCase {
         private Boolean includeCCSMetadata = null;
 
         private CheckedConsumer<XContentBuilder, IOException> filter;
-        private Boolean allowPartialResults = null;
+        private Boolean allPartialResults = null;
 
         public RequestObjectBuilder() throws IOException {
             this(randomFrom(XContentType.values()));
@@ -210,15 +210,11 @@ public abstract class RestEsqlTestCase extends ESRestTestCase {
             return this;
         }
 
-        public RequestObjectBuilder allowPartialResults(boolean allowPartialResults) {
+        public RequestObjectBuilder allPartialResults(boolean allPartialResults) {
-            this.allowPartialResults = allowPartialResults;
+            this.allPartialResults = allPartialResults;
             return this;
         }
 
-        public Boolean allowPartialResults() {
-            return allowPartialResults;
-        }
-
         public RequestObjectBuilder build() throws IOException {
             if (isBuilt == false) {
                 if (tables != null) {
@@ -1373,8 +1369,8 @@ public abstract class RestEsqlTestCase extends ESRestTestCase {
         requestObject.build();
         Request request = prepareRequest(mode);
         String mediaType = attachBody(requestObject, request);
-        if (requestObject.allowPartialResults != null) {
+        if (requestObject.allPartialResults != null) {
-            request.addParameter("allow_partial_results", String.valueOf(requestObject.allowPartialResults));
+            request.addParameter("allow_partial_results", String.valueOf(requestObject.allPartialResults));
         }
 
         RequestOptions.Builder options = request.getOptions().toBuilder();

x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractCrossClusterTestCase.java

@@ -25,7 +25,6 @@ import org.elasticsearch.test.XContentTestUtils;
 import org.elasticsearch.transport.RemoteClusterAware;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.json.JsonXContent;
-import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
 import org.junit.After;
 import org.junit.Before;
 
@@ -77,11 +76,6 @@ public abstract class AbstractCrossClusterTestCase extends AbstractMultiClusters
         return plugins;
     }
 
-    @Override
-    protected Settings nodeSettings() {
-        return Settings.builder().put(super.nodeSettings()).put(EsqlPlugin.QUERY_ALLOW_PARTIAL_RESULTS.getKey(), false).build();
-    }
-
     public static class InternalExchangePlugin extends Plugin {
         @Override
         public List<Setting<?>> getSettings() {

x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractEsqlIntegTestCase.java

@@ -139,14 +139,6 @@ public abstract class AbstractEsqlIntegTestCase extends ESIntegTestCase {
         return CollectionUtils.appendToCopy(super.nodePlugins(), EsqlPlugin.class);
     }
 
-    @Override
-    protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
-        return Settings.builder()
-            .put(super.nodeSettings(nodeOrdinal, otherSettings))
-            .put(EsqlPlugin.QUERY_ALLOW_PARTIAL_RESULTS.getKey(), false)
-            .build();
-    }
-
     protected void setRequestCircuitBreakerLimit(ByteSizeValue limit) {
         if (limit != null) {
             assertAcked(

x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterCancellationIT.java

@@ -14,20 +14,25 @@ import org.elasticsearch.action.bulk.BulkRequestBuilder;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.action.support.WriteRequest;
-import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.compute.operator.DriverTaskRunner;
 import org.elasticsearch.compute.operator.exchange.ExchangeService;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.tasks.TaskCancelledException;
 import org.elasticsearch.tasks.TaskInfo;
+import org.elasticsearch.test.AbstractMultiClustersTestCase;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.json.JsonXContent;
 import org.elasticsearch.xpack.esql.EsqlTestUtils;
 import org.elasticsearch.xpack.esql.plugin.ComputeService;
+import org.junit.After;
+import org.junit.Before;
 
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
 
@@ -39,7 +44,7 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
 
-public class CrossClusterCancellationIT extends AbstractCrossClusterTestCase {
+public class CrossClusterCancellationIT extends AbstractMultiClustersTestCase {
     private static final String REMOTE_CLUSTER = "cluster-a";
 
     @Override
@@ -48,11 +53,35 @@ public class CrossClusterCancellationIT extends AbstractCrossClusterTestCase {
     }
 
     @Override
-    protected Settings nodeSettings() {
+    protected Collection<Class<? extends Plugin>> nodePlugins(String clusterAlias) {
-        return Settings.builder()
+        List<Class<? extends Plugin>> plugins = new ArrayList<>(super.nodePlugins(clusterAlias));
-            .put(super.nodeSettings())
+        plugins.add(EsqlPluginWithEnterpriseOrTrialLicense.class);
-            .put(ExchangeService.INACTIVE_SINKS_INTERVAL_SETTING, TimeValue.timeValueMillis(between(3000, 4000)))
+        plugins.add(InternalExchangePlugin.class);
-            .build();
+        plugins.add(SimplePauseFieldPlugin.class);
+        return plugins;
+    }
+
+    public static class InternalExchangePlugin extends Plugin {
+        @Override
+        public List<Setting<?>> getSettings() {
+            return List.of(
+                Setting.timeSetting(
+                    ExchangeService.INACTIVE_SINKS_INTERVAL_SETTING,
+                    TimeValue.timeValueMillis(between(3000, 4000)),
+                    Setting.Property.NodeScope
+                )
+            );
+        }
+    }
+
+    @Before
+    public void resetPlugin() {
+        SimplePauseFieldPlugin.resetPlugin();
+    }
+
+    @After
+    public void releasePlugin() {
+        SimplePauseFieldPlugin.release();
     }
 
     @Override

x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/ManyShardsIT.java

@@ -72,7 +72,6 @@ public class ManyShardsIT extends AbstractEsqlIntegTestCase {
     @Override
     protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
         return Settings.builder()
-            .put(super.nodeSettings(nodeOrdinal, otherSettings))
             .put(ExchangeService.INACTIVE_SINKS_INTERVAL_SETTING, TimeValue.timeValueMillis(between(3000, 5000)))
             .build();
     }

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java

@@ -105,7 +105,7 @@ public class EsqlPlugin extends Plugin implements ActionPlugin {
 
     public static final Setting<Boolean> QUERY_ALLOW_PARTIAL_RESULTS = Setting.boolSetting(
         "esql.query.allow_partial_results",
-        true,
+        false,
         Setting.Property.NodeScope,
         Setting.Property.Dynamic
     );

x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/CrossClusterEsqlRCS1MissingIndicesIT.java

@@ -47,7 +47,6 @@ public class CrossClusterEsqlRCS1MissingIndicesIT extends AbstractRemoteClusterS
             .apply(commonClusterConfig)
             .setting("remote_cluster.port", "0")
             .setting("xpack.ml.enabled", "false")
-            .setting("esql.query.allow_partial_results", "false")
             .setting("xpack.security.remote_cluster_server.ssl.enabled", () -> String.valueOf(SSL_ENABLED_REF.get()))
             .setting("xpack.security.remote_cluster_server.ssl.key", "remote-cluster.key")
             .setting("xpack.security.remote_cluster_server.ssl.certificate", "remote-cluster.crt")
@@ -63,7 +62,6 @@ public class CrossClusterEsqlRCS1MissingIndicesIT extends AbstractRemoteClusterS
             .module("x-pack-enrich")
             .apply(commonClusterConfig)
             .setting("xpack.ml.enabled", "false")
-            .setting("esql.query.allow_partial_results", "false")
             .setting("xpack.security.remote_cluster_client.ssl.enabled", () -> String.valueOf(SSL_ENABLED_REF.get()))
             .build();
     }

x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityEsqlIT.java

@@ -78,7 +78,6 @@ public class RemoteClusterSecurityEsqlIT extends AbstractRemoteClusterSecurityTe
             .apply(commonClusterConfig)
             .setting("remote_cluster.port", "0")
             .setting("xpack.ml.enabled", "false")
-            .setting("esql.query.allow_partial_results", "false")
             .setting("xpack.security.remote_cluster_server.ssl.enabled", () -> String.valueOf(SSL_ENABLED_REF.get()))
             .setting("xpack.security.remote_cluster_server.ssl.key", "remote-cluster.key")
             .setting("xpack.security.remote_cluster_server.ssl.certificate", "remote-cluster.crt")
@@ -98,7 +97,6 @@ public class RemoteClusterSecurityEsqlIT extends AbstractRemoteClusterSecurityTe
             .module("ingest-common")
             .apply(commonClusterConfig)
             .setting("xpack.ml.enabled", "false")
-            .setting("esql.query.allow_partial_results", "false")
             .setting("xpack.security.remote_cluster_client.ssl.enabled", () -> String.valueOf(SSL_ENABLED_REF.get()))
             .setting("xpack.security.remote_cluster_client.ssl.certificate_authorities", "remote-cluster-ca.crt")
             .setting("xpack.security.authc.token.enabled", "true")