mirrors
/
elasticsearch
mirror of https://mirror.osredm.com/root/elasticsearch.git
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add tracking for query rule types (#116357) 

* Add total rule type counts to list calls and xpack usage

* Add feature

* Update docs/changelog/116357.yaml

* Fix docs test failure & update yaml tests

* remove additional spaces

---------

Co-authored-by: Mark J. Hoy <mark.hoy@elastic.co>
This commit is contained in:
Kathleen DeRusso 2024-11-13 11:05:05 -05:00 committed by GitHub
parent 5ec0a843e1
commit 1b03a96e52
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 184 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
docs/changelog/116357.yaml Normal file
View File

@ -0,0 +1,5 @@
pr: 116357
summary: Add tracking for query rule types
area: Relevance
type: enhancement
issues: []

12
docs/reference/query-rules/apis/list-query-rulesets.asciidoc
View File

@ -124,7 +124,7 @@ PUT _query_rules/ruleset-3
},
{
"rule_id": "rule-3",
"type": "pinned",
"type": "exclude",
"criteria": [
{
"type": "fuzzy",
@ -178,6 +178,9 @@ A sample response:
"rule_total_count": 1,
"rule_criteria_types_counts": {
"exact": 1
},
"rule_type_counts": {
"pinned": 1
}
},
{
@ -186,6 +189,9 @@ A sample response:
"rule_criteria_types_counts": {
"exact": 1,
"fuzzy": 1
},
"rule_type_counts": {
"pinned": 2
}
},
{
@ -194,6 +200,10 @@ A sample response:
"rule_criteria_types_counts": {
"exact": 1,
"fuzzy": 2
},
"rule_type_counts": {
"pinned": 2,
"exclude": 1
}
}
]

2
server/src/main/java/org/elasticsearch/TransportVersions.java
View File

@ -196,6 +196,8 @@ public class TransportVersions {
public static final TransportVersion ADD_COMPATIBILITY_VERSIONS_TO_NODE_INFO = def(8_789_00_0);
public static final TransportVersion VERTEX_AI_INPUT_TYPE_ADDED = def(8_790_00_0);
public static final TransportVersion SKIP_INNER_HITS_SEARCH_SOURCE = def(8_791_00_0);
public static final TransportVersion QUERY_RULES_LIST_INCLUDES_TYPES = def(8_792_00_0);
/*
* STOP! READ THIS FIRST! No, really,
* ____ _____ ___ ____ _ ____ _____ _ ____ _____ _ _ ___ ____ _____ ___ ____ ____ _____ _

1
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/application/EnterpriseSearchFeatureSetUsage.java
View File

@ -34,6 +34,7 @@ public class EnterpriseSearchFeatureSetUsage extends XPackFeatureUsage {
public static final String MIN_RULE_COUNT = "min_rule_count";
public static final String MAX_RULE_COUNT = "max_rule_count";
public static final String RULE_CRITERIA_TOTAL_COUNTS = "rule_criteria_total_counts";
public static final String RULE_TYPE_TOTAL_COUNTS = "rule_type_total_counts";
private final Map<String, Object> searchApplicationsUsage;
private final Map<String, Object> analyticsCollectionsUsage;

91
x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/20_query_ruleset_list.yml
View File

@ -1,7 +1,4 @@
setup:
- requires:
cluster_features: [ "gte_v8.10.0" ]
reason: Introduced in 8.10.0
- do:
query_rules.put_ruleset:
ruleset_id: test-query-ruleset-3
@ -222,7 +219,7 @@ teardown:
body:
rules:
- rule_id: query-rule-id1
type: pinned
type: exclude
criteria:
- type: exact
metadata: query_string
@ -307,3 +304,89 @@ teardown:
- match: { error.type: 'security_exception' }
---
'List query rulesets - include rule types':
- requires:
cluster_features: [ "query_rule_list_types" ]
reason: 'List responses updated in 8.15.5 and 8.16.1'
- do:
query_rules.put_ruleset:
ruleset_id: a-test-query-ruleset-with-lots-of-criteria
body:
rules:
- rule_id: query-rule-id1
type: exclude
criteria:
- type: exact
metadata: query_string
values: [ puggles ]
- type: gt
metadata: year
values: [ 2023 ]
actions:
ids:
- 'id1'
- 'id2'
- rule_id: query-rule-id2
type: pinned
criteria:
- type: exact
metadata: query_string
values: [ pug ]
actions:
ids:
- 'id3'
- 'id4'
- rule_id: query-rule-id3
type: pinned
criteria:
- type: fuzzy
metadata: query_string
values: [ puggles ]
actions:
ids:
- 'id5'
- 'id6'
- rule_id: query-rule-id4
type: pinned
criteria:
- type: always
actions:
ids:
- 'id7'
- 'id8'
- rule_id: query-rule-id5
type: pinned
criteria:
- type: prefix
metadata: query_string
values: [ pug ]
- type: suffix
metadata: query_string
values: [ gle ]
actions:
ids:
- 'id9'
- 'id10'
- do:
query_rules.list_rulesets:
from: 0
size: 1
- match: { count: 4 }
# Alphabetical order by ruleset_id for results
- match: { results.0.ruleset_id: "a-test-query-ruleset-with-lots-of-criteria" }
- match: { results.0.rule_total_count: 5 }
- match:
results.0.rule_criteria_types_counts:
exact: 2
gt: 1
fuzzy: 1
prefix: 1
suffix: 1
always: 1
- match: { results.0.rule_type_counts: { pinned: 4, exclude: 1 } }

7
x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchFeatures.java
View File

@ -12,6 +12,7 @@ import org.elasticsearch.features.FeatureSpecification;
import org.elasticsearch.features.NodeFeature;
import org.elasticsearch.xpack.application.analytics.AnalyticsTemplateRegistry;
import org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry;
import org.elasticsearch.xpack.application.rules.action.ListQueryRulesetsAction;
import org.elasticsearch.xpack.application.rules.retriever.QueryRuleRetrieverBuilder;
import java.util.Map;
@ -23,7 +24,11 @@ public class EnterpriseSearchFeatures implements FeatureSpecification {
@Override
public Set<NodeFeature> getFeatures() {
return Set.of(QUERY_RULES_TEST_API, QueryRuleRetrieverBuilder.QUERY_RULE_RETRIEVERS_SUPPORTED);
return Set.of(
QUERY_RULES_TEST_API,
QueryRuleRetrieverBuilder.QUERY_RULE_RETRIEVERS_SUPPORTED,
ListQueryRulesetsAction.QUERY_RULE_LIST_TYPES
);
}
@Override

27
x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchUsageTransportAction.java
View File

@ -27,7 +27,6 @@ import org.elasticsearch.tasks.Task;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService;
import org.elasticsearch.xpack.application.analytics.action.GetAnalyticsCollectionAction;
import org.elasticsearch.xpack.application.rules.QueryRuleCriteriaType;
import org.elasticsearch.xpack.application.rules.QueryRulesIndexService;
import org.elasticsearch.xpack.application.rules.QueryRulesetListItem;
import org.elasticsearch.xpack.application.rules.action.ListQueryRulesetsAction;
@ -41,7 +40,6 @@ import org.elasticsearch.xpack.core.action.util.PageParams;
import org.elasticsearch.xpack.core.application.EnterpriseSearchFeatureSetUsage;
import java.util.Collections;
import java.util.EnumMap;
import java.util.HashMap;
import java.util.IntSummaryStatistics;
import java.util.List;
@ -226,20 +224,29 @@ public class EnterpriseSearchUsageTransportAction extends XPackUsageFeatureTrans
List<QueryRulesetListItem> results = response.queryPage().results();
IntSummaryStatistics ruleStats = results.stream().mapToInt(QueryRulesetListItem::ruleTotalCount).summaryStatistics();
Map<QueryRuleCriteriaType, Integer> criteriaTypeCountMap = new EnumMap<>(QueryRuleCriteriaType.class);
results.stream()
.flatMap(result -> result.criteriaTypeToCountMap().entrySet().stream())
.forEach(entry -> criteriaTypeCountMap.merge(entry.getKey(), entry.getValue(), Integer::sum));
Map<String, Object> ruleCriteriaTypeCountMap = new HashMap<>();
Map<String, Object> ruleTypeCountMap = new HashMap<>();
Map<String, Object> rulesTypeCountMap = new HashMap<>();
criteriaTypeCountMap.forEach((criteriaType, count) -> rulesTypeCountMap.put(criteriaType.name().toLowerCase(Locale.ROOT), count));
results.forEach(result -> {
populateCounts(ruleCriteriaTypeCountMap, result.criteriaTypeToCountMap());
populateCounts(ruleTypeCountMap, result.ruleTypeToCountMap());
});
queryRulesUsage.put(TOTAL_COUNT, response.queryPage().count());
queryRulesUsage.put(TOTAL_RULE_COUNT, ruleStats.getSum());
queryRulesUsage.put(MIN_RULE_COUNT, results.isEmpty() ? 0 : ruleStats.getMin());
queryRulesUsage.put(MAX_RULE_COUNT, results.isEmpty() ? 0 : ruleStats.getMax());
if (rulesTypeCountMap.isEmpty() == false) {
queryRulesUsage.put(RULE_CRITERIA_TOTAL_COUNTS, rulesTypeCountMap);
if (ruleCriteriaTypeCountMap.isEmpty() == false) {
queryRulesUsage.put(RULE_CRITERIA_TOTAL_COUNTS, ruleCriteriaTypeCountMap);
}
if (ruleTypeCountMap.isEmpty() == false) {
queryRulesUsage.put(EnterpriseSearchFeatureSetUsage.RULE_TYPE_TOTAL_COUNTS, ruleTypeCountMap);
}
}
private void populateCounts(Map<String, Object> targetMap, Map<? extends Enum<?>, Integer> sourceMap) {
sourceMap.forEach(
(key, value) -> targetMap.merge(key.name().toLowerCase(Locale.ROOT), value, (v1, v2) -> (Integer) v1 + (Integer) v2)
);
}
}

6
x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesIndexService.java
View File

@ -445,6 +445,7 @@ public class QueryRulesIndexService {
final List<LinkedHashMap<?, ?>> rules = ((List<LinkedHashMap<?, ?>>) sourceMap.get(QueryRuleset.RULES_FIELD.getPreferredName()));
final int numRules = rules.size();
final Map<QueryRuleCriteriaType, Integer> queryRuleCriteriaTypeToCountMap = new EnumMap<>(QueryRuleCriteriaType.class);
final Map<QueryRule.QueryRuleType, Integer> ruleTypeToCountMap = new EnumMap<>(QueryRule.QueryRuleType.class);
for (LinkedHashMap<?, ?> rule : rules) {
@SuppressWarnings("unchecked")
List<LinkedHashMap<?, ?>> criteriaList = ((List<LinkedHashMap<?, ?>>) rule.get(QueryRule.CRITERIA_FIELD.getPreferredName()));
@ -453,9 +454,12 @@ public class QueryRulesIndexService {
final QueryRuleCriteriaType queryRuleCriteriaType = QueryRuleCriteriaType.type(criteriaType);
queryRuleCriteriaTypeToCountMap.compute(queryRuleCriteriaType, (k, v) -> v == null ? 1 : v + 1);
}
final String ruleType = ((String) rule.get(QueryRule.TYPE_FIELD.getPreferredName()));
final QueryRule.QueryRuleType queryRuleType = QueryRule.QueryRuleType.queryRuleType(ruleType);
ruleTypeToCountMap.compute(queryRuleType, (k, v) -> v == null ? 1 : v + 1);
}
return new QueryRulesetListItem(rulesetId, numRules, queryRuleCriteriaTypeToCountMap);
return new QueryRulesetListItem(rulesetId, numRules, queryRuleCriteriaTypeToCountMap, ruleTypeToCountMap);
}
public record QueryRulesetResult(List<QueryRulesetListItem> rulesets, long totalResults) {}

32
x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesetListItem.java
View File

@ -32,10 +32,12 @@ public class QueryRulesetListItem implements Writeable, ToXContentObject {
public static final ParseField RULESET_ID_FIELD = new ParseField("ruleset_id");
public static final ParseField RULE_TOTAL_COUNT_FIELD = new ParseField("rule_total_count");
public static final ParseField RULE_CRITERIA_TYPE_COUNTS_FIELD = new ParseField("rule_criteria_types_counts");
public static final ParseField RULE_TYPE_COUNTS_FIELD = new ParseField("rule_type_counts");
private final String rulesetId;
private final int ruleTotalCount;
private final Map<QueryRuleCriteriaType, Integer> criteriaTypeToCountMap;
private final Map<QueryRule.QueryRuleType, Integer> ruleTypeToCountMap;
/**
* Constructs a QueryRulesetListItem.
@ -44,11 +46,17 @@ public class QueryRulesetListItem implements Writeable, ToXContentObject {
* @param ruleTotalCount The number of rules contained within the ruleset.
* @param criteriaTypeToCountMap A map of criteria type to the number of rules of that type.
*/
public QueryRulesetListItem(String rulesetId, int ruleTotalCount, Map<QueryRuleCriteriaType, Integer> criteriaTypeToCountMap) {
public QueryRulesetListItem(
String rulesetId,
int ruleTotalCount,
Map<QueryRuleCriteriaType, Integer> criteriaTypeToCountMap,
Map<QueryRule.QueryRuleType, Integer> ruleTypeToCountMap
) {
Objects.requireNonNull(rulesetId, "rulesetId cannot be null on a QueryRuleListItem");
this.rulesetId = rulesetId;
this.ruleTotalCount = ruleTotalCount;
this.criteriaTypeToCountMap = criteriaTypeToCountMap;
this.ruleTypeToCountMap = ruleTypeToCountMap;
}
public QueryRulesetListItem(StreamInput in) throws IOException {
@ -59,6 +67,11 @@ public class QueryRulesetListItem implements Writeable, ToXContentObject {
} else {
this.criteriaTypeToCountMap = Map.of();
}
if (in.getTransportVersion().onOrAfter(TransportVersions.QUERY_RULES_LIST_INCLUDES_TYPES)) {
this.ruleTypeToCountMap = in.readMap(m -> in.readEnum(QueryRule.QueryRuleType.class), StreamInput::readInt);
} else {
this.ruleTypeToCountMap = Map.of();
}
}
@Override
@ -71,6 +84,11 @@ public class QueryRulesetListItem implements Writeable, ToXContentObject {
builder.field(criteriaType.name().toLowerCase(Locale.ROOT), criteriaTypeToCountMap.get(criteriaType));
}
builder.endObject();
builder.startObject(RULE_TYPE_COUNTS_FIELD.getPreferredName());
for (QueryRule.QueryRuleType ruleType : ruleTypeToCountMap.keySet()) {
builder.field(ruleType.name().toLowerCase(Locale.ROOT), ruleTypeToCountMap.get(ruleType));
}
builder.endObject();
builder.endObject();
return builder;
}
@ -82,6 +100,9 @@ public class QueryRulesetListItem implements Writeable, ToXContentObject {
if (out.getTransportVersion().onOrAfter(EXPANDED_RULESET_COUNT_TRANSPORT_VERSION)) {
out.writeMap(criteriaTypeToCountMap, StreamOutput::writeEnum, StreamOutput::writeInt);
}
if (out.getTransportVersion().onOrAfter(TransportVersions.QUERY_RULES_LIST_INCLUDES_TYPES)) {
out.writeMap(ruleTypeToCountMap, StreamOutput::writeEnum, StreamOutput::writeInt);
}
}
/**
@ -106,6 +127,10 @@ public class QueryRulesetListItem implements Writeable, ToXContentObject {
return criteriaTypeToCountMap;
}
public Map<QueryRule.QueryRuleType, Integer> ruleTypeToCountMap() {
return ruleTypeToCountMap;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
@ -113,11 +138,12 @@ public class QueryRulesetListItem implements Writeable, ToXContentObject {
QueryRulesetListItem that = (QueryRulesetListItem) o;
return ruleTotalCount == that.ruleTotalCount
&& Objects.equals(rulesetId, that.rulesetId)
&& Objects.equals(criteriaTypeToCountMap, that.criteriaTypeToCountMap);
&& Objects.equals(criteriaTypeToCountMap, that.criteriaTypeToCountMap)
&& Objects.equals(ruleTypeToCountMap, that.ruleTypeToCountMap);
}
@Override
public int hashCode() {
return Objects.hash(rulesetId, ruleTotalCount, criteriaTypeToCountMap);
return Objects.hash(rulesetId, ruleTotalCount, criteriaTypeToCountMap, ruleTypeToCountMap);
}
}

3
x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/ListQueryRulesetsAction.java
View File

@ -13,6 +13,7 @@ import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.action.ActionType;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.features.NodeFeature;
import org.elasticsearch.xcontent.ConstructingObjectParser;
import org.elasticsearch.xcontent.ParseField;
import org.elasticsearch.xcontent.ToXContentObject;
@ -33,6 +34,8 @@ public class ListQueryRulesetsAction {
public static final String NAME = "cluster:admin/xpack/query_rules/list";
public static final ActionType<ListQueryRulesetsAction.Response> INSTANCE = new ActionType<>(NAME);
public static final NodeFeature QUERY_RULE_LIST_TYPES = new NodeFeature("query_rule_list_types");
private ListQueryRulesetsAction() {/* no instances */}
public static class Request extends ActionRequest implements ToXContentObject {

22
x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/ListQueryRulesetsActionResponseBWCSerializingTests.java
View File

@ -8,8 +8,10 @@
package org.elasticsearch.xpack.application.rules.action;
import org.elasticsearch.TransportVersion;
import org.elasticsearch.TransportVersions;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.xpack.application.EnterpriseSearchModuleTestUtils;
import org.elasticsearch.xpack.application.rules.QueryRule;
import org.elasticsearch.xpack.application.rules.QueryRuleCriteriaType;
import org.elasticsearch.xpack.application.rules.QueryRuleset;
import org.elasticsearch.xpack.application.rules.QueryRulesetListItem;
@ -32,9 +34,13 @@ public class ListQueryRulesetsActionResponseBWCSerializingTests extends Abstract
QueryRuleset queryRuleset = EnterpriseSearchModuleTestUtils.randomQueryRuleset();
Map<QueryRuleCriteriaType, Integer> criteriaTypeToCountMap = Map.of(
randomFrom(QueryRuleCriteriaType.values()),
randomIntBetween(0, 10)
randomIntBetween(1, 10)
);
return new QueryRulesetListItem(queryRuleset.id(), queryRuleset.rules().size(), criteriaTypeToCountMap);
Map<QueryRule.QueryRuleType, Integer> ruleTypeToCountMap = Map.of(
randomFrom(QueryRule.QueryRuleType.values()),
randomIntBetween(1, 10)
);
return new QueryRulesetListItem(queryRuleset.id(), queryRuleset.rules().size(), criteriaTypeToCountMap, ruleTypeToCountMap);
}), randomLongBetween(0, 1000));
}
@ -53,12 +59,20 @@ public class ListQueryRulesetsActionResponseBWCSerializingTests extends Abstract
ListQueryRulesetsAction.Response instance,
TransportVersion version
) {
if (version.onOrAfter(QueryRulesetListItem.EXPANDED_RULESET_COUNT_TRANSPORT_VERSION)) {
if (version.onOrAfter(TransportVersions.QUERY_RULES_LIST_INCLUDES_TYPES)) {
return instance;
} else if (version.onOrAfter(QueryRulesetListItem.EXPANDED_RULESET_COUNT_TRANSPORT_VERSION)) {
List<QueryRulesetListItem> updatedResults = new ArrayList<>();
for (QueryRulesetListItem listItem : instance.queryPage.results()) {
updatedResults.add(
new QueryRulesetListItem(listItem.rulesetId(), listItem.ruleTotalCount(), listItem.criteriaTypeToCountMap(), Map.of())
);
}
return new ListQueryRulesetsAction.Response(updatedResults, instance.queryPage.count());
} else {
List<QueryRulesetListItem> updatedResults = new ArrayList<>();
for (QueryRulesetListItem listItem : instance.queryPage.results()) {
updatedResults.add(new QueryRulesetListItem(listItem.rulesetId(), listItem.ruleTotalCount(), Map.of()));
updatedResults.add(new QueryRulesetListItem(listItem.rulesetId(), listItem.ruleTotalCount(), Map.of(), Map.of()));
}
return new ListQueryRulesetsAction.Response(updatedResults, instance.queryPage.count());
}