mirrors
/
elasticsearch
mirror of https://mirror.osredm.com/root/elasticsearch.git
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Split Grok's responsibilities (#93987) 

* extract ECS compatibility modes
* moves builtin patterns loading logic to its own class
* split loading logic into their own method and extracting commonalities
* 2 new alias methods to clearly state what pattern are you trying to get
This commit is contained in:
Pablo Alcantar Morales 2023-03-01 10:10:54 +01:00 committed by GitHub
parent 044afe0813
commit 66fcf65a58
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 239 additions and 194 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

123
libs/grok/src/main/java/org/elasticsearch/grok/Grok.java
View File

@ -16,16 +16,9 @@ import org.joni.Regex;
import org.joni.Region;
import org.joni.Syntax;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@ -33,14 +26,6 @@ import java.util.function.Consumer;
public final class Grok {
public static final String[] ECS_COMPATIBILITY_MODES = { "disabled", "v1" };
/**
* Patterns built in to the grok library.
*/
private static Map<String, String> LEGACY_PATTERNS;
private static Map<String, String> ECS_V1_PATTERNS;
private static final String NAME_GROUP = "name";
private static final String SUBNAME_GROUP = "subname";
private static final String PATTERN_GROUP = "pattern";
@ -329,112 +314,4 @@ public final class Grok {
return compiledExpression;
}
/**
* Load built-in patterns.
*/
public static synchronized Map<String, String> getBuiltinPatterns(boolean ecsCompatibility) {
if (ecsCompatibility) {
if (ECS_V1_PATTERNS == null) {
ECS_V1_PATTERNS = loadPatterns(ecsCompatibility);
}
return ECS_V1_PATTERNS;
} else {
if (LEGACY_PATTERNS == null) {
LEGACY_PATTERNS = loadPatterns(ecsCompatibility);
}
return LEGACY_PATTERNS;
}
}
public static Map<String, String> getBuiltinPatterns(String ecsCompatibility) {
if (isValidEcsCompatibilityMode(ecsCompatibility)) {
return getBuiltinPatterns(ECS_COMPATIBILITY_MODES[1].equals(ecsCompatibility));
} else {
throw new IllegalArgumentException("unsupported ECS compatibility mode [" + ecsCompatibility + "]");
}
}
public static boolean isValidEcsCompatibilityMode(String ecsCompatibility) {
return Arrays.asList(ECS_COMPATIBILITY_MODES).contains(ecsCompatibility);
}
private static Map<String, String> loadPatterns(boolean ecsCompatibility) {
String[] legacyPatternNames = {
"aws",
"bacula",
"bind",
"bro",
"exim",
"firewalls",
"grok-patterns",
"haproxy",
"httpd",
"java",
"junos",
"linux-syslog",
"maven",
"mcollective-patterns",
"mongodb",
"nagios",
"postgresql",
"rails",
"redis",
"ruby",
"squid" };
String[] ecsPatternNames = {
"aws",
"bacula",
"bind",
"bro",
"exim",
"firewalls",
"grok-patterns",
"haproxy",
"httpd",
"java",
"junos",
"linux-syslog",
"maven",
"mcollective",
"mongodb",
"nagios",
"postgresql",
"rails",
"redis",
"ruby",
"squid",
"zeek" };
String[] patternNames = ecsCompatibility ? ecsPatternNames : legacyPatternNames;
String directory = ecsCompatibility ? "/patterns/ecs-v1/" : "/patterns/legacy/";
Map<String, String> builtinPatterns = new LinkedHashMap<>();
for (String pattern : patternNames) {
try {
try (InputStream is = Grok.class.getResourceAsStream(directory + pattern)) {
loadPatterns(builtinPatterns, is);
}
} catch (IOException e) {
throw new RuntimeException("failed to load built-in patterns", e);
}
}
return Collections.unmodifiableMap(builtinPatterns);
}
private static void loadPatterns(Map<String, String> patternBank, InputStream inputStream) throws IOException {
String line;
BufferedReader br = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8));
while ((line = br.readLine()) != null) {
String trimmedLine = line.replaceAll("^\\s+", "");
if (trimmedLine.startsWith("#") || trimmedLine.length() == 0) {
continue;
}
String[] parts = trimmedLine.split("\\s+", 2);
if (parts.length == 2) {
patternBank.put(parts[0], parts[1]);
}
}
}
}

151
libs/grok/src/main/java/org/elasticsearch/grok/GrokBuiltinPatterns.java Normal file
View File

@ -0,0 +1,151 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
package org.elasticsearch.grok;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
public class GrokBuiltinPatterns {
public static final String[] ECS_COMPATIBILITY_MODES = { "disabled", "v1" };
/**
* Patterns built in to the grok library.
*/
private static Map<String, String> LEGACY_PATTERNS;
private static Map<String, String> ECS_V1_PATTERNS;
public static synchronized Map<String, String> legacyPatterns() {
return get(false);
}
public static synchronized Map<String, String> ecsV1Patterns() {
return get(true);
}
/**
* Load built-in patterns.
*/
public static synchronized Map<String, String> get(boolean ecsCompatibility) {
if (ecsCompatibility) {
if (ECS_V1_PATTERNS == null) {
ECS_V1_PATTERNS = loadEcsPatterns();
}
return ECS_V1_PATTERNS;
} else {
if (LEGACY_PATTERNS == null) {
LEGACY_PATTERNS = loadLegacyPatterns();
}
return LEGACY_PATTERNS;
}
}
public static Map<String, String> get(String ecsCompatibility) {
if (isValidEcsCompatibilityMode(ecsCompatibility)) {
return get(ECS_COMPATIBILITY_MODES[1].equals(ecsCompatibility));
} else {
throw new IllegalArgumentException("unsupported ECS compatibility mode [" + ecsCompatibility + "]");
}
}
public static boolean isValidEcsCompatibilityMode(String ecsCompatibility) {
return Arrays.asList(ECS_COMPATIBILITY_MODES).contains(ecsCompatibility);
}
private static Map<String, String> loadLegacyPatterns() {
var patternNames = List.of(
"aws",
"bacula",
"bind",
"bro",
"exim",
"firewalls",
"grok-patterns",
"haproxy",
"httpd",
"java",
"junos",
"linux-syslog",
"maven",
"mcollective-patterns",
"mongodb",
"nagios",
"postgresql",
"rails",
"redis",
"ruby",
"squid"
);
return loadPatternsFromDirectory(patternNames, "/patterns/legacy/");
}
private static Map<String, String> loadEcsPatterns() {
var patternNames = List.of(
"aws",
"bacula",
"bind",
"bro",
"exim",
"firewalls",
"grok-patterns",
"haproxy",
"httpd",
"java",
"junos",
"linux-syslog",
"maven",
"mcollective",
"mongodb",
"nagios",
"postgresql",
"rails",
"redis",
"ruby",
"squid",
"zeek"
);
return loadPatternsFromDirectory(patternNames, "/patterns/ecs-v1/");
}
private static Map<String, String> loadPatternsFromDirectory(List<String> patternNames, String directory) {
Map<String, String> builtinPatterns = new LinkedHashMap<>();
for (String pattern : patternNames) {
try {
try (InputStream is = GrokBuiltinPatterns.class.getResourceAsStream(directory + pattern)) {
loadPatternsFromFile(builtinPatterns, is);
}
} catch (IOException e) {
throw new RuntimeException("failed to load built-in patterns", e);
}
}
return Collections.unmodifiableMap(builtinPatterns);
}
private static void loadPatternsFromFile(Map<String, String> patternBank, InputStream inputStream) throws IOException {
String line;
BufferedReader br = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8));
while ((line = br.readLine()) != null) {
String trimmedLine = line.replaceAll("^\\s+", "");
if (trimmedLine.startsWith("#") || trimmedLine.length() == 0) {
continue;
}
String[] parts = trimmedLine.split("\\s+", 2);
if (parts.length == 2) {
patternBank.put(parts[0], parts[1]);
}
}
}
}

40
libs/grok/src/test/java/org/elasticsearch/grok/GrokTests.java
View File

@ -49,7 +49,7 @@ public class GrokTests extends ESTestCase {
}
private void testMatchWithoutCaptures(boolean ecsCompatibility) {
Grok grok = new Grok(Grok.getBuiltinPatterns(ecsCompatibility), "value", logger::warn);
Grok grok = new Grok(GrokBuiltinPatterns.get(ecsCompatibility), "value", logger::warn);
assertThat(grok.captures("value"), equalTo(Map.of()));
assertThat(grok.captures("prefix_value"), equalTo(Map.of()));
assertThat(grok.captures("no_match"), nullValue());
@ -61,7 +61,7 @@ public class GrokTests extends ESTestCase {
}
private void testCapturesBytes(boolean ecsCompatibility) {
Grok grok = new Grok(Grok.getBuiltinPatterns(ecsCompatibility), "%{NUMBER:n:int}", logger::warn);
Grok grok = new Grok(GrokBuiltinPatterns.get(ecsCompatibility), "%{NUMBER:n:int}", logger::warn);
byte[] utf8 = "10".getBytes(StandardCharsets.UTF_8);
assertThat(captureBytes(grok, utf8, 0, utf8.length), equalTo(Map.of("n", 10)));
assertThat(captureBytes(grok, utf8, 0, 1), equalTo(Map.of("n", 1)));
@ -130,7 +130,7 @@ public class GrokTests extends ESTestCase {
List<String> acceptedDuplicates
) {
String line = "Mar 16 00:01:25 evita postfix/smtpd[1713]: connect from camomile.cloud9.net[168.100.1.3]";
Grok grok = new Grok(Grok.getBuiltinPatterns(ecsCompatibility), "%{SYSLOGLINE}", logger::warn);
Grok grok = new Grok(GrokBuiltinPatterns.get(ecsCompatibility), "%{SYSLOGLINE}", logger::warn);
Map<String, GrokCaptureType> captureTypes = new HashMap<>();
captureTypes.put(facility.v1().getKey(), facility.v1().getValue());
@ -213,7 +213,7 @@ public class GrokTests extends ESTestCase {
) {
String line = "<191>1 2009-06-30T18:30:00+02:00 paxton.local grokdebug 4123 - [id1 foo=\\\"bar\\\"][id2 baz=\\\"something\\\"] "
+ "Hello, syslog.";
Grok grok = new Grok(Grok.getBuiltinPatterns(ecsCompatibility), "%{SYSLOG5424LINE}", logger::warn);
Grok grok = new Grok(GrokBuiltinPatterns.get(ecsCompatibility), "%{SYSLOG5424LINE}", logger::warn);
assertCaptureConfig(
grok,
Map.ofEntries(app.v1(), host.v1(), msg.v1(), msgid.v1(), pri.v1(), proc.v1(), sd.v1(), ts.v1(), ver.v1())
@ -237,7 +237,7 @@ public class GrokTests extends ESTestCase {
private void testDatePattern(boolean ecsCompatibility) {
String line = "fancy 12-12-12 12:12:12";
Grok grok = new Grok(Grok.getBuiltinPatterns(ecsCompatibility), "(?<timestamp>%{DATE_EU} %{TIME})", logger::warn);
Grok grok = new Grok(GrokBuiltinPatterns.get(ecsCompatibility), "(?<timestamp>%{DATE_EU} %{TIME})", logger::warn);
assertCaptureConfig(grok, Map.of("timestamp", STRING));
Map<String, Object> matches = grok.captures(line);
assertEquals("12-12-12 12:12:12", matches.get("timestamp"));
@ -249,7 +249,7 @@ public class GrokTests extends ESTestCase {
}
private void testNilCoercedValues(boolean ecsCompatibility) {
Grok grok = new Grok(Grok.getBuiltinPatterns(ecsCompatibility), "test (N/A|%{BASE10NUM:duration:float}ms)", logger::warn);
Grok grok = new Grok(GrokBuiltinPatterns.get(ecsCompatibility), "test (N/A|%{BASE10NUM:duration:float}ms)", logger::warn);
assertCaptureConfig(grok, Map.of("duration", FLOAT));
Map<String, Object> matches = grok.captures("test 28.4ms");
assertEquals(28.4f, matches.get("duration"));
@ -263,7 +263,7 @@ public class GrokTests extends ESTestCase {
}
private void testNilWithNoCoercion(boolean ecsCompatibility) {
Grok grok = new Grok(Grok.getBuiltinPatterns(ecsCompatibility), "test (N/A|%{BASE10NUM:duration}ms)", logger::warn);
Grok grok = new Grok(GrokBuiltinPatterns.get(ecsCompatibility), "test (N/A|%{BASE10NUM:duration}ms)", logger::warn);
assertCaptureConfig(grok, Map.of("duration", STRING));
Map<String, Object> matches = grok.captures("test 28.4ms");
assertEquals("28.4", matches.get("duration"));
@ -278,7 +278,7 @@ public class GrokTests extends ESTestCase {
private void testUnicodeSyslog(boolean ecsCompatibility) {
Grok grok = new Grok(
Grok.getBuiltinPatterns(ecsCompatibility),
GrokBuiltinPatterns.get(ecsCompatibility),
"<%{POSINT:syslog_pri}>%{SPACE}%{SYSLOGTIMESTAMP:syslog_timestamp} "
+ "%{SYSLOGHOST:syslog_hostname} %{PROG:syslog_program}(:?)(?:\\[%{GREEDYDATA:syslog_pid}\\])?(:?) "
+ "%{GREEDYDATA:syslog_message}",
@ -311,7 +311,7 @@ public class GrokTests extends ESTestCase {
}
private void testNamedFieldsWithWholeTextMatch(boolean ecsCompatibility) {
Grok grok = new Grok(Grok.getBuiltinPatterns(ecsCompatibility), "%{DATE_EU:stimestamp}", logger::warn);
Grok grok = new Grok(GrokBuiltinPatterns.get(ecsCompatibility), "%{DATE_EU:stimestamp}", logger::warn);
assertCaptureConfig(grok, Map.of("stimestamp", STRING));
Map<String, Object> matches = grok.captures("11/01/01");
assertThat(matches.get("stimestamp"), equalTo("11/01/01"));
@ -323,7 +323,7 @@ public class GrokTests extends ESTestCase {
}
private void testWithOniguramaNamedCaptures(boolean ecsCompatibility) {
Grok grok = new Grok(Grok.getBuiltinPatterns(ecsCompatibility), "(?<foo>\\w+)", logger::warn);
Grok grok = new Grok(GrokBuiltinPatterns.get(ecsCompatibility), "(?<foo>\\w+)", logger::warn);
assertCaptureConfig(grok, Map.of("foo", STRING));
Map<String, Object> matches = grok.captures("hello world");
assertThat(matches.get("foo"), equalTo("hello"));
@ -335,7 +335,7 @@ public class GrokTests extends ESTestCase {
}
private void testISO8601(boolean ecsCompatibility) {
Grok grok = new Grok(Grok.getBuiltinPatterns(ecsCompatibility), "^%{TIMESTAMP_ISO8601}$", logger::warn);
Grok grok = new Grok(GrokBuiltinPatterns.get(ecsCompatibility), "^%{TIMESTAMP_ISO8601}$", logger::warn);
assertCaptureConfig(grok, Map.of());
List<String> timeMessages = Arrays.asList(
"2001-01-01T00:00:00",
@ -365,7 +365,7 @@ public class GrokTests extends ESTestCase {
}
private void testNotISO8601(boolean ecsCompatibility, List<String> additionalCases) {
Grok grok = new Grok(Grok.getBuiltinPatterns(ecsCompatibility), "^%{TIMESTAMP_ISO8601}$", logger::warn);
Grok grok = new Grok(GrokBuiltinPatterns.get(ecsCompatibility), "^%{TIMESTAMP_ISO8601}$", logger::warn);
assertCaptureConfig(grok, Map.of());
List<String> timeMessages = Arrays.asList(
"2001-13-01T00:00:00", // invalid month
@ -498,7 +498,7 @@ public class GrokTests extends ESTestCase {
private void testBooleanCaptures(boolean ecsCompatibility) {
String pattern = "%{WORD:name}=%{WORD:status:boolean}";
Grok g = new Grok(Grok.getBuiltinPatterns(ecsCompatibility), pattern, logger::warn);
Grok g = new Grok(GrokBuiltinPatterns.get(ecsCompatibility), pattern, logger::warn);
assertCaptureConfig(g, Map.of("name", STRING, "status", BOOLEAN));
String text = "active=true";
@ -658,7 +658,7 @@ public class GrokTests extends ESTestCase {
31.184.238.164 - - [24/Jul/2014:05:35:37 +0530] "GET /logs/access.log HTTP/1.0" 200 69849 "http://8rursodiol.enjin.com" \
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.12785 YaBrowser/13.12.1599.12785 \
Safari/537.36" "www.dlwindianrailways.com\"""";
Grok grok = new Grok(Grok.getBuiltinPatterns(ecsCompatibility), "%{COMBINEDAPACHELOG}", logger::warn);
Grok grok = new Grok(GrokBuiltinPatterns.get(ecsCompatibility), "%{COMBINEDAPACHELOG}", logger::warn);
Map<String, GrokCaptureType> captureTypes = new HashMap<>();
captureTypes.put(agent.v1().getKey(), agent.v1().getValue());
@ -838,7 +838,7 @@ public class GrokTests extends ESTestCase {
t.start();
};
Grok grok = new Grok(
Grok.getBuiltinPatterns(ecsCompatibility),
GrokBuiltinPatterns.get(ecsCompatibility),
grokPattern,
MatcherWatchdog.newInstance(10, 200, System::currentTimeMillis, scheduler),
logger::warn
@ -887,7 +887,7 @@ public class GrokTests extends ESTestCase {
}
private void testUnsupportedBracketsInFieldName(boolean ecsCompatibility) {
Grok grok = new Grok(Grok.getBuiltinPatterns(ecsCompatibility), "%{WORD:unsuppo(r)ted}", logger::warn);
Grok grok = new Grok(GrokBuiltinPatterns.get(ecsCompatibility), "%{WORD:unsuppo(r)ted}", logger::warn);
Map<String, Object> matches = grok.captures("line");
assertNull(matches);
}
@ -898,7 +898,7 @@ public class GrokTests extends ESTestCase {
}
private void testJavaClassPatternWithUnderscore(boolean ecsCompatibility) {
Grok grok = new Grok(Grok.getBuiltinPatterns(ecsCompatibility), "%{JAVACLASS}", logger::warn);
Grok grok = new Grok(GrokBuiltinPatterns.get(ecsCompatibility), "%{JAVACLASS}", logger::warn);
assertThat(grok.match("Test_Class.class"), is(true));
}
@ -908,7 +908,7 @@ public class GrokTests extends ESTestCase {
}
private void testJavaFilePatternWithSpaces(boolean ecsCompatibility) {
Grok grok = new Grok(Grok.getBuiltinPatterns(ecsCompatibility), "%{JAVAFILE}", logger::warn);
Grok grok = new Grok(GrokBuiltinPatterns.get(ecsCompatibility), "%{JAVAFILE}", logger::warn);
assertThat(grok.match("Test Class.java"), is(true));
}
@ -919,7 +919,7 @@ public class GrokTests extends ESTestCase {
private void testLogCallBack(boolean ecsCompatibility) {
AtomicReference<String> message = new AtomicReference<>();
Grok grok = new Grok(Grok.getBuiltinPatterns(ecsCompatibility), ".*\\[.*%{SPACE}*\\].*", message::set);
Grok grok = new Grok(GrokBuiltinPatterns.get(ecsCompatibility), ".*\\[.*%{SPACE}*\\].*", message::set);
grok.match("[foo]");
// this message comes from Joni, so updates to Joni may change the expectation
assertThat(message.get(), containsString("regular expression has redundant nested repeat operator"));
@ -929,7 +929,7 @@ public class GrokTests extends ESTestCase {
String line = "foo";
// test both with and without ECS compatibility
for (boolean ecsCompatibility : new boolean[] { false, true }) {
Grok grok = new Grok(Grok.getBuiltinPatterns(ecsCompatibility), "%{WORD:" + fieldName + "}", logger::warn);
Grok grok = new Grok(GrokBuiltinPatterns.get(ecsCompatibility), "%{WORD:" + fieldName + "}", logger::warn);
Map<String, Object> matches = grok.captures(line);
assertEquals(line, matches.get(fieldName));
}

7
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java
View File

@ -11,6 +11,7 @@ package org.elasticsearch.ingest.common;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.grok.Grok;
import org.elasticsearch.grok.GrokBuiltinPatterns;
import org.elasticsearch.grok.MatcherWatchdog;
import org.elasticsearch.ingest.AbstractProcessor;
import org.elasticsearch.ingest.ConfigurationUtils;
@ -26,7 +27,7 @@ import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationExcept
public final class GrokProcessor extends AbstractProcessor {
public static final String TYPE = "grok";
public static final String DEFAULT_ECS_COMPATIBILITY_MODE = Grok.ECS_COMPATIBILITY_MODES[0];
public static final String DEFAULT_ECS_COMPATIBILITY_MODE = GrokBuiltinPatterns.ECS_COMPATIBILITY_MODES[0];
private static final String PATTERN_MATCH_KEY = "_ingest._grok_match_index";
private static final Logger logger = LogManager.getLogger(GrokProcessor.class);
@ -159,7 +160,7 @@ public final class GrokProcessor extends AbstractProcessor {
"ecs_compatibility",
DEFAULT_ECS_COMPATIBILITY_MODE
);
if (Grok.isValidEcsCompatibilityMode(ecsCompatibility) == false) {
if (GrokBuiltinPatterns.isValidEcsCompatibilityMode(ecsCompatibility) == false) {
throw newConfigurationException(TYPE, processorTag, "ecs_compatibility", "unsupported mode '" + ecsCompatibility + "'");
}
@ -167,7 +168,7 @@ public final class GrokProcessor extends AbstractProcessor {
throw newConfigurationException(TYPE, processorTag, "patterns", "List of patterns must not be empty");
}
Map<String, String> customPatternBank = ConfigurationUtils.readOptionalMap(TYPE, processorTag, config, "pattern_definitions");
Map<String, String> patternBank = new HashMap<>(Grok.getBuiltinPatterns(ecsCompatibility));
Map<String, String> patternBank = new HashMap<>(GrokBuiltinPatterns.get(ecsCompatibility));
if (customPatternBank != null) {
patternBank.putAll(customPatternBank);
}

8
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessorGetAction.java
View File

@ -19,7 +19,7 @@ import org.elasticsearch.client.internal.node.NodeClient;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.grok.Grok;
import org.elasticsearch.grok.GrokBuiltinPatterns;
import org.elasticsearch.rest.BaseRestHandler;
import org.elasticsearch.rest.RestRequest;
import org.elasticsearch.rest.Scope;
@ -127,7 +127,7 @@ public class GrokProcessorGetAction extends ActionType<GrokProcessorGetAction.Re
@Inject
public TransportAction(TransportService transportService, ActionFilters actionFilters) {
this(transportService, actionFilters, Grok.getBuiltinPatterns(false), Grok.getBuiltinPatterns(true));
this(transportService, actionFilters, GrokBuiltinPatterns.legacyPatterns(), GrokBuiltinPatterns.ecsV1Patterns());
}
// visible for testing
@ -149,7 +149,7 @@ public class GrokProcessorGetAction extends ActionType<GrokProcessorGetAction.Re
ActionListener.completeWith(
listener,
() -> new Response(
request.getEcsCompatibility().equals(Grok.ECS_COMPATIBILITY_MODES[0])
request.getEcsCompatibility().equals(GrokBuiltinPatterns.ECS_COMPATIBILITY_MODES[0])
? request.sorted() ? sortedLegacyGrokPatterns : legacyGrokPatterns
: request.sorted() ? sortedEcsV1GrokPatterns
: ecsV1GrokPatterns
@ -175,7 +175,7 @@ public class GrokProcessorGetAction extends ActionType<GrokProcessorGetAction.Re
protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) {
boolean sorted = request.paramAsBoolean("s", false);
String ecsCompatibility = request.param("ecs_compatibility", GrokProcessor.DEFAULT_ECS_COMPATIBILITY_MODE);
if (Grok.isValidEcsCompatibilityMode(ecsCompatibility) == false) {
if (GrokBuiltinPatterns.isValidEcsCompatibilityMode(ecsCompatibility) == false) {
throw new IllegalArgumentException("unsupported ECS compatibility mode [" + ecsCompatibility + "]");
}
Request grokPatternsRequest = new Request(sorted, ecsCompatibility);

3
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RedactProcessor.java
View File

@ -12,6 +12,7 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.ElasticsearchTimeoutException;
import org.elasticsearch.grok.Grok;
import org.elasticsearch.grok.GrokBuiltinPatterns;
import org.elasticsearch.grok.GrokCaptureExtracter;
import org.elasticsearch.grok.MatcherWatchdog;
import org.elasticsearch.ingest.AbstractProcessor;
@ -350,7 +351,7 @@ public class RedactProcessor extends AbstractProcessor {
throw newConfigurationException(TYPE, processorTag, "patterns", "List of patterns must not be empty");
}
Map<String, String> customPatternBank = ConfigurationUtils.readOptionalMap(TYPE, processorTag, config, "pattern_definitions");
Map<String, String> patternBank = new HashMap<>(Grok.getBuiltinPatterns(true));
Map<String, String> patternBank = new HashMap<>(GrokBuiltinPatterns.ecsV1Patterns());
if (customPatternBank != null) {
patternBank.putAll(customPatternBank);
}

24
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorGetActionTests.java
View File

@ -14,7 +14,7 @@ import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.grok.Grok;
import org.elasticsearch.grok.GrokBuiltinPatterns;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.transport.TransportService;
import org.elasticsearch.xcontent.ToXContent;
@ -115,17 +115,21 @@ public class GrokProcessorGetActionTests extends ESTestCase {
ECS_TEST_PATTERNS
);
GrokProcessorGetAction.Response[] receivedResponse = new GrokProcessorGetAction.Response[1];
transportAction.doExecute(null, new GrokProcessorGetAction.Request(true, Grok.ECS_COMPATIBILITY_MODES[1]), new ActionListener<>() {
@Override
public void onResponse(GrokProcessorGetAction.Response response) {
receivedResponse[0] = response;
}
transportAction.doExecute(
null,
new GrokProcessorGetAction.Request(true, GrokBuiltinPatterns.ECS_COMPATIBILITY_MODES[1]),
new ActionListener<>() {
@Override
public void onResponse(GrokProcessorGetAction.Response response) {
receivedResponse[0] = response;
}
@Override
public void onFailure(Exception e) {
fail();
@Override
public void onFailure(Exception e) {
fail();
}
}
});
);
assertThat(receivedResponse[0], notNullValue());
assertThat(receivedResponse[0].getGrokPatterns().keySet().toArray(), equalTo(sortedKeys.toArray()));
}

5
modules/runtime-fields-common/src/main/java/org/elasticsearch/runtimefields/NamedGroupExtractor.java
View File

@ -13,6 +13,7 @@ import org.elasticsearch.common.util.LazyInitializable;
import org.elasticsearch.core.TimeValue;
import org.elasticsearch.dissect.DissectParser;
import org.elasticsearch.grok.Grok;
import org.elasticsearch.grok.GrokBuiltinPatterns;
import org.elasticsearch.grok.MatcherWatchdog;
import org.elasticsearch.threadpool.ThreadPool;
@ -99,13 +100,13 @@ public interface NamedGroupExtractor {
try {
// Try to collect warnings up front and refuse to compile the expression if there are any
List<String> warnings = new ArrayList<>();
new Grok(Grok.getBuiltinPatterns(false), pattern, watchdog, warnings::add).match("__nomatch__");
new Grok(GrokBuiltinPatterns.legacyPatterns(), pattern, watchdog, warnings::add).match("__nomatch__");
if (false == warnings.isEmpty()) {
throw new IllegalArgumentException("emitted warnings: " + warnings);
}
return new Grok(
Grok.getBuiltinPatterns(false),
GrokBuiltinPatterns.legacyPatterns(),
pattern,
watchdog,
w -> { throw new IllegalArgumentException("grok [" + pattern + "] emitted a warning: " + w); }

10
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/textstructure/action/FindStructureAction.java
View File

@ -17,7 +17,7 @@ import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.xcontent.StatusToXContentObject;
import org.elasticsearch.core.TimeValue;
import org.elasticsearch.grok.Grok;
import org.elasticsearch.grok.GrokBuiltinPatterns;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.xcontent.ParseField;
import org.elasticsearch.xcontent.XContentBuilder;
@ -32,8 +32,8 @@ import java.util.Objects;
import static org.elasticsearch.action.ValidateActions.addValidationError;
public class FindStructureAction extends ActionType<FindStructureAction.Response> {
public static final String ECS_COMPATIBILITY_DISABLED = Grok.ECS_COMPATIBILITY_MODES[0];
public static final String ECS_COMPATIBILITY_V1 = Grok.ECS_COMPATIBILITY_MODES[1];
public static final String ECS_COMPATIBILITY_DISABLED = GrokBuiltinPatterns.ECS_COMPATIBILITY_MODES[0];
public static final String ECS_COMPATIBILITY_V1 = GrokBuiltinPatterns.ECS_COMPATIBILITY_MODES[1];
public static final FindStructureAction INSTANCE = new FindStructureAction();
public static final String NAME = "cluster:monitor/text_structure/findstructure";
@ -359,12 +359,12 @@ public class FindStructureAction extends ActionType<FindStructureAction.Response
}
}
if (ecsCompatibility != null && Grok.isValidEcsCompatibilityMode(ecsCompatibility) == false) {
if (ecsCompatibility != null && GrokBuiltinPatterns.isValidEcsCompatibilityMode(ecsCompatibility) == false) {
validationException = addValidationError(
"["
+ ECS_COMPATIBILITY.getPreferredName()
+ "] must be one of ["
+ String.join(", ", Grok.ECS_COMPATIBILITY_MODES)
+ String.join(", ", GrokBuiltinPatterns.ECS_COMPATIBILITY_MODES)
+ "] if specified",
validationException
);

8
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/textstructure/structurefinder/TextStructure.java
View File

@ -10,7 +10,7 @@ import org.elasticsearch.TransportVersion;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.grok.Grok;
import org.elasticsearch.grok.GrokBuiltinPatterns;
import org.elasticsearch.xcontent.ObjectParser;
import org.elasticsearch.xcontent.ParseField;
import org.elasticsearch.xcontent.ToXContentObject;
@ -131,7 +131,7 @@ public class TextStructure implements ToXContentObject, Writeable {
}
private static String getNonNullEcsCompatibilityString(String ecsCompatibility) {
return (ecsCompatibility == null || ecsCompatibility.isEmpty()) ? Grok.ECS_COMPATIBILITY_MODES[0] : ecsCompatibility;
return (ecsCompatibility == null || ecsCompatibility.isEmpty()) ? GrokBuiltinPatterns.ECS_COMPATIBILITY_MODES[0] : ecsCompatibility;
}
private final int numLinesAnalyzed;
@ -755,11 +755,11 @@ public class TextStructure implements ToXContentObject, Writeable {
}
if (ecsCompatibility != null
&& ecsCompatibility.isEmpty() == false
&& Grok.isValidEcsCompatibilityMode(ecsCompatibility) == false) {
&& GrokBuiltinPatterns.isValidEcsCompatibilityMode(ecsCompatibility) == false) {
throw new IllegalArgumentException(
ECS_COMPATIBILITY.getPreferredName()
+ "] must be one of ["
+ String.join(", ", Grok.ECS_COMPATIBILITY_MODES)
+ String.join(", ", GrokBuiltinPatterns.ECS_COMPATIBILITY_MODES)
+ "] if specified"
);
}

3
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/categorization/GrokPatternCreator.java
View File

@ -9,6 +9,7 @@ package org.elasticsearch.xpack.ml.job.categorization;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.grok.Grok;
import org.elasticsearch.grok.GrokBuiltinPatterns;
import java.util.ArrayList;
import java.util.Arrays;
@ -286,7 +287,7 @@ public final class GrokPatternCreator {
this.grokPatternName = grokPatternName;
this.fieldName = fieldName;
this.grok = new Grok(
Grok.getBuiltinPatterns(ECS_COMPATIBILITY),
GrokBuiltinPatterns.get(ECS_COMPATIBILITY),
"%{DATA:" + PREFACE + "}" + preBreak + "%{" + grokPatternName + ":this}" + postBreak + "%{GREEDYDATA:" + EPILOGUE + "}",
logger::warn
);

19
x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/structurefinder/GrokPatternCreator.java
View File

@ -10,6 +10,7 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.core.Tuple;
import org.elasticsearch.grok.Grok;
import org.elasticsearch.grok.GrokBuiltinPatterns;
import org.elasticsearch.xpack.core.textstructure.structurefinder.FieldStats;
import java.util.ArrayList;
@ -233,9 +234,9 @@ public final class GrokPatternCreator {
this.mappings = mappings;
this.fieldStats = fieldStats;
if (customGrokPatternDefinitions.isEmpty()) {
grokPatternDefinitions = Grok.getBuiltinPatterns(ecsCompatibility);
grokPatternDefinitions = GrokBuiltinPatterns.get(ecsCompatibility);
} else {
grokPatternDefinitions = new HashMap<>(Grok.getBuiltinPatterns(ecsCompatibility));
grokPatternDefinitions = new HashMap<>(GrokBuiltinPatterns.get(ecsCompatibility));
grokPatternDefinitions.putAll(customGrokPatternDefinitions);
}
this.timeoutChecker = Objects.requireNonNull(timeoutChecker);
@ -640,7 +641,7 @@ public final class GrokPatternCreator {
fieldName,
"\\b",
"\\b",
Grok.getBuiltinPatterns(ecsCompatibility)
GrokBuiltinPatterns.get(ecsCompatibility)
);
}
@ -688,7 +689,7 @@ public final class GrokPatternCreator {
fieldName,
preBreak,
postBreak,
Grok.getBuiltinPatterns(ecsCompatibility)
GrokBuiltinPatterns.get(ecsCompatibility)
);
}
@ -840,7 +841,7 @@ public final class GrokPatternCreator {
throw new IllegalStateException("Cannot process KV matches until a field name has been determined");
}
Grok grok = new Grok(
Grok.getBuiltinPatterns(ecsCompatibility),
GrokBuiltinPatterns.get(ecsCompatibility),
"(?m)%{DATA:" + PREFACE + "}\\b" + fieldName + "=%{USER:" + VALUE + "}%{GREEDYDATA:" + EPILOGUE + "}",
TimeoutChecker.watchdog,
logger::warn
@ -929,7 +930,7 @@ public final class GrokPatternCreator {
return new FullMatchGrokPatternCandidate(
"%{" + grokPatternName + "}",
timeField,
Grok.getBuiltinPatterns(ECS_COMPATIBILITY_DISABLED)
GrokBuiltinPatterns.get(ECS_COMPATIBILITY_DISABLED)
);
}
@ -937,7 +938,7 @@ public final class GrokPatternCreator {
return new FullMatchGrokPatternCandidate(
"%{" + grokPatternName + "}",
timeField,
Grok.getBuiltinPatterns(ECS_COMPATIBILITY_ENABLED)
GrokBuiltinPatterns.get(ECS_COMPATIBILITY_ENABLED)
);
}
@ -950,11 +951,11 @@ public final class GrokPatternCreator {
}
static FullMatchGrokPatternCandidate fromGrokPatternLegacy(String grokPattern, String timeField) {
return new FullMatchGrokPatternCandidate(grokPattern, timeField, Grok.getBuiltinPatterns(ECS_COMPATIBILITY_DISABLED));
return new FullMatchGrokPatternCandidate(grokPattern, timeField, GrokBuiltinPatterns.get(ECS_COMPATIBILITY_DISABLED));
}
static FullMatchGrokPatternCandidate fromGrokPatternEcs(String grokPattern, String timeField) {
return new FullMatchGrokPatternCandidate(grokPattern, timeField, Grok.getBuiltinPatterns(ECS_COMPATIBILITY_ENABLED));
return new FullMatchGrokPatternCandidate(grokPattern, timeField, GrokBuiltinPatterns.get(ECS_COMPATIBILITY_ENABLED));
}
static FullMatchGrokPatternCandidate fromGrokPattern(

8
x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/structurefinder/LogTextStructureFinder.java
View File

@ -7,7 +7,7 @@
package org.elasticsearch.xpack.textstructure.structurefinder;
import org.elasticsearch.core.Tuple;
import org.elasticsearch.grok.Grok;
import org.elasticsearch.grok.GrokBuiltinPatterns;
import org.elasticsearch.xpack.core.textstructure.action.FindStructureAction;
import org.elasticsearch.xpack.core.textstructure.structurefinder.FieldStats;
import org.elasticsearch.xpack.core.textstructure.structurefinder.TextStructure;
@ -69,7 +69,7 @@ public class LogTextStructureFinder implements TextStructureFinder {
fieldStats,
customGrokPatternDefinitions,
timeoutChecker,
Grok.ECS_COMPATIBILITY_MODES[1].equals(overrides.getEcsCompatibility())
GrokBuiltinPatterns.ECS_COMPATIBILITY_MODES[1].equals(overrides.getEcsCompatibility())
);
String grokPattern = overrides.getGrokPattern();
@ -237,7 +237,7 @@ public class LogTextStructureFinder implements TextStructureFinder {
fieldStats,
customGrokPatternDefinitions,
timeoutChecker,
Grok.ECS_COMPATIBILITY_MODES[1].equals(overrides.getEcsCompatibility())
GrokBuiltinPatterns.ECS_COMPATIBILITY_MODES[1].equals(overrides.getEcsCompatibility())
);
// We can't parse directly into @timestamp using Grok, so parse to some other time field, which the date filter will then remove
@ -360,7 +360,7 @@ public class LogTextStructureFinder implements TextStructureFinder {
false,
false,
timeoutChecker,
Grok.ECS_COMPATIBILITY_MODES[1].equals(overrides.getEcsCompatibility())
GrokBuiltinPatterns.ECS_COMPATIBILITY_MODES[1].equals(overrides.getEcsCompatibility())
);
for (String sampleLine : sampleLines) {

11
x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/structurefinder/TextStructureUtils.java
View File

@ -10,6 +10,7 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.core.Tuple;
import org.elasticsearch.grok.Grok;
import org.elasticsearch.grok.GrokBuiltinPatterns;
import org.elasticsearch.ingest.Pipeline;
import org.elasticsearch.xpack.core.textstructure.structurefinder.FieldStats;
@ -61,19 +62,19 @@ public final class TextStructureUtils {
"(?:%{WKT_POINT}|%{WKT_LINESTRING}|%{WKT_MULTIPOINT}|%{WKT_POLYGON}|%{WKT_MULTILINESTRING}|%{WKT_MULTIPOLYGON}|%{WKT_BBOX})"
);
patterns.put("WKT_GEOMETRYCOLLECTION", "GEOMETRYCOLLECTION \\(%{WKT_ANY}(?:, %{WKT_ANY})\\)");
patterns.putAll(Grok.getBuiltinPatterns(false));
patterns.putAll(GrokBuiltinPatterns.legacyPatterns());
EXTENDED_PATTERNS = Collections.unmodifiableMap(patterns);
}
private static final int NUM_TOP_HITS = 10;
// NUMBER Grok pattern doesn't support scientific notation, so we extend it
private static final Grok NUMBER_GROK = new Grok(
Grok.getBuiltinPatterns(false),
GrokBuiltinPatterns.legacyPatterns(),
"^%{NUMBER}(?:[eE][+-]?[0-3]?[0-9]{1,2})?$",
TimeoutChecker.watchdog,
logger::warn
);
private static final Grok IP_GROK = new Grok(Grok.getBuiltinPatterns(false), "^%{IP}$", TimeoutChecker.watchdog, logger::warn);
private static final Grok IP_GROK = new Grok(GrokBuiltinPatterns.legacyPatterns(), "^%{IP}$", TimeoutChecker.watchdog, logger::warn);
private static final Grok GEO_POINT_WKT = new Grok(EXTENDED_PATTERNS, "^%{WKT_POINT}$", TimeoutChecker.watchdog, logger::warn);
private static final Grok GEO_WKT = new Grok(
EXTENDED_PATTERNS,
@ -228,7 +229,7 @@ public final class TextStructureUtils {
true,
true,
timeoutChecker,
Grok.ECS_COMPATIBILITY_MODES[1].equals(overrides.getEcsCompatibility())
GrokBuiltinPatterns.ECS_COMPATIBILITY_MODES[1].equals(overrides.getEcsCompatibility())
);
try {
timestampFormatFinder.addSample(value.toString());
@ -642,7 +643,7 @@ public final class TextStructureUtils {
}
grokProcessorSettings.put(
"ecs_compatibility",
(ecsCompatibility == null || ecsCompatibility.isEmpty()) ? Grok.ECS_COMPATIBILITY_MODES[0] : ecsCompatibility
(ecsCompatibility == null || ecsCompatibility.isEmpty()) ? GrokBuiltinPatterns.ECS_COMPATIBILITY_MODES[0] : ecsCompatibility
);
processors.add(Collections.singletonMap("grok", grokProcessorSettings));
} else {

5
x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/structurefinder/TimestampFormatFinder.java
View File

@ -12,6 +12,7 @@ import org.elasticsearch.core.Nullable;
import org.elasticsearch.core.SuppressForbidden;
import org.elasticsearch.core.Tuple;
import org.elasticsearch.grok.Grok;
import org.elasticsearch.grok.GrokBuiltinPatterns;
import java.time.DateTimeException;
import java.time.Instant;
@ -1802,13 +1803,13 @@ public final class TimestampFormatFinder {
this.strictGrokPattern = Objects.requireNonNull(strictGrokPattern);
// The (?m) here has the Ruby meaning, which is equivalent to (?s) in Java
this.strictSearchGrok = new Grok(
Grok.getBuiltinPatterns(false),
GrokBuiltinPatterns.legacyPatterns(),
"(?m)%{DATA:" + PREFACE + "}" + strictGrokPattern + "%{GREEDYDATA:" + EPILOGUE + "}",
TimeoutChecker.watchdog,
logger::warn
);
this.strictFullMatchGrok = new Grok(
Grok.getBuiltinPatterns(false),
GrokBuiltinPatterns.legacyPatterns(),
"^" + strictGrokPattern + "$",
TimeoutChecker.watchdog,
logger::warn

8
x-pack/plugin/text-structure/src/test/java/org/elasticsearch/xpack/textstructure/structurefinder/TimeoutCheckerTests.java
View File

@ -9,6 +9,7 @@ package org.elasticsearch.xpack.textstructure.structurefinder;
import org.elasticsearch.ElasticsearchTimeoutException;
import org.elasticsearch.core.TimeValue;
import org.elasticsearch.grok.Grok;
import org.elasticsearch.grok.GrokBuiltinPatterns;
import org.elasticsearch.threadpool.Scheduler;
import org.joni.Matcher;
import org.junit.After;
@ -81,7 +82,12 @@ public class TimeoutCheckerTests extends TextStructureTestCase {
}
public void testGrokCaptures() throws Exception {
Grok grok = new Grok(Grok.getBuiltinPatterns(false), "{%DATA:data}{%GREEDYDATA:greedydata}", TimeoutChecker.watchdog, logger::warn);
Grok grok = new Grok(
GrokBuiltinPatterns.legacyPatterns(),
"{%DATA:data}{%GREEDYDATA:greedydata}",
TimeoutChecker.watchdog,
logger::warn
);
TimeValue timeout = TimeValue.timeValueMillis(1);
try (TimeoutChecker timeoutChecker = new TimeoutChecker("grok captures test", timeout, scheduler)) {