mirrors
/
elasticsearch
mirror of https://mirror.osredm.com/root/elasticsearch.git
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

ECS support for Grok processor (#76885)

This commit is contained in:
Dan Hermann 2021-08-31 06:40:52 -05:00 committed by GitHub
parent 2d0773cd3c
commit 90d2899323
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
57 changed files with 1267 additions and 167 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
libs/grok/src/main/java/org/elasticsearch/grok/Grok.java
View File

@ -22,6 +22,7 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
import java.util.LinkedHashMap;
@ -31,10 +32,14 @@ import java.util.Map;
import java.util.function.Consumer;
public final class Grok {
public static final String[] ECS_COMPATIBILITY_MODES = {"disabled", "v1"};
/**
* Patterns built in to the grok library.
*/
public static final Map<String, String> BUILTIN_PATTERNS = loadBuiltinPatterns();
private static Map<String, String> LEGACY_PATTERNS;
private static Map<String, String> ECS_V1_PATTERNS;
private static final String NAME_GROUP = "name";
private static final String SUBNAME_GROUP = "subname";
@ -296,16 +301,51 @@ public final class Grok {
/**
* Load built-in patterns.
*/
private static Map<String, String> loadBuiltinPatterns() {
String[] patternNames = new String[] {
public static synchronized Map<String, String> getBuiltinPatterns(boolean ecsCompatibility) {
if (ecsCompatibility) {
if (ECS_V1_PATTERNS == null) {
ECS_V1_PATTERNS = loadPatterns(ecsCompatibility);
}
return ECS_V1_PATTERNS;
} else {
if (LEGACY_PATTERNS == null) {
LEGACY_PATTERNS = loadPatterns(ecsCompatibility);
}
return LEGACY_PATTERNS;
}
}
public static Map<String, String> getBuiltinPatterns(String ecsCompatibility) {
if (isValidEcsCompatibilityMode(ecsCompatibility)) {
return getBuiltinPatterns(ECS_COMPATIBILITY_MODES[1].equals(ecsCompatibility));
} else {
throw new IllegalArgumentException("unsupported ECS compatibility mode [" + ecsCompatibility + "]");
}
}
public static boolean isValidEcsCompatibilityMode(String ecsCompatibility) {
return Arrays.asList(ECS_COMPATIBILITY_MODES).contains(ecsCompatibility);
}
private static Map<String, String> loadPatterns(boolean ecsCompatibility) {
String[] legacyPatternNames = {
"aws", "bacula", "bind", "bro", "exim", "firewalls", "grok-patterns", "haproxy",
"httpd", "java", "junos", "linux-syslog", "maven", "mcollective-patterns", "mongodb", "nagios",
"postgresql", "rails", "redis", "ruby", "squid"
};
String[] ecsPatternNames = {
"aws", "bacula", "bind", "bro", "exim", "firewalls", "grok-patterns", "haproxy",
"httpd", "java", "junos", "linux-syslog", "maven", "mcollective", "mongodb", "nagios",
"postgresql", "rails", "redis", "ruby", "squid", "zeek"
};
String[] patternNames = ecsCompatibility ? ecsPatternNames : legacyPatternNames;
String directory = ecsCompatibility ? "/patterns/ecs-v1/" : "/patterns/legacy/";
Map<String, String> builtinPatterns = new LinkedHashMap<>();
for (String pattern : patternNames) {
try {
try(InputStream is = Grok.class.getResourceAsStream("/patterns/" + pattern)) {
try (InputStream is = Grok.class.getResourceAsStream(directory + pattern)) {
loadPatterns(builtinPatterns, is);
}
} catch (IOException e) {

28
libs/grok/src/main/resources/patterns/ecs-v1/aws Normal file
View File

@ -0,0 +1,28 @@
S3_REQUEST_LINE (?:%{WORD:http.request.method} %{NOTSPACE:url.original}(?: HTTP/%{NUMBER:http.version})?)
S3_ACCESS_LOG %{WORD:aws.s3access.bucket_owner} %{NOTSPACE:aws.s3access.bucket} \%{HTTPDATE:timestamp}\ (?:-|%{IP:client.ip}) (?:-|%{NOTSPACE:client.user.id}) %{NOTSPACE:aws.s3access.request_id} %{NOTSPACE:aws.s3access.operation} (?:-|%{NOTSPACE:aws.s3access.key}) (?:-|"%{S3_REQUEST_LINE:aws.s3access.request_uri}") (?:-|%{INT:http.response.status_code:int}) (?:-|%{NOTSPACE:aws.s3access.error_code}) (?:-|%{INT:aws.s3access.bytes_sent:long}) (?:-|%{INT:aws.s3access.object_size:long}) (?:-|%{INT:aws.s3access.total_time:int}) (?:-|%{INT:aws.s3access.turn_around_time:int}) "(?:-|%{DATA:http.request.referrer})" "(?:-|%{DATA:user_agent.original})" (?:-|%{NOTSPACE:aws.s3access.version_id})(?: (?:-|%{NOTSPACE:aws.s3access.host_id}) (?:-|%{NOTSPACE:aws.s3access.signature_version}) (?:-|%{NOTSPACE:tls.cipher}) (?:-|%{NOTSPACE:aws.s3access.authentication_type}) (?:-|%{NOTSPACE:aws.s3access.host_header}) (?:-|%{NOTSPACE:aws.s3access.tls_version}))?
# :long - %{INT:aws.s3access.bytes_sent:int}
# :long - %{INT:aws.s3access.object_size:int}
ELB_URIHOST %{IPORHOST:url.domain}(?::%{POSINT:url.port:int})?
ELB_URIPATHQUERY %{URIPATH:url.path}(?:\?%{URIQUERY:url.query})?
# deprecated - old name:
ELB_URIPATHPARAM %{ELB_URIPATHQUERY}
ELB_URI %{URIPROTO:url.scheme}://(?:%{USER:url.username}(?::^@*)?@)?(?:%{ELB_URIHOST})?(?:%{ELB_URIPATHQUERY})?
ELB_REQUEST_LINE (?:%{WORD:http.request.method} %{ELB_URI:url.original}(?: HTTP/%{NUMBER:http.version})?)
# pattern supports 'regular' HTTP ELB format
ELB_V1_HTTP_LOG %{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE:aws.elb.name} %{IP:source.ip}:%{INT:source.port:int} (?:-|(?:%{IP:aws.elb.backend.ip}:%{INT:aws.elb.backend.port:int})) (?:-1|%{NUMBER:aws.elb.request_processing_time.sec:float}) (?:-1|%{NUMBER:aws.elb.backend_processing_time.sec:float}) (?:-1|%{NUMBER:aws.elb.response_processing_time.sec:float}) %{INT:http.response.status_code:int} (?:-|%{INT:aws.elb.backend.http.response.status_code:int}) %{INT:http.request.body.bytes:long} %{INT:http.response.body.bytes:long} "%{ELB_REQUEST_LINE}"(?: "(?:-|%{DATA:user_agent.original})" (?:-|%{NOTSPACE:tls.cipher}) (?:-|%{NOTSPACE:aws.elb.ssl_protocol}))?
# :long - %{INT:http.request.body.bytes:int}
# :long - %{INT:http.response.body.bytes:int}
ELB_ACCESS_LOG %{ELB_V1_HTTP_LOG}
# pattern used to match a shorted format, that's why we have the optional part (starting with *http.version*) at the end
CLOUDFRONT_ACCESS_LOG (?<timestamp>%{YEAR}-%{MONTHNUM}-%{MONTHDAY}\t%{TIME})\t%{WORD:aws.cloudfront.x_edge_location}\t(?:-|%{INT:destination.bytes:long})\t%{IPORHOST:source.ip}\t%{WORD:http.request.method}\t%{HOSTNAME:url.domain}\t%{NOTSPACE:url.path}\t(?:(?:000)|%{INT:http.response.status_code:int})\t(?:-|%{DATA:http.request.referrer})\t%{DATA:user_agent.original}\t(?:-|%{DATA:url.query})\t(?:-|%{DATA:aws.cloudfront.http.request.cookie})\t%{WORD:aws.cloudfront.x_edge_result_type}\t%{NOTSPACE:aws.cloudfront.x_edge_request_id}\t%{HOSTNAME:aws.cloudfront.http.request.host}\t%{URIPROTO:network.protocol}\t(?:-|%{INT:source.bytes:long})\t%{NUMBER:aws.cloudfront.time_taken:float}\t(?:-|%{IP:network.forwarded_ip})\t(?:-|%{DATA:aws.cloudfront.ssl_protocol})\t(?:-|%{NOTSPACE:tls.cipher})\t%{WORD:aws.cloudfront.x_edge_response_result_type}(?:\t(?:-|HTTP/%{NUMBER:http.version})\t(?:-|%{DATA:aws.cloudfront.fle_status})\t(?:-|%{DATA:aws.cloudfront.fle_encrypted_fields})\t%{INT:source.port:int}\t%{NUMBER:aws.cloudfront.time_to_first_byte:float}\t(?:-|%{DATA:aws.cloudfront.x_edge_detailed_result_type})\t(?:-|%{NOTSPACE:http.request.mime_type})\t(?:-|%{INT:aws.cloudfront.http.request.size:long})\t(?:-|%{INT:aws.cloudfront.http.request.range.start:long})\t(?:-|%{INT:aws.cloudfront.http.request.range.end:long}))?
# :long - %{INT:destination.bytes:int}
# :long - %{INT:source.bytes:int}
# :long - %{INT:aws.cloudfront.http.request.size:int}
# :long - %{INT:aws.cloudfront.http.request.range.start:int}
# :long - %{INT:aws.cloudfront.http.request.range.end:int}

53
libs/grok/src/main/resources/patterns/ecs-v1/bacula Normal file
View File

@ -0,0 +1,53 @@
BACULA_TIMESTAMP %{MONTHDAY}-%{MONTH}(?:-%{YEAR})? %{HOUR}:%{MINUTE}
BACULA_HOST %{HOSTNAME}
BACULA_VOLUME %{USER}
BACULA_DEVICE %{USER}
BACULA_DEVICEPATH %{UNIXPATH}
BACULA_CAPACITY %{INT}{1,3}(,%{INT}{3})*
BACULA_VERSION %{USER}
BACULA_JOB %{USER}
BACULA_LOG_MAX_CAPACITY User defined maximum volume capacity %{BACULA_CAPACITY:bacula.volume.max_capacity} exceeded on device \"%{BACULA_DEVICE:bacula.volume.device}\" \(%{BACULA_DEVICEPATH:bacula.volume.path}\).?
BACULA_LOG_END_VOLUME End of medium on Volume \"%{BACULA_VOLUME:bacula.volume.name}\" Bytes=%{BACULA_CAPACITY:bacula.volume.bytes} Blocks=%{BACULA_CAPACITY:bacula.volume.blocks} at %{BACULA_TIMESTAMP:bacula.timestamp}.
BACULA_LOG_NEW_VOLUME Created new Volume \"%{BACULA_VOLUME:bacula.volume.name}\" in catalog.
BACULA_LOG_NEW_LABEL Labeled new Volume \"%{BACULA_VOLUME:bacula.volume.name}\" on (?:file )?device \"%{BACULA_DEVICE:bacula.volume.device}\" \(%{BACULA_DEVICEPATH:bacula.volume.path}\).
BACULA_LOG_WROTE_LABEL Wrote label to prelabeled Volume \"%{BACULA_VOLUME:bacula.volume.name}\" on device \"%{BACULA_DEVICE:bacula.volume.device}\" \(%{BACULA_DEVICEPATH:bacula.volume.path}\)
BACULA_LOG_NEW_MOUNT New volume \"%{BACULA_VOLUME:bacula.volume.name}\" mounted on device \"%{BACULA_DEVICE:bacula.volume.device}\" \(%{BACULA_DEVICEPATH:bacula.volume.path}\) at %{BACULA_TIMESTAMP:bacula.timestamp}.
BACULA_LOG_NOOPEN \s*Cannot open %{DATA}: ERR=%{GREEDYDATA:error.message}
BACULA_LOG_NOOPENDIR \s*Could not open directory \"?%{DATA:file.path}\"?: ERR=%{GREEDYDATA:error.message}
BACULA_LOG_NOSTAT \s*Could not stat %{DATA:file.path}: ERR=%{GREEDYDATA:error.message}
BACULA_LOG_NOJOBS There are no more Jobs associated with Volume \"%{BACULA_VOLUME:bacula.volume.name}\". Marking it purged.
BACULA_LOG_ALL_RECORDS_PRUNED .*?All records pruned from Volume \"%{BACULA_VOLUME:bacula.volume.name}\"; marking it \"Purged\"
BACULA_LOG_BEGIN_PRUNE_JOBS Begin pruning Jobs older than %{INT} month %{INT} days .
BACULA_LOG_BEGIN_PRUNE_FILES Begin pruning Files.
BACULA_LOG_PRUNED_JOBS Pruned %{INT} Jobs* for client %{BACULA_HOST:bacula.client.name} from catalog.
BACULA_LOG_PRUNED_FILES Pruned Files from %{INT} Jobs* for client %{BACULA_HOST:bacula.client.name} from catalog.
BACULA_LOG_ENDPRUNE End auto prune.
BACULA_LOG_STARTJOB Start Backup JobId %{INT}, Job=%{BACULA_JOB:bacula.job.name}
BACULA_LOG_STARTRESTORE Start Restore Job %{BACULA_JOB:bacula.job.name}
BACULA_LOG_USEDEVICE Using Device \"%{BACULA_DEVICE:bacula.volume.device}\"
BACULA_LOG_DIFF_FS \s*%{UNIXPATH} is a different filesystem. Will not descend from %{UNIXPATH} into it.
BACULA_LOG_JOBEND Job write elapsed time = %{DATA:bacula.job.elapsed_time}, Transfer rate = %{NUMBER} (K|M|G)? Bytes/second
BACULA_LOG_NOPRUNE_JOBS No Jobs found to prune.
BACULA_LOG_NOPRUNE_FILES No Files found to prune.
BACULA_LOG_VOLUME_PREVWRITTEN Volume \"?%{BACULA_VOLUME:bacula.volume.name}\"? previously written, moving to end of data.
BACULA_LOG_READYAPPEND Ready to append to end of Volume \"%{BACULA_VOLUME:bacula.volume.name}\" size=%{INT:bacula.volume.size:long}
# :long - %{INT:bacula.volume.size:int}
BACULA_LOG_CANCELLING Cancelling duplicate JobId=%{INT:bacula.job.other_id}.
BACULA_LOG_MARKCANCEL JobId %{INT:bacula.job.id}, Job %{BACULA_JOB:bacula.job.name} marked to be canceled.
BACULA_LOG_CLIENT_RBJ shell command: run ClientRunBeforeJob \"%{GREEDYDATA:bacula.job.client_run_before_command}\"
BACULA_LOG_VSS (Generate )?VSS (Writer)?
BACULA_LOG_MAXSTART Fatal [eE]rror: Job canceled because max start delay time exceeded.
BACULA_LOG_DUPLICATE Fatal [eE]rror: JobId %{INT:bacula.job.other_id} already running. Duplicate job not allowed.
BACULA_LOG_NOJOBSTAT Fatal [eE]rror: No Job status returned from FD.
BACULA_LOG_FATAL_CONN Fatal [eE]rror: bsock.c:133 Unable to connect to (Client: %{BACULA_HOST:bacula.client.name}|Storage daemon) on %{IPORHOST:client.address}:%{POSINT:client.port:int}. ERR=%{GREEDYDATA:error.message}
BACULA_LOG_NO_CONNECT Warning: bsock.c:127 Could not connect to (Client: %{BACULA_HOST:bacula.client.name}|Storage daemon) on %{IPORHOST:client.address}:%{POSINT:client.port:int}. ERR=%{GREEDYDATA:error.message}
BACULA_LOG_NO_AUTH Fatal error: Unable to authenticate with File daemon at \"?%{IPORHOST:client.address}(?::%{POSINT:client.port:int})?\"?. Possible causes:
BACULA_LOG_NOSUIT No prior or suitable Full backup found in catalog. Doing FULL backup.
BACULA_LOG_NOPRIOR No prior Full backup Job record found.
BACULA_LOG_JOB (Error: )?Bacula %{BACULA_HOST} %{BACULA_VERSION} \(%{BACULA_VERSION}\):
BACULA_LOG %{BACULA_TIMESTAMP:timestamp} %{BACULA_HOST:host.hostname}(?: JobId %{INT:bacula.job.id})?:? (%{BACULA_LOG_MAX_CAPACITY}|%{BACULA_LOG_END_VOLUME}|%{BACULA_LOG_NEW_VOLUME}|%{BACULA_LOG_NEW_LABEL}|%{BACULA_LOG_WROTE_LABEL}|%{BACULA_LOG_NEW_MOUNT}|%{BACULA_LOG_NOOPEN}|%{BACULA_LOG_NOOPENDIR}|%{BACULA_LOG_NOSTAT}|%{BACULA_LOG_NOJOBS}|%{BACULA_LOG_ALL_RECORDS_PRUNED}|%{BACULA_LOG_BEGIN_PRUNE_JOBS}|%{BACULA_LOG_BEGIN_PRUNE_FILES}|%{BACULA_LOG_PRUNED_JOBS}|%{BACULA_LOG_PRUNED_FILES}|%{BACULA_LOG_ENDPRUNE}|%{BACULA_LOG_STARTJOB}|%{BACULA_LOG_STARTRESTORE}|%{BACULA_LOG_USEDEVICE}|%{BACULA_LOG_DIFF_FS}|%{BACULA_LOG_JOBEND}|%{BACULA_LOG_NOPRUNE_JOBS}|%{BACULA_LOG_NOPRUNE_FILES}|%{BACULA_LOG_VOLUME_PREVWRITTEN}|%{BACULA_LOG_READYAPPEND}|%{BACULA_LOG_CANCELLING}|%{BACULA_LOG_MARKCANCEL}|%{BACULA_LOG_CLIENT_RBJ}|%{BACULA_LOG_VSS}|%{BACULA_LOG_MAXSTART}|%{BACULA_LOG_DUPLICATE}|%{BACULA_LOG_NOJOBSTAT}|%{BACULA_LOG_FATAL_CONN}|%{BACULA_LOG_NO_CONNECT}|%{BACULA_LOG_NO_AUTH}|%{BACULA_LOG_NOSUIT}|%{BACULA_LOG_JOB}|%{BACULA_LOG_NOPRIOR})
# old (deprecated) name :
BACULA_LOGLINE %{BACULA_LOG}

13
libs/grok/src/main/resources/patterns/ecs-v1/bind Normal file
View File

@ -0,0 +1,13 @@
BIND9_TIMESTAMP %{MONTHDAY}[-]%{MONTH}[-]%{YEAR} %{TIME}
BIND9_DNSTYPE (?:A|AAAA|CAA|CDNSKEY|CDS|CERT|CNAME|CSYNC|DLV|DNAME|DNSKEY|DS|HINFO|LOC|MX|NAPTR|NS|NSEC|NSEC3|OPENPGPKEY|PTR|RRSIG|RP|SIG|SMIMEA|SOA|SRV|TSIG|TXT|URI)
BIND9_CATEGORY (?:queries)
# dns.question.class is static - only 'IN' is supported by Bind9
# bind.log.question.name is expected to be a 'duplicate' (same as the dns.question.name capture)
BIND9_QUERYLOGBASE client(:? @0x(?:[0-9A-Fa-f]+))? %{IP:client.ip}#%{POSINT:client.port:int} \(%{GREEDYDATA:bind.log.question.name}\): query: %{GREEDYDATA:dns.question.name} (?<dns.question.class>IN) %{BIND9_DNSTYPE:dns.question.type}(:? %{DATA:bind.log.question.flags})? \(%{IP:server.ip}\)
# for query-logging category and severity are always fixed as "queries: info: "
BIND9_QUERYLOG %{BIND9_TIMESTAMP:timestamp} %{BIND9_CATEGORY:bing.log.category}: %{LOGLEVEL:log.level}: %{BIND9_QUERYLOGBASE}
BIND9 %{BIND9_QUERYLOG}

30
libs/grok/src/main/resources/patterns/ecs-v1/bro Normal file
View File

@ -0,0 +1,30 @@
# supports the 'old' BRO log files, for updated Zeek log format see the patters/ecs-v1/zeek
# https://www.bro.org/sphinx/script-reference/log-files.html
BRO_BOOL [TF]
BRO_DATA [^\t]+
# http.log - old format (before the Zeek rename) :
BRO_HTTP %{NUMBER:timestamp}\t%{NOTSPACE:zeek.session_id}\t%{IP:source.ip}\t%{INT:source.port:int}\t%{IP:destination.ip}\t%{INT:destination.port:int}\t%{INT:zeek.http.trans_depth:int}\t(?:-|%{WORD:http.request.method})\t(?:-|%{BRO_DATA:url.domain})\t(?:-|%{BRO_DATA:url.original})\t(?:-|%{BRO_DATA:http.request.referrer})\t(?:-|%{BRO_DATA:user_agent.original})\t(?:-|%{NUMBER:http.request.body.bytes:long})\t(?:-|%{NUMBER:http.response.body.bytes:long})\t(?:-|%{POSINT:http.response.status_code:int})\t(?:-|%{DATA:zeek.http.status_msg})\t(?:-|%{POSINT:zeek.http.info_code:int})\t(?:-|%{DATA:zeek.http.info_msg})\t(?:-|%{BRO_DATA:zeek.http.filename})\t(?:\(empty\)|%{BRO_DATA:zeek.http.tags})\t(?:-|%{BRO_DATA:url.username})\t(?:-|%{BRO_DATA:url.password})\t(?:-|%{BRO_DATA:zeek.http.proxied})\t(?:-|%{BRO_DATA:zeek.http.orig_fuids})\t(?:-|%{BRO_DATA:http.request.mime_type})\t(?:-|%{BRO_DATA:zeek.http.resp_fuids})\t(?:-|%{BRO_DATA:http.response.mime_type})
# :long - %{NUMBER:http.request.body.bytes:int}
# :long - %{NUMBER:http.response.body.bytes:int}
# dns.log - old format
BRO_DNS %{NUMBER:timestamp}\t%{NOTSPACE:zeek.session_id}\t%{IP:source.ip}\t%{INT:source.port:int}\t%{IP:destination.ip}\t%{INT:destination.port:int}\t%{WORD:network.transport}\t(?:-|%{INT:dns.id:int})\t(?:-|%{BRO_DATA:dns.question.name})\t(?:-|%{INT:zeek.dns.qclass:int})\t(?:-|%{BRO_DATA:zeek.dns.qclass_name})\t(?:-|%{INT:zeek.dns.qtype:int})\t(?:-|%{BRO_DATA:dns.question.type})\t(?:-|%{INT:zeek.dns.rcode:int})\t(?:-|%{BRO_DATA:dns.response_code})\t(?:-|%{BRO_BOOL:zeek.dns.AA})\t(?:-|%{BRO_BOOL:zeek.dns.TC})\t(?:-|%{BRO_BOOL:zeek.dns.RD})\t(?:-|%{BRO_BOOL:zeek.dns.RA})\t(?:-|%{NONNEGINT:zeek.dns.Z:int})\t(?:-|%{BRO_DATA:zeek.dns.answers})\t(?:-|%{DATA:zeek.dns.TTLs})\t(?:-|%{BRO_BOOL:zeek.dns.rejected})
# conn.log - old bro, also supports 'newer' format (optional *zeek.connection.local_resp* flag) compared to non-ecs mode
BRO_CONN %{NUMBER:timestamp}\t%{NOTSPACE:zeek.session_id}\t%{IP:source.ip}\t%{INT:source.port:int}\t%{IP:destination.ip}\t%{INT:destination.port:int}\t%{WORD:network.transport}\t(?:-|%{BRO_DATA:network.protocol})\t(?:-|%{NUMBER:zeek.connection.duration:float})\t(?:-|%{INT:zeek.connection.orig_bytes:long})\t(?:-|%{INT:zeek.connection.resp_bytes:long})\t(?:-|%{BRO_DATA:zeek.connection.state})\t(?:-|%{BRO_BOOL:zeek.connection.local_orig})\t(?:(?:-|%{BRO_BOOL:zeek.connection.local_resp})\t)?(?:-|%{INT:zeek.connection.missed_bytes:long})\t(?:-|%{BRO_DATA:zeek.connection.history})\t(?:-|%{INT:source.packets:long})\t(?:-|%{INT:source.bytes:long})\t(?:-|%{INT:destination.packets:long})\t(?:-|%{INT:destination.bytes:long})\t(?:\(empty\)|%{BRO_DATA:zeek.connection.tunnel_parents})
# :long - %{INT:zeek.connection.orig_bytes:int}
# :long - %{INT:zeek.connection.resp_bytes:int}
# :long - %{INT:zeek.connection.missed_bytes:int}
# :long - %{INT:source.packets:int}
# :long - %{INT:source.bytes:int}
# :long - %{INT:destination.packets:int}
# :long - %{INT:destination.bytes:int}
# files.log - old format
BRO_FILES %{NUMBER:timestamp}\t%{NOTSPACE:zeek.files.fuid}\t(?:-|%{IP:server.ip})\t(?:-|%{IP:client.ip})\t(?:-|%{BRO_DATA:zeek.files.session_ids})\t(?:-|%{BRO_DATA:zeek.files.source})\t(?:-|%{INT:zeek.files.depth:int})\t(?:-|%{BRO_DATA:zeek.files.analyzers})\t(?:-|%{BRO_DATA:file.mime_type})\t(?:-|%{BRO_DATA:file.name})\t(?:-|%{NUMBER:zeek.files.duration:float})\t(?:-|%{BRO_DATA:zeek.files.local_orig})\t(?:-|%{BRO_BOOL:zeek.files.is_orig})\t(?:-|%{INT:zeek.files.seen_bytes:long})\t(?:-|%{INT:file.size:long})\t(?:-|%{INT:zeek.files.missing_bytes:long})\t(?:-|%{INT:zeek.files.overflow_bytes:long})\t(?:-|%{BRO_BOOL:zeek.files.timedout})\t(?:-|%{BRO_DATA:zeek.files.parent_fuid})\t(?:-|%{BRO_DATA:file.hash.md5})\t(?:-|%{BRO_DATA:file.hash.sha1})\t(?:-|%{BRO_DATA:file.hash.sha256})\t(?:-|%{BRO_DATA:zeek.files.extracted})
# :long - %{INT:zeek.files.seen_bytes:int}
# :long - %{INT:file.size:int}
# :long - %{INT:zeek.files.missing_bytes:int}
# :long - %{INT:zeek.files.overflow_bytes:int}

26
libs/grok/src/main/resources/patterns/ecs-v1/exim Normal file
View File

@ -0,0 +1,26 @@
EXIM_MSGID [0-9A-Za-z]{6}-[0-9A-Za-z]{6}-[0-9A-Za-z]{2}
# <= message arrival
# => normal message delivery
# -> additional address in same delivery
# *> delivery suppressed by -N
# ** delivery failed; address bounced
# == delivery deferred; temporary problem
EXIM_FLAGS (?:<=|=>|->|\*>|\*\*|==|<>|>>)
EXIM_DATE (:?%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME})
EXIM_PID \[%{POSINT:process.pid:int}\]
EXIM_QT ((\d+y)?(\d+w)?(\d+d)?(\d+h)?(\d+m)?(\d+s)?)
EXIM_EXCLUDE_TERMS (Message is frozen|(Start|End) queue run| Warning: | retry time not reached | no (IP address|host name) found for (IP address|host) | unexpected disconnection while reading SMTP command | no immediate delivery: |another process is handling this message)
EXIM_REMOTE_HOST (H=(%{NOTSPACE:source.address} )?(\(%{NOTSPACE:exim.log.remote_address}\) )?\%{IP:source.ip}\](?::%{POSINT:source.port:int})?)
EXIM_INTERFACE (I=\[%{IP:destination.ip}\](?::%{NUMBER:destination.port:int}))
EXIM_PROTOCOL (P=%{NOTSPACE:network.protocol})
EXIM_MSG_SIZE (S=%{NUMBER:exim.log.message.size:int})
EXIM_HEADER_ID (id=%{NOTSPACE:exim.log.header_id})
EXIM_QUOTED_CONTENT (?:\\.|[^\\"])*
EXIM_SUBJECT (T="%{EXIM_QUOTED_CONTENT:exim.log.message.subject}")
EXIM_UNKNOWN_FIELD (?:[A-Za-z0-9]{1,4}=(?:%{QUOTEDSTRING}|%{NOTSPACE}))
EXIM_NAMED_FIELDS (?: (?:%{EXIM_REMOTE_HOST}|%{EXIM_INTERFACE}|%{EXIM_PROTOCOL}|%{EXIM_MSG_SIZE}|%{EXIM_HEADER_ID}|%{EXIM_SUBJECT}|%{EXIM_UNKNOWN_FIELD}))*
EXIM_MESSAGE_ARRIVAL %{EXIM_DATE:timestamp} (?:%{EXIM_PID} )?%{EXIM_MSGID:exim.log.message.id} (?<exim.log.flags><=) (?<exim.log.status>[a-z:] )?%{EMAILADDRESS:exim.log.sender.email}%{EXIM_NAMED_FIELDS}(?:(?: from <?%{DATA:exim.log.sender.original}>?)? for %{EMAILADDRESS:exim.log.recipient.email})?
EXIM %{EXIM_MESSAGE_ARRIVAL}

111
libs/grok/src/main/resources/patterns/ecs-v1/firewalls Normal file
View File

@ -0,0 +1,111 @@
# NetScreen firewall logs
NETSCREENSESSIONLOG %{SYSLOGTIMESTAMP:timestamp} %{IPORHOST:observer.hostname} %{NOTSPACE:observer.name}: (?<observer.product>NetScreen) device_id=%{WORD:netscreen.device_id} .*?(system-\w+-%{NONNEGINT:event.code}\(%{WORD:netscreen.session.type}\))?: start_time="%{DATA:netscreen.session.start_time}" duration=%{INT:netscreen.session.duration:int} policy_id=%{INT:netscreen.policy_id} service=%{DATA:netscreen.service} proto=%{INT:netscreen.protocol_number:int} src zone=%{WORD:observer.ingress.zone} dst zone=%{WORD:observer.egress.zone} action=%{WORD:event.action} sent=%{INT:source.bytes:long} rcvd=%{INT:destination.bytes:long} src=%{IPORHOST:source.address} dst=%{IPORHOST:destination.address}(?: src_port=%{INT:source.port:int} dst_port=%{INT:destination.port:int})?(?: src-xlated ip=%{IP:source.nat.ip} port=%{INT:source.nat.port:int} dst-xlated ip=%{IP:destination.nat.ip} port=%{INT:destination.nat.port:int})?(?: session_id=%{INT:netscreen.session.id} reason=%{GREEDYDATA:netscreen.session.reason})?
# :long - %{INT:source.bytes:int}
# :long - %{INT:destination.bytes:int}
#== Cisco ASA ==
CISCO_TAGGED_SYSLOG ^<%{POSINT:log.syslog.priority:int}>%{CISCOTIMESTAMP:timestamp}( %{SYSLOGHOST:host.hostname})? ?: %%{CISCOTAG:cisco.asa.tag}:
CISCOTIMESTAMP %{MONTH} +%{MONTHDAY}(?: %{YEAR})? %{TIME}
CISCOTAG [A-Z0-9]+-%{INT}-(?:[A-Z0-9_]+)
# Common Particles
CISCO_ACTION Built|Teardown|Deny|Denied|denied|requested|permitted|denied by ACL|discarded|est-allowed|Dropping|created|deleted
CISCO_REASON Duplicate TCP SYN|Failed to locate egress interface|Invalid transport field|No matching connection|DNS Response|DNS Query|(?:%{WORD}\s*)*
CISCO_DIRECTION Inbound|inbound|Outbound|outbound
CISCO_INTERVAL first hit|%{INT}-second interval
CISCO_XLATE_TYPE static|dynamic
# helpers
CISCO_HITCOUNT_INTERVAL hit-cnt %{INT:cisco.asa.hit_count:int} (?:first hit|%{INT:cisco.asa.interval:int}-second interval)
CISCO_SRC_IP_USER %{NOTSPACE:observer.ingress.interface.name}:%{IP:source.ip}(?:\(%{DATA:source.user.name}\))?
CISCO_DST_IP_USER %{NOTSPACE:observer.egress.interface.name}:%{IP:destination.ip}(?:\(%{DATA:destination.user.name}\))?
CISCO_SRC_HOST_PORT_USER %{NOTSPACE:observer.ingress.interface.name}:(?:(?:%{IP:source.ip})|(?:%{HOSTNAME:source.address}))(?:/%{INT:source.port:int})?(?:\(%{DATA:source.user.name}\))?
CISCO_DST_HOST_PORT_USER %{NOTSPACE:observer.egress.interface.name}:(?:(?:%{IP:destination.ip})|(?:%{HOSTNAME:destination.address}))(?:/%{INT:destination.port:int})?(?:\(%{DATA:destination.user.name}\))?
# ASA-1-104001
CISCOFW104001 \((?:Primary|Secondary)\) Switching to ACTIVE - %{GREEDYDATA:event.reason}
# ASA-1-104002
CISCOFW104002 \((?:Primary|Secondary)\) Switching to STANDBY - %{GREEDYDATA:event.reason}
# ASA-1-104003
CISCOFW104003 \((?:Primary|Secondary)\) Switching to FAILED\.
# ASA-1-104004
CISCOFW104004 \((?:Primary|Secondary)\) Switching to OK\.
# ASA-1-105003
CISCOFW105003 \((?:Primary|Secondary)\) Monitoring on [Ii]nterface %{NOTSPACE:network.interface.name} waiting
# ASA-1-105004
CISCOFW105004 \((?:Primary|Secondary)\) Monitoring on [Ii]nterface %{NOTSPACE:network.interface.name} normal
# ASA-1-105005
CISCOFW105005 \((?:Primary|Secondary)\) Lost Failover communications with mate on [Ii]nterface %{NOTSPACE:network.interface.name}
# ASA-1-105008
CISCOFW105008 \((?:Primary|Secondary)\) Testing [Ii]nterface %{NOTSPACE:network.interface.name}
# ASA-1-105009
CISCOFW105009 \((?:Primary|Secondary)\) Testing on [Ii]nterface %{NOTSPACE:network.interface.name} (?:Passed|Failed)
# ASA-2-106001
CISCOFW106001 %{CISCO_DIRECTION:cisco.asa.network.direction} %{WORD:cisco.asa.network.transport} connection %{CISCO_ACTION:cisco.asa.outcome} from %{IP:source.ip}/%{INT:source.port:int} to %{IP:destination.ip}/%{INT:destination.port:int} flags %{DATA:cisco.asa.tcp_flags} on interface %{NOTSPACE:observer.egress.interface.name}
# ASA-2-106006, ASA-2-106007, ASA-2-106010
CISCOFW106006_106007_106010 %{CISCO_ACTION:cisco.asa.outcome} %{CISCO_DIRECTION:cisco.asa.network.direction} %{WORD:cisco.asa.network.transport} (?:from|src) %{IP:source.ip}/%{INT:source.port:int}(?:\(%{DATA:source.user.name}\))? (?:to|dst) %{IP:destination.ip}/%{INT:destination.port:int}(?:\(%{DATA:destination.user.name}\))? (?:(?:on interface %{NOTSPACE:observer.egress.interface.name})|(?:due to %{CISCO_REASON:event.reason}))
# ASA-3-106014
CISCOFW106014 %{CISCO_ACTION:cisco.asa.outcome} %{CISCO_DIRECTION:cisco.asa.network.direction} %{WORD:cisco.asa.network.transport} src %{CISCO_SRC_IP_USER} dst %{CISCO_DST_IP_USER}\s?\(type %{INT:cisco.asa.icmp_type:int}, code %{INT:cisco.asa.icmp_code:int}\)
# ASA-6-106015
CISCOFW106015 %{CISCO_ACTION:cisco.asa.outcome} %{WORD:cisco.asa.network.transport} \(%{DATA:cisco.asa.rule_name}\) from %{IP:source.ip}/%{INT:source.port:int} to %{IP:destination.ip}/%{INT:destination.port:int} flags %{DATA:cisco.asa.tcp_flags} on interface %{NOTSPACE:observer.egress.interface.name}
# ASA-1-106021
CISCOFW106021 %{CISCO_ACTION:cisco.asa.outcome} %{WORD:cisco.asa.network.transport} reverse path check from %{IP:source.ip} to %{IP:destination.ip} on interface %{NOTSPACE:observer.egress.interface.name}
# ASA-4-106023
CISCOFW106023 %{CISCO_ACTION:cisco.asa.outcome}(?: protocol)? %{WORD:cisco.asa.network.transport} src %{CISCO_SRC_HOST_PORT_USER} dst %{CISCO_DST_HOST_PORT_USER}( \(type %{INT:cisco.asa.icmp_type:int}, code %{INT:cisco.asa.icmp_code:int}\))? by access-group "?%{DATA:cisco.asa.rule_name}"? \%{DATA:[@metadata.cisco.asa.hashcode1}, %{DATA:@metadata.cisco.asa.hashcode2}\]
# ASA-4-106100, ASA-4-106102, ASA-4-106103
CISCOFW106100_2_3 access-list %{NOTSPACE:cisco.asa.rule_name} %{CISCO_ACTION:cisco.asa.outcome} %{WORD:cisco.asa.network.transport} for user '%{DATA:user.name}' %{DATA:observer.ingress.interface.name}/%{IP:source.ip}\(%{INT:source.port:int}\) -> %{DATA:observer.egress.interface.name}/%{IP:destination.ip}\(%{INT:destination.port:int}\) %{CISCO_HITCOUNT_INTERVAL} \%{DATA:[@metadata.cisco.asa.hashcode1}, %{DATA:@metadata.cisco.asa.hashcode2}\]
# ASA-5-106100
CISCOFW106100 access-list %{NOTSPACE:cisco.asa.rule_name} %{CISCO_ACTION:cisco.asa.outcome} %{WORD:cisco.asa.network.transport} %{DATA:observer.ingress.interface.name}/%{IP:source.ip}\(%{INT:source.port:int}\)(?:\(%{DATA:source.user.name}\))? -> %{DATA:observer.egress.interface.name}/%{IP:destination.ip}\(%{INT:destination.port:int}\)(?:\(%{DATA:source.user.name}\))? hit-cnt %{INT:cisco.asa.hit_count:int} %{CISCO_INTERVAL} \%{DATA:[@metadata.cisco.asa.hashcode1}, %{DATA:@metadata.cisco.asa.hashcode2}\]
# ASA-5-304001
CISCOFW304001 %{IP:source.ip}(?:\(%{DATA:source.user.name}\))? Accessed URL %{IP:destination.ip}:%{GREEDYDATA:url.original}
# ASA-6-110002
CISCOFW110002 %{CISCO_REASON:event.reason} for %{WORD:cisco.asa.network.transport} from %{DATA:observer.ingress.interface.name}:%{IP:source.ip}/%{INT:source.port:int} to %{IP:destination.ip}/%{INT:destination.port:int}
# ASA-6-302010
CISCOFW302010 %{INT:cisco.asa.connections.in_use:int} in use, %{INT:cisco.asa.connections.most_used:int} most used
# ASA-6-302013, ASA-6-302014, ASA-6-302015, ASA-6-302016
CISCOFW302013_302014_302015_302016 %{CISCO_ACTION:cisco.asa.outcome}(?: %{CISCO_DIRECTION:cisco.asa.network.direction})? %{WORD:cisco.asa.network.transport} connection %{INT:cisco.asa.connection_id} for %{NOTSPACE:observer.ingress.interface.name}:%{IP:source.ip}/%{INT:source.port:int}(?: \(%{IP:source.nat.ip}/%{INT:source.nat.port:int}\))?(?:\(%{DATA:source.user.name?}\))? to %{NOTSPACE:observer.egress.interface.name}:%{IP:destination.ip}/%{INT:destination.port:int}( \(%{IP:destination.nat.ip}/%{INT:destination.nat.port:int}\))?(?:\(%{DATA:destination.user.name}\))?( duration %{TIME:cisco.asa.duration} bytes %{INT:network.bytes:long})?(?: %{CISCO_REASON:event.reason})?(?: \(%{DATA:user.name}\))?
# :long - %{INT:network.bytes:int}
# ASA-6-302020, ASA-6-302021
CISCOFW302020_302021 %{CISCO_ACTION:cisco.asa.outcome}(?: %{CISCO_DIRECTION:cisco.asa.network.direction})? %{WORD:cisco.asa.network.transport} connection for faddr %{IP:destination.ip}/%{INT:cisco.asa.icmp_seq:int}(?:\(%{DATA:destination.user.name}\))? gaddr %{IP:source.nat.ip}/%{INT:cisco.asa.icmp_type:int} laddr %{IP:source.ip}/%{INT}(?: \(%{DATA:source.user.name}\))?
# ASA-6-305011
CISCOFW305011 %{CISCO_ACTION:cisco.asa.outcome} %{CISCO_XLATE_TYPE} %{WORD:cisco.asa.network.transport} translation from %{DATA:observer.ingress.interface.name}:%{IP:source.ip}(/%{INT:source.port:int})?(?:\(%{DATA:source.user.name}\))? to %{DATA:observer.egress.interface.name}:%{IP:destination.ip}/%{INT:destination.port:int}
# ASA-3-313001, ASA-3-313004, ASA-3-313008
CISCOFW313001_313004_313008 %{CISCO_ACTION:cisco.asa.outcome} %{WORD:cisco.asa.network.transport} type=%{INT:cisco.asa.icmp_type:int}, code=%{INT:cisco.asa.icmp_code:int} from %{IP:source.ip} on interface %{NOTSPACE:observer.egress.interface.name}(?: to %{IP:destination.ip})?
# ASA-4-313005
CISCOFW313005 %{CISCO_REASON:event.reason} for %{WORD:cisco.asa.network.transport} error message: %{WORD} src %{CISCO_SRC_IP_USER} dst %{CISCO_DST_IP_USER} \(type %{INT:cisco.asa.icmp_type:int}, code %{INT:cisco.asa.icmp_code:int}\) on %{NOTSPACE} interface\.\s+Original IP payload: %{WORD:cisco.asa.original_ip_payload.network.transport} src %{IP:cisco.asa.original_ip_payload.source.ip}/%{INT:cisco.asa.original_ip_payload.source.port:int}(?:\(%{DATA:cisco.asa.original_ip_payload.source.user.name}\))? dst %{IP:cisco.asa.original_ip_payload.destination.ip}/%{INT:cisco.asa.original_ip_payload.destination.port:int}(?:\(%{DATA:cisco.asa.original_ip_payload.destination.user.name}\))?
# ASA-5-321001
CISCOFW321001 Resource '%{DATA:cisco.asa.resource.name}' limit of %{POSINT:cisco.asa.resource.limit:int} reached for system
# ASA-4-402117
CISCOFW402117 %{WORD:cisco.asa.network.type}: Received a non-IPSec packet \(protocol=\s?%{WORD:cisco.asa.network.transport}\) from %{IP:source.ip} to %{IP:destination.ip}\.?
# ASA-4-402119
CISCOFW402119 %{WORD:cisco.asa.network.type}: Received an %{WORD:cisco.asa.ipsec.protocol} packet \(SPI=\s?%{DATA:cisco.asa.ipsec.spi}, sequence number=\s?%{DATA:cisco.asa.ipsec.seq_num}\) from %{IP:source.ip} \(user=\s?%{DATA:source.user.name}\) to %{IP:destination.ip} that failed anti-replay checking\.?
# ASA-4-419001
CISCOFW419001 %{CISCO_ACTION:cisco.asa.outcome} %{WORD:cisco.asa.network.transport} packet from %{NOTSPACE:observer.ingress.interface.name}:%{IP:source.ip}/%{INT:source.port:int} to %{NOTSPACE:observer.egress.interface.name}:%{IP:destination.ip}/%{INT:destination.port:int}, reason: %{GREEDYDATA:event.reason}
# ASA-4-419002
CISCOFW419002 %{CISCO_REASON:event.reason} from %{DATA:observer.ingress.interface.name}:%{IP:source.ip}/%{INT:source.port:int} to %{DATA:observer.egress.interface.name}:%{IP:destination.ip}/%{INT:destination.port:int} with different initial sequence number
# ASA-4-500004
CISCOFW500004 %{CISCO_REASON:event.reason} for protocol=%{WORD:cisco.asa.network.transport}, from %{IP:source.ip}/%{INT:source.port:int} to %{IP:destination.ip}/%{INT:destination.port:int}
# ASA-6-602303, ASA-6-602304
CISCOFW602303_602304 %{WORD:cisco.asa.network.type}: An %{CISCO_DIRECTION:cisco.asa.network.direction} %{DATA:cisco.asa.ipsec.tunnel_type} SA \(SPI=\s?%{DATA:cisco.asa.ipsec.spi}\) between %{IP:source.ip} and %{IP:destination.ip} \(user=\s?%{DATA:source.user.name}\) has been %{CISCO_ACTION:cisco.asa.outcome}
# ASA-7-710001, ASA-7-710002, ASA-7-710003, ASA-7-710005, ASA-7-710006
CISCOFW710001_710002_710003_710005_710006 %{WORD:cisco.asa.network.transport} (?:request|access) %{CISCO_ACTION:cisco.asa.outcome} from %{IP:source.ip}/%{INT:source.port:int} to %{DATA:observer.egress.interface.name}:%{IP:destination.ip}/%{INT:destination.port:int}
# ASA-6-713172
CISCOFW713172 Group = %{DATA:cisco.asa.source.group}, IP = %{IP:source.ip}, Automatic NAT Detection Status:\s+Remote end\s*%{DATA:@metadata.cisco.asa.remote_nat}\s*behind a NAT device\s+This\s+end\s*%{DATA:@metadata.cisco.asa.local_nat}\s*behind a NAT device
# ASA-4-733100
CISCOFW733100 \\s*%{DATA:[cisco.asa.burst.object}\s*\] drop %{DATA:cisco.asa.burst.id} exceeded. Current burst rate is %{INT:cisco.asa.burst.current_rate:int} per second, max configured rate is %{INT:cisco.asa.burst.configured_rate:int}; Current average rate is %{INT:cisco.asa.burst.avg_rate:int} per second, max configured rate is %{INT:cisco.asa.burst.configured_avg_rate:int}; Cumulative total count is %{INT:cisco.asa.burst.cumulative_count:int}
#== End Cisco ASA ==
IPTABLES_TCP_FLAGS (CWR |ECE |URG |ACK |PSH |RST |SYN |FIN )*
IPTABLES_TCP_PART (?:SEQ=%{INT:iptables.tcp.seq:int}\s+)?(?:ACK=%{INT:iptables.tcp.ack:int}\s+)?WINDOW=%{INT:iptables.tcp.window:int}\s+RES=0x%{BASE16NUM:iptables.tcp_reserved_bits}\s+%{IPTABLES_TCP_FLAGS:iptables.tcp.flags}
IPTABLES4_FRAG (?:(?<= )(?:CE|DF|MF))*
IPTABLES4_PART SRC=%{IPV4:source.ip}\s+DST=%{IPV4:destination.ip}\s+LEN=(?:%{INT:iptables.length:int})?\s+TOS=(?:0|0x%{BASE16NUM:iptables.tos})?\s+PREC=(?:0x%{BASE16NUM:iptables.precedence_bits})?\s+TTL=(?:%{INT:iptables.ttl:int})?\s+ID=(?:%{INT:iptables.id})?\s+(?:%{IPTABLES4_FRAG:iptables.fragment_flags})?(?:\s+FRAG: %{INT:iptables.fragment_offset:int})?
IPTABLES6_PART SRC=%{IPV6:source.ip}\s+DST=%{IPV6:destination.ip}\s+LEN=(?:%{INT:iptables.length:int})?\s+TC=(?:0|0x%{BASE16NUM:iptables.tos})?\s+HOPLIMIT=(?:%{INT:iptables.ttl:int})?\s+FLOWLBL=(?:%{INT:iptables.flow_label})?
IPTABLES IN=(?:%{NOTSPACE:observer.ingress.interface.name})?\s+OUT=(?:%{NOTSPACE:observer.egress.interface.name})?\s+(?:MAC=(?:%{COMMONMAC:destination.mac})?(?::%{COMMONMAC:source.mac})?(?::A-Fa-f0-9{2}:A-Fa-f0-9{2})?\s+)?(:?%{IPTABLES4_PART}|%{IPTABLES6_PART}).*?PROTO=(?:%{WORD:network.transport})?\s+SPT=(?:%{INT:source.port:int})?\s+DPT=(?:%{INT:destination.port:int})?\s+(?:%{IPTABLES_TCP_PART})?
# Shorewall firewall logs
SHOREWALL (?:%{SYSLOGTIMESTAMP:timestamp}) (?:%{WORD:observer.hostname}) .*Shorewall:(?:%{WORD:shorewall.firewall.type})?:(?:%{WORD:shorewall.firewall.action})?.*%{IPTABLES}
#== End Shorewall
#== SuSE Firewall 2 ==
SFW2_LOG_PREFIX SFW2\-INext\-%{NOTSPACE:suse.firewall.action}
SFW2 ((?:%{SYSLOGTIMESTAMP:timestamp})|(?:%{TIMESTAMP_ISO8601:timestamp}))\s*%{HOSTNAME:observer.hostname}.*?%{SFW2_LOG_PREFIX:suse.firewall.log_prefix}\s*%{IPTABLES}
#== End SuSE ==

95
libs/grok/src/main/resources/patterns/ecs-v1/grok-patterns Normal file
View File

@ -0,0 +1,95 @@
USERNAME [a-zA-Z0-9._-]+
USER %{USERNAME}
EMAILLOCALPART [a-zA-Z0-9!#$%&'*+\-/=?^_`{|}~]{1,64}(?:\.[a-zA-Z0-9!#$%&'*+\-/=?^_`{|}~]{1,62}){0,63}
EMAILADDRESS %{EMAILLOCALPART}@%{HOSTNAME}
INT (?:[+-]?(?:[0-9]+))
BASE10NUM (?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+)))
NUMBER (?:%{BASE10NUM})
BASE16NUM (?<![0-9A-Fa-f])(?:[+-]?(?:0x)?(?:[0-9A-Fa-f]+))
BASE16FLOAT \b(?<![0-9A-Fa-f.])(?:[+-]?(?:0x)?(?:(?:[0-9A-Fa-f]+(?:\.[0-9A-Fa-f]*)?)|(?:\.[0-9A-Fa-f]+)))\b
POSINT \b(?:[1-9][0-9]*)\b
NONNEGINT \b(?:[0-9]+)\b
WORD \b\w+\b
NOTSPACE \S+
SPACE \s*
DATA .*?
GREEDYDATA .*
QUOTEDSTRING (?>(?<!\\)(?>"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``))
UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}
# URN, allowing use of RFC 2141 section 2.3 reserved characters
URN urn:[0-9A-Za-z][0-9A-Za-z-]{0,31}:(?:%[0-9a-fA-F]{2}|[0-9A-Za-z()+,.:=@;$_!*'/?#-])+
# Networking
MAC (?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC})
CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4})
WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2})
COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2})
IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?
IPV4 (?<![0-9])(?:(?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5]))(?![0-9])
IP (?:%{IPV6}|%{IPV4})
HOSTNAME \b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b)
IPORHOST (?:%{IP}|%{HOSTNAME})
HOSTPORT %{IPORHOST}:%{POSINT}
# paths (only absolute paths are matched)
PATH (?:%{UNIXPATH}|%{WINPATH})
UNIXPATH (/[[[:alnum:]]_%!$@:.,+~-]*)+
TTY (?:/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+))
WINPATH (?>[A-Za-z]+:|\\)(?:\\[^\\?*]*)+
URIPROTO [A-Za-z]([A-Za-z0-9+\-.]+)+
URIHOST %{IPORHOST}(?::%{POSINT})?
# uripath comes loosely from RFC1738, but mostly from what Firefox doesn't turn into %XX
URIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%&_\-]*)+
URIQUERY [A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]<>]*
# deprecated (kept due compatibility):
URIPARAM \?%{URIQUERY}
URIPATHPARAM %{URIPATH}(?:\?%{URIQUERY})?
URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATH}(?:\?%{URIQUERY})?)?
# Months: January, Feb, 3, 03, 12, December
MONTH \b(?:[Jj]an(?:uary|uar)?|[Ff]eb(?:ruary|ruar)?|[Mm](?:a|ä)?r(?:ch|z)?|[Aa]pr(?:il)?|[Mm]a(?:y|i)?|[Jj]un(?:e|i)?|[Jj]ul(?:y|i)?|[Aa]ug(?:ust)?|[Ss]ep(?:tember)?|[Oo](?:c|k)?t(?:ober)?|[Nn]ov(?:ember)?|[Dd]e(?:c|z)(?:ember)?)\b
MONTHNUM (?:0?[1-9]|1[0-2])
MONTHNUM2 (?:0[1-9]|1[0-2])
MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])
# Days: Monday, Tue, Thu, etc...
DAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?)
# Years?
YEAR (?>\d\d){1,2}
HOUR (?:2[0123]|[01]?[0-9])
MINUTE (?:[0-5][0-9])
# '60' is a leap second in most time standards and thus is valid.
SECOND (?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)
TIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9])
# datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it)
DATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR}
DATE_EU %{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR}
ISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE}))
ISO8601_SECOND %{SECOND}
TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}?
DATE %{DATE_US}|%{DATE_EU}
DATESTAMP %{DATE}[- ]%{TIME}
TZ (?:[APMCE][SD]T|UTC)
DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ}
DATESTAMP_RFC2822 %{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE}
DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR}
DATESTAMP_EVENTLOG %{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND}
# Syslog Dates: Month Day HH:MM:SS
SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME}
PROG [\x21-\x5a\x5c\x5e-\x7e]+
SYSLOGPROG %{PROG:process.name}(?:\[%{POSINT:process.pid:int}\])?
SYSLOGHOST %{IPORHOST}
SYSLOGFACILITY <%{NONNEGINT:log.syslog.facility.code:int}.%{NONNEGINT:log.syslog.priority:int}>
HTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT}
# Shortcuts
QS %{QUOTEDSTRING}
# Log formats
SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:host.hostname} %{SYSLOGPROG}:
# Log Levels
LOGLEVEL ([Aa]lert|ALERT|[Tt]race|TRACE|[Dd]ebug|DEBUG|[Nn]otice|NOTICE|[Ii]nfo?(?:rmation)?|INFO?(?:RMATION)?|[Ww]arn?(?:ing)?|WARN?(?:ING)?|[Ee]rr?(?:or)?|ERR?(?:OR)?|[Cc]rit?(?:ical)?|CRIT?(?:ICAL)?|[Ff]atal|FATAL|[Ss]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)

40
libs/grok/src/main/resources/patterns/ecs-v1/haproxy Normal file
View File

@ -0,0 +1,40 @@
HAPROXYTIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9])
HAPROXYDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{HAPROXYTIME}.%{INT}
# Override these default patterns to parse out what is captured in your haproxy.cfg
HAPROXYCAPTUREDREQUESTHEADERS %{DATA:haproxy.http.request.captured_headers}
HAPROXYCAPTUREDRESPONSEHEADERS %{DATA:haproxy.http.response.captured_headers}
# Example:
# These haproxy config lines will add data to the logs that are captured
# by the patterns below. Place them in your custom patterns directory to
# override the defaults.
#
# capture request header Host len 40
# capture request header X-Forwarded-For len 50
# capture request header Accept-Language len 50
# capture request header Referer len 200
# capture request header User-Agent len 200
#
# capture response header Content-Type len 30
# capture response header Content-Encoding len 10
# capture response header Cache-Control len 200
# capture response header Last-Modified len 200
#
# HAPROXYCAPTUREDREQUESTHEADERS %{DATA:haproxy.http.request.host}\|%{DATA:haproxy.http.request.x_forwarded_for}\|%{DATA:haproxy.http.request.accept_language}\|%{DATA:http.request.referrer}\|%{DATA:user_agent.original}
# HAPROXYCAPTUREDRESPONSEHEADERS %{DATA:http.response.mime_type}\|%{DATA:haproxy.http.response.encoding}\|%{DATA:haproxy.http.response.cache_control}\|%{DATA:haproxy.http.response.last_modified}
HAPROXYURI (?:%{URIPROTO:url.scheme}://)?(?:%{USER:url.username}(?::[^@]*)?@)?(?:%{IPORHOST:url.domain}(?::%{POSINT:url.port:int})?)?(?:%{URIPATH:url.path}(?:\?%{URIQUERY:url.query})?)?
HAPROXYHTTPREQUESTLINE (?:<BADREQ>|(?:%{WORD:http.request.method} %{HAPROXYURI:url.original}(?: HTTP/%{NUMBER:http.version})?))
# parse a haproxy 'httplog' line
HAPROXYHTTPBASE %{IP:source.address}:%{INT:source.port:int} \[%{HAPROXYDATE:haproxy.request_date}\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/(?:<NOSRV>|%{NOTSPACE:haproxy.server_name}) (?:-1|%{INT:haproxy.http.request.time_wait_ms:int})/(?:-1|%{INT:haproxy.total_waiting_time_ms:int})/(?:-1|%{INT:haproxy.connection_wait_time_ms:int})/(?:-1|%{INT:haproxy.http.request.time_wait_without_data_ms:int})/%{NOTSPACE:haproxy.total_time_ms} %{INT:http.response.status_code:int} %{INT:source.bytes:long} (?:-|%{DATA:haproxy.http.request.captured_cookie}) (?:-|%{DATA:haproxy.http.response.captured_cookie}) %{NOTSPACE:haproxy.termination_state} %{INT:haproxy.connections.active:int}/%{INT:haproxy.connections.frontend:int}/%{INT:haproxy.connections.backend:int}/%{INT:haproxy.connections.server:int}/%{INT:haproxy.connections.retries:int} %{INT:haproxy.server_queue:int}/%{INT:haproxy.backend_queue:int}(?: \{%{HAPROXYCAPTUREDREQUESTHEADERS}\}(?: \{%{HAPROXYCAPTUREDRESPONSEHEADERS}\})?)?(?: "%{HAPROXYHTTPREQUESTLINE}"?)?
# :long - %{INT:source.bytes:int}
HAPROXYHTTP (?:%{SYSLOGTIMESTAMP:timestamp}|%{TIMESTAMP_ISO8601:timestamp}) %{IPORHOST:host.hostname} %{SYSLOGPROG}: %{HAPROXYHTTPBASE}
# parse a haproxy 'tcplog' line
HAPROXYTCP (?:%{SYSLOGTIMESTAMP:timestamp}|%{TIMESTAMP_ISO8601:timestamp}) %{IPORHOST:host.hostname} %{SYSLOGPROG}: %{IP:source.address}:%{INT:source.port:int} \[%{HAPROXYDATE:haproxy.request_date}\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/(?:<NOSRV>|%{NOTSPACE:haproxy.server_name}) (?:-1|%{INT:haproxy.total_waiting_time_ms:int})/(?:-1|%{INT:haproxy.connection_wait_time_ms:int})/%{NOTSPACE:haproxy.total_time_ms} %{INT:source.bytes:long} %{NOTSPACE:haproxy.termination_state} %{INT:haproxy.connections.active:int}/%{INT:haproxy.connections.frontend:int}/%{INT:haproxy.connections.backend:int}/%{INT:haproxy.connections.server:int}/%{INT:haproxy.connections.retries:int} %{INT:haproxy.server_queue:int}/%{INT:haproxy.backend_queue:int}
# :long - %{INT:source.bytes:int}

17
libs/grok/src/main/resources/patterns/ecs-v1/httpd Normal file
View File

@ -0,0 +1,17 @@
HTTPDUSER %{EMAILADDRESS}|%{USER}
HTTPDERROR_DATE %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}
# Log formats
HTTPD_COMMONLOG %{IPORHOST:source.address} (?:-|%{HTTPDUSER:apache.access.user.identity}) (?:-|%{HTTPDUSER:user.name}) \[%{HTTPDATE:timestamp}\] "(?:%{WORD:http.request.method} %{NOTSPACE:url.original}(?: HTTP/%{NUMBER:http.version})?|%{DATA})" (?:-|%{INT:http.response.status_code:int}) (?:-|%{INT:http.response.body.bytes:long})
# :long - %{INT:http.response.body.bytes:int}
HTTPD_COMBINEDLOG %{HTTPD_COMMONLOG} "(?:-|%{DATA:http.request.referrer})" "(?:-|%{DATA:user_agent.original})"
# Error logs
HTTPD20_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{LOGLEVEL:log.level}\] (?:\[client %{IPORHOST:source.address}\] )?%{GREEDYDATA:message}
HTTPD24_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[(?:%{WORD:apache.error.module})?:%{LOGLEVEL:log.level}\] \[pid %{POSINT:process.pid:long}(:tid %{INT:process.thread.id:int})?\](?: \(%{POSINT:apache.error.proxy.error.code?}\)%{DATA:apache.error.proxy.error.message}:)?(?: \[client %{IPORHOST:source.address}(?::%{POSINT:source.port:int})?\])?(?: %{DATA:error.code}:)? %{GREEDYDATA:message}
# :long - %{INT:process.thread.id:int}
HTTPD_ERRORLOG %{HTTPD20_ERRORLOG}|%{HTTPD24_ERRORLOG}
# Deprecated
COMMONAPACHELOG %{HTTPD_COMMONLOG}
COMBINEDAPACHELOG %{HTTPD_COMBINEDLOG}

34
libs/grok/src/main/resources/patterns/ecs-v1/java Normal file
View File

@ -0,0 +1,34 @@
JAVACLASS (?:[a-zA-Z$_][a-zA-Z$_0-9]*\.)*[a-zA-Z$_][a-zA-Z$_0-9]*
#Space is an allowed character to match special cases like 'Native Method' or 'Unknown Source'
JAVAFILE (?:[a-zA-Z$_0-9. -]+)
#Allow special <init>, <clinit> methods
JAVAMETHOD (?:(<(?:cl)?init>)|[a-zA-Z$_][a-zA-Z$_0-9]*)
#Line number is optional in special cases 'Native method' or 'Unknown source'
JAVASTACKTRACEPART %{SPACE}at %{JAVACLASS:java.log.origin.class.name}\.%{JAVAMETHOD:log.origin.function}\(%{JAVAFILE:log.origin.file.name}(?::%{INT:log.origin.file.line:int})?\)
# Java Logs
JAVATHREAD (?:[A-Z]{2}-Processor[\d]+)
JAVALOGMESSAGE (?:.*)
# MMM dd, yyyy HH:mm:ss eg: Jan 9, 2014 7:13:13 AM
# matches default logging configuration in Tomcat 4.1, 5.0, 5.5, 6.0, 7.0
CATALINA7_DATESTAMP %{MONTH} %{MONTHDAY}, %{YEAR} %{HOUR}:%{MINUTE}:%{SECOND} (?:AM|PM)
CATALINA7_LOG %{CATALINA7_DATESTAMP:timestamp} %{JAVACLASS:java.log.origin.class.name}(?: %{JAVAMETHOD:log.origin.function})?\s*(?:%{LOGLEVEL:log.level}:)? %{JAVALOGMESSAGE:message}
# 31-Jul-2020 16:40:38.578 in Tomcat 8.5/9.0
CATALINA8_DATESTAMP %{MONTHDAY}-%{MONTH}-%{YEAR} %{HOUR}:%{MINUTE}:%{SECOND}
CATALINA8_LOG %{CATALINA8_DATESTAMP:timestamp} %{LOGLEVEL:log.level} \[%{DATA:java.log.origin.thread.name}\] %{JAVACLASS:java.log.origin.class.name}\.(?:%{JAVAMETHOD:log.origin.function})? %{JAVALOGMESSAGE:message}
CATALINA_DATESTAMP (?:%{CATALINA8_DATESTAMP})|(?:%{CATALINA7_DATESTAMP})
CATALINALOG (?:%{CATALINA8_LOG})|(?:%{CATALINA7_LOG})
# in Tomcat 5.5, 6.0, 7.0 it is the same as catalina.out logging format
TOMCAT7_LOG %{CATALINA7_LOG}
TOMCAT8_LOG %{CATALINA8_LOG}
# NOTE: a weird log we started with - not sure what TC version this should match out of the box (due the | delimiters)
TOMCATLEGACY_DATESTAMP %{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:%{MINUTE}:%{SECOND}(?: %{ISO8601_TIMEZONE})?
TOMCATLEGACY_LOG %{TOMCATLEGACY_DATESTAMP:timestamp} \| %{LOGLEVEL:log.level} \| %{JAVACLASS:java.log.origin.class.name} - %{JAVALOGMESSAGE:message}
TOMCAT_DATESTAMP (?:%{CATALINA8_DATESTAMP})|(?:%{CATALINA7_DATESTAMP})|(?:%{TOMCATLEGACY_DATESTAMP})
TOMCATLOG (?:%{TOMCAT8_LOG})|(?:%{TOMCAT7_LOG})|(?:%{TOMCATLEGACY_LOG})

13
libs/grok/src/main/resources/patterns/ecs-v1/junos Normal file
View File

@ -0,0 +1,13 @@
# JUNOS 11.4 RT_FLOW patterns
RT_FLOW_TAG (?:RT_FLOW_SESSION_CREATE|RT_FLOW_SESSION_CLOSE|RT_FLOW_SESSION_DENY)
# deprecated legacy name:
RT_FLOW_EVENT RT_FLOW_TAG
RT_FLOW1 %{RT_FLOW_TAG:juniper.srx.tag}: %{GREEDYDATA:juniper.srx.reason}: %{IP:source.ip}/%{INT:source.port:int}->%{IP:destination.ip}/%{INT:destination.port:int} %{DATA:juniper.srx.service_name} %{IP:source.nat.ip}/%{INT:source.nat.port:int}->%{IP:destination.nat.ip}/%{INT:destination.nat.port:int} (?:(?:None)|(?:%{DATA:juniper.srx.src_nat_rule_name})) (?:(?:None)|(?:%{DATA:juniper.srx.dst_nat_rule_name})) %{INT:network.iana_number} %{DATA:rule.name} %{DATA:observer.ingress.zone} %{DATA:observer.egress.zone} %{INT:juniper.srx.session_id} \d+\(%{INT:source.bytes:long}\) \d+\(%{INT:destination.bytes:long}\) %{INT:juniper.srx.elapsed_time:int} .*
# :long - %{INT:source.bytes:int}
# :long - %{INT:destination.bytes:int}
RT_FLOW2 %{RT_FLOW_TAG:juniper.srx.tag}: session created %{IP:source.ip}/%{INT:source.port:int}->%{IP:destination.ip}/%{INT:destination.port:int} %{DATA:juniper.srx.service_name} %{IP:source.nat.ip}/%{INT:source.nat.port:int}->%{IP:destination.nat.ip}/%{INT:destination.nat.port:int} (?:(?:None)|(?:%{DATA:juniper.srx.src_nat_rule_name})) (?:(?:None)|(?:%{DATA:juniper.srx.dst_nat_rule_name})) %{INT:network.iana_number} %{DATA:rule.name} %{DATA:observer.ingress.zone} %{DATA:observer.egress.zone} %{INT:juniper.srx.session_id} .*
RT_FLOW3 %{RT_FLOW_TAG:juniper.srx.tag}: session denied %{IP:source.ip}/%{INT:source.port:int}->%{IP:destination.ip}/%{INT:destination.port:int} %{DATA:juniper.srx.service_name} %{INT:network.iana_number}\(\d\) %{DATA:rule.name} %{DATA:observer.ingress.zone} %{DATA:observer.egress.zone} .*

16
libs/grok/src/main/resources/patterns/ecs-v1/linux-syslog Normal file
View File

@ -0,0 +1,16 @@
SYSLOG5424PRINTASCII [!-~]+
SYSLOGBASE2 (?:%{SYSLOGTIMESTAMP:timestamp}|%{TIMESTAMP_ISO8601:timestamp})(?: %{SYSLOGFACILITY})?(?: %{SYSLOGHOST:host.hostname})?(?: %{SYSLOGPROG}:)?
SYSLOGPAMSESSION %{SYSLOGBASE} (?=%{GREEDYDATA:message})%{WORD:system.auth.pam.module}\(%{DATA:system.auth.pam.origin}\): session %{WORD:system.auth.pam.session_state} for user %{USERNAME:user.name}(?: by %{GREEDYDATA})?
CRON_ACTION [A-Z ]+
CRONLOG %{SYSLOGBASE} \(%{USER:user.name}\) %{CRON_ACTION:system.cron.action} \(%{DATA:message}\)
SYSLOGLINE %{SYSLOGBASE2} %{GREEDYDATA:message}
# IETF 5424 syslog(8) format (see http://www.rfc-editor.org/info/rfc5424)
SYSLOG5424PRI <%{NONNEGINT:log.syslog.priority:int}>
SYSLOG5424SD \[%{DATA}\]+
SYSLOG5424BASE %{SYSLOG5424PRI}%{NONNEGINT:system.syslog.version} +(?:-|%{TIMESTAMP_ISO8601:timestamp}) +(?:-|%{IPORHOST:host.hostname}) +(?:-|%{SYSLOG5424PRINTASCII:process.name}) +(?:-|%{POSINT:process.pid:int}) +(?:-|%{SYSLOG5424PRINTASCII:event.code}) +(?:-|%{SYSLOG5424SD:system.syslog.structured_data})?
SYSLOG5424LINE %{SYSLOG5424BASE} +%{GREEDYDATA:message}

0
libs/grok/src/main/resources/patterns/maven → libs/grok/src/main/resources/patterns/ecs-v1/maven
View File

4
libs/grok/src/main/resources/patterns/ecs-v1/mcollective Normal file
View File

@ -0,0 +1,4 @@
# Remember, these can be multi-line events.
MCOLLECTIVE ., \[%{TIMESTAMP_ISO8601:timestamp} #%{POSINT:process.pid:int}\]%{SPACE}%{LOGLEVEL:log.level}
MCOLLECTIVEAUDIT %{TIMESTAMP_ISO8601:timestamp}:

7
libs/grok/src/main/resources/patterns/ecs-v1/mongodb Normal file
View File

@ -0,0 +1,7 @@
MONGO_LOG %{SYSLOGTIMESTAMP:timestamp} \[%{WORD:mongodb.component}\] %{GREEDYDATA:message}
MONGO_QUERY \{ (?<={ ).*(?= } ntoreturn:) \}
MONGO_SLOWQUERY %{WORD:mongodb.profile.op} %{MONGO_WORDDASH:mongodb.database}\.%{MONGO_WORDDASH:mongodb.collection} %{WORD}: %{MONGO_QUERY:mongodb.query.original} ntoreturn:%{NONNEGINT:mongodb.profile.ntoreturn:int} ntoskip:%{NONNEGINT:mongodb.profile.ntoskip:int} nscanned:%{NONNEGINT:mongodb.profile.nscanned:int}.*? nreturned:%{NONNEGINT:mongodb.profile.nreturned:int}.*? %{INT:mongodb.profile.duration:int}ms
MONGO_WORDDASH \b[\w-]+\b
MONGO3_SEVERITY \w
MONGO3_COMPONENT %{WORD}
MONGO3_LOG %{TIMESTAMP_ISO8601:timestamp} %{MONGO3_SEVERITY:log.level} (?:-|%{MONGO3_COMPONENT:mongodb.component})%{SPACE}(?:\[%{DATA:mongodb.context}\])? %{GREEDYDATA:message}

124
libs/grok/src/main/resources/patterns/ecs-v1/nagios Normal file
View File

@ -0,0 +1,124 @@
##################################################################################
##################################################################################
# Chop Nagios log files to smithereens!
#
# A set of GROK filters to process logfiles generated by Nagios.
# While it does not, this set intends to cover all possible Nagios logs.
#
# Some more work needs to be done to cover all External Commands:
# http://old.nagios.org/developerinfo/externalcommands/commandlist.php
#
# If you need some support on these rules please contact:
# Jelle Smet http://smetj.net
#
#################################################################################
#################################################################################
NAGIOSTIME \[%{NUMBER:timestamp}\]
###############################################
######## Begin nagios log types
###############################################
NAGIOS_TYPE_CURRENT_SERVICE_STATE CURRENT SERVICE STATE
NAGIOS_TYPE_CURRENT_HOST_STATE CURRENT HOST STATE
NAGIOS_TYPE_SERVICE_NOTIFICATION SERVICE NOTIFICATION
NAGIOS_TYPE_HOST_NOTIFICATION HOST NOTIFICATION
NAGIOS_TYPE_SERVICE_ALERT SERVICE ALERT
NAGIOS_TYPE_HOST_ALERT HOST ALERT
NAGIOS_TYPE_SERVICE_FLAPPING_ALERT SERVICE FLAPPING ALERT
NAGIOS_TYPE_HOST_FLAPPING_ALERT HOST FLAPPING ALERT
NAGIOS_TYPE_SERVICE_DOWNTIME_ALERT SERVICE DOWNTIME ALERT
NAGIOS_TYPE_HOST_DOWNTIME_ALERT HOST DOWNTIME ALERT
NAGIOS_TYPE_PASSIVE_SERVICE_CHECK PASSIVE SERVICE CHECK
NAGIOS_TYPE_PASSIVE_HOST_CHECK PASSIVE HOST CHECK
NAGIOS_TYPE_SERVICE_EVENT_HANDLER SERVICE EVENT HANDLER
NAGIOS_TYPE_HOST_EVENT_HANDLER HOST EVENT HANDLER
NAGIOS_TYPE_EXTERNAL_COMMAND EXTERNAL COMMAND
NAGIOS_TYPE_TIMEPERIOD_TRANSITION TIMEPERIOD TRANSITION
###############################################
######## End nagios log types
###############################################
###############################################
######## Begin external check types
###############################################
NAGIOS_EC_DISABLE_SVC_CHECK DISABLE_SVC_CHECK
NAGIOS_EC_ENABLE_SVC_CHECK ENABLE_SVC_CHECK
NAGIOS_EC_DISABLE_HOST_CHECK DISABLE_HOST_CHECK
NAGIOS_EC_ENABLE_HOST_CHECK ENABLE_HOST_CHECK
NAGIOS_EC_PROCESS_SERVICE_CHECK_RESULT PROCESS_SERVICE_CHECK_RESULT
NAGIOS_EC_PROCESS_HOST_CHECK_RESULT PROCESS_HOST_CHECK_RESULT
NAGIOS_EC_SCHEDULE_SERVICE_DOWNTIME SCHEDULE_SERVICE_DOWNTIME
NAGIOS_EC_SCHEDULE_HOST_DOWNTIME SCHEDULE_HOST_DOWNTIME
NAGIOS_EC_DISABLE_HOST_SVC_NOTIFICATIONS DISABLE_HOST_SVC_NOTIFICATIONS
NAGIOS_EC_ENABLE_HOST_SVC_NOTIFICATIONS ENABLE_HOST_SVC_NOTIFICATIONS
NAGIOS_EC_DISABLE_HOST_NOTIFICATIONS DISABLE_HOST_NOTIFICATIONS
NAGIOS_EC_ENABLE_HOST_NOTIFICATIONS ENABLE_HOST_NOTIFICATIONS
NAGIOS_EC_DISABLE_SVC_NOTIFICATIONS DISABLE_SVC_NOTIFICATIONS
NAGIOS_EC_ENABLE_SVC_NOTIFICATIONS ENABLE_SVC_NOTIFICATIONS
###############################################
######## End external check types
###############################################
NAGIOS_WARNING Warning:%{SPACE}%{GREEDYDATA:message}
NAGIOS_CURRENT_SERVICE_STATE %{NAGIOS_TYPE_CURRENT_SERVICE_STATE:nagios.log.type}: %{DATA:host.hostname};%{DATA:service.name};%{DATA:service.state};%{DATA:nagios.log.state_type};%{INT:nagios.log.attempt:int};%{GREEDYDATA:message}
NAGIOS_CURRENT_HOST_STATE %{NAGIOS_TYPE_CURRENT_HOST_STATE:nagios.log.type}: %{DATA:host.hostname};%{DATA:service.state};%{DATA:nagios.log.state_type};%{INT:nagios.log.attempt:int};%{GREEDYDATA:message}
NAGIOS_SERVICE_NOTIFICATION %{NAGIOS_TYPE_SERVICE_NOTIFICATION:nagios.log.type}: %{DATA:user.name};%{DATA:host.hostname};%{DATA:service.name};%{DATA:service.state};%{DATA:nagios.log.notification_command};%{GREEDYDATA:message}
NAGIOS_HOST_NOTIFICATION %{NAGIOS_TYPE_HOST_NOTIFICATION:nagios.log.type}: %{DATA:user.name};%{DATA:host.hostname};%{DATA:service.state};%{DATA:nagios.log.notification_command};%{GREEDYDATA:message}
NAGIOS_SERVICE_ALERT %{NAGIOS_TYPE_SERVICE_ALERT:nagios.log.type}: %{DATA:host.hostname};%{DATA:service.name};%{DATA:service.state};%{DATA:nagios.log.state_type};%{INT:nagios.log.attempt:int};%{GREEDYDATA:message}
NAGIOS_HOST_ALERT %{NAGIOS_TYPE_HOST_ALERT:nagios.log.type}: %{DATA:host.hostname};%{DATA:service.state};%{DATA:nagios.log.state_type};%{INT:nagios.log.attempt:int};%{GREEDYDATA:message}
NAGIOS_SERVICE_FLAPPING_ALERT %{NAGIOS_TYPE_SERVICE_FLAPPING_ALERT:nagios.log.type}: %{DATA:host.hostname};%{DATA:service.name};%{DATA:service.state};%{GREEDYDATA:message}
NAGIOS_HOST_FLAPPING_ALERT %{NAGIOS_TYPE_HOST_FLAPPING_ALERT:nagios.log.type}: %{DATA:host.hostname};%{DATA:service.state};%{GREEDYDATA:message}
NAGIOS_SERVICE_DOWNTIME_ALERT %{NAGIOS_TYPE_SERVICE_DOWNTIME_ALERT:nagios.log.type}: %{DATA:host.hostname};%{DATA:service.name};%{DATA:service.state};%{GREEDYDATA:nagios.log.comment}
NAGIOS_HOST_DOWNTIME_ALERT %{NAGIOS_TYPE_HOST_DOWNTIME_ALERT:nagios.log.type}: %{DATA:host.hostname};%{DATA:service.state};%{GREEDYDATA:nagios.log.comment}
NAGIOS_PASSIVE_SERVICE_CHECK %{NAGIOS_TYPE_PASSIVE_SERVICE_CHECK:nagios.log.type}: %{DATA:host.hostname};%{DATA:service.name};%{DATA:service.state};%{GREEDYDATA:nagios.log.comment}
NAGIOS_PASSIVE_HOST_CHECK %{NAGIOS_TYPE_PASSIVE_HOST_CHECK:nagios.log.type}: %{DATA:host.hostname};%{DATA:service.state};%{GREEDYDATA:nagios.log.comment}
NAGIOS_SERVICE_EVENT_HANDLER %{NAGIOS_TYPE_SERVICE_EVENT_HANDLER:nagios.log.type}: %{DATA:host.hostname};%{DATA:service.name};%{DATA:service.state};%{DATA:nagios.log.state_type};%{DATA:nagios.log.event_handler_name}
NAGIOS_HOST_EVENT_HANDLER %{NAGIOS_TYPE_HOST_EVENT_HANDLER:nagios.log.type}: %{DATA:host.hostname};%{DATA:service.state};%{DATA:nagios.log.state_type};%{DATA:nagios.log.event_handler_name}
NAGIOS_TIMEPERIOD_TRANSITION %{NAGIOS_TYPE_TIMEPERIOD_TRANSITION:nagios.log.type}: %{DATA:service.name};%{NUMBER:nagios.log.period_from:int};%{NUMBER:nagios.log.period_to:int}
####################
#### External checks
####################
#Disable host & service check
NAGIOS_EC_LINE_DISABLE_SVC_CHECK %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios.log.type}: %{NAGIOS_EC_DISABLE_SVC_CHECK:nagios.log.command};%{DATA:host.hostname};%{DATA:service.name}
NAGIOS_EC_LINE_DISABLE_HOST_CHECK %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios.log.type}: %{NAGIOS_EC_DISABLE_HOST_CHECK:nagios.log.command};%{DATA:host.hostname}
#Enable host & service check
NAGIOS_EC_LINE_ENABLE_SVC_CHECK %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios.log.type}: %{NAGIOS_EC_ENABLE_SVC_CHECK:nagios.log.command};%{DATA:host.hostname};%{DATA:service.name}
NAGIOS_EC_LINE_ENABLE_HOST_CHECK %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios.log.type}: %{NAGIOS_EC_ENABLE_HOST_CHECK:nagios.log.command};%{DATA:host.hostname}
#Process host & service check
NAGIOS_EC_LINE_PROCESS_SERVICE_CHECK_RESULT %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios.log.type}: %{NAGIOS_EC_PROCESS_SERVICE_CHECK_RESULT:nagios.log.command};%{DATA:host.hostname};%{DATA:service.name};%{DATA:service.state};%{GREEDYDATA:nagios.log.check_result}
NAGIOS_EC_LINE_PROCESS_HOST_CHECK_RESULT %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios.log.type}: %{NAGIOS_EC_PROCESS_HOST_CHECK_RESULT:nagios.log.command};%{DATA:host.hostname};%{DATA:service.state};%{GREEDYDATA:nagios.log.check_result}
#Disable host & service notifications
NAGIOS_EC_LINE_DISABLE_HOST_SVC_NOTIFICATIONS %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios.log.type}: %{NAGIOS_EC_DISABLE_HOST_SVC_NOTIFICATIONS:nagios.log.command};%{GREEDYDATA:host.hostname}
NAGIOS_EC_LINE_DISABLE_HOST_NOTIFICATIONS %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios.log.type}: %{NAGIOS_EC_DISABLE_HOST_NOTIFICATIONS:nagios.log.command};%{GREEDYDATA:host.hostname}
NAGIOS_EC_LINE_DISABLE_SVC_NOTIFICATIONS %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios.log.type}: %{NAGIOS_EC_DISABLE_SVC_NOTIFICATIONS:nagios.log.command};%{DATA:host.hostname};%{GREEDYDATA:service.name}
#Enable host & service notifications
NAGIOS_EC_LINE_ENABLE_HOST_SVC_NOTIFICATIONS %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios.log.type}: %{NAGIOS_EC_ENABLE_HOST_SVC_NOTIFICATIONS:nagios.log.command};%{GREEDYDATA:host.hostname}
NAGIOS_EC_LINE_ENABLE_HOST_NOTIFICATIONS %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios.log.type}: %{NAGIOS_EC_ENABLE_HOST_NOTIFICATIONS:nagios.log.command};%{GREEDYDATA:host.hostname}
NAGIOS_EC_LINE_ENABLE_SVC_NOTIFICATIONS %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios.log.type}: %{NAGIOS_EC_ENABLE_SVC_NOTIFICATIONS:nagios.log.command};%{DATA:host.hostname};%{GREEDYDATA:service.name}
#Schedule host & service downtime
NAGIOS_EC_LINE_SCHEDULE_HOST_DOWNTIME %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios.log.type}: %{NAGIOS_EC_SCHEDULE_HOST_DOWNTIME:nagios.log.command};%{DATA:host.hostname};%{NUMBER:nagios.log.start_time};%{NUMBER:nagios.log.end_time};%{NUMBER:nagios.log.fixed};%{NUMBER:nagios.log.trigger_id};%{NUMBER:nagios.log.duration:int};%{DATA:user.name};%{DATA:nagios.log.comment}
#End matching line
NAGIOSLOGLINE %{NAGIOSTIME} (?:%{NAGIOS_WARNING}|%{NAGIOS_CURRENT_SERVICE_STATE}|%{NAGIOS_CURRENT_HOST_STATE}|%{NAGIOS_SERVICE_NOTIFICATION}|%{NAGIOS_HOST_NOTIFICATION}|%{NAGIOS_SERVICE_ALERT}|%{NAGIOS_HOST_ALERT}|%{NAGIOS_SERVICE_FLAPPING_ALERT}|%{NAGIOS_HOST_FLAPPING_ALERT}|%{NAGIOS_SERVICE_DOWNTIME_ALERT}|%{NAGIOS_HOST_DOWNTIME_ALERT}|%{NAGIOS_PASSIVE_SERVICE_CHECK}|%{NAGIOS_PASSIVE_HOST_CHECK}|%{NAGIOS_SERVICE_EVENT_HANDLER}|%{NAGIOS_HOST_EVENT_HANDLER}|%{NAGIOS_TIMEPERIOD_TRANSITION}|%{NAGIOS_EC_LINE_DISABLE_SVC_CHECK}|%{NAGIOS_EC_LINE_ENABLE_SVC_CHECK}|%{NAGIOS_EC_LINE_DISABLE_HOST_CHECK}|%{NAGIOS_EC_LINE_ENABLE_HOST_CHECK}|%{NAGIOS_EC_LINE_PROCESS_HOST_CHECK_RESULT}|%{NAGIOS_EC_LINE_PROCESS_SERVICE_CHECK_RESULT}|%{NAGIOS_EC_LINE_SCHEDULE_HOST_DOWNTIME}|%{NAGIOS_EC_LINE_DISABLE_HOST_SVC_NOTIFICATIONS}|%{NAGIOS_EC_LINE_ENABLE_HOST_SVC_NOTIFICATIONS}|%{NAGIOS_EC_LINE_DISABLE_HOST_NOTIFICATIONS}|%{NAGIOS_EC_LINE_ENABLE_HOST_NOTIFICATIONS}|%{NAGIOS_EC_LINE_DISABLE_SVC_NOTIFICATIONS}|%{NAGIOS_EC_LINE_ENABLE_SVC_NOTIFICATIONS})

2
libs/grok/src/main/resources/patterns/ecs-v1/postgresql Normal file
View File

@ -0,0 +1,2 @@
# Default postgresql pg_log format pattern
POSTGRESQL %{DATESTAMP:timestamp} %{TZ:event.timezone} %{DATA:user.name} %{GREEDYDATA:postgresql.log.connection_id} %{POSINT:process.pid:int}

13
libs/grok/src/main/resources/patterns/ecs-v1/rails Normal file
View File

@ -0,0 +1,13 @@
RUUID \h{32}
# rails controller with action
RCONTROLLER (?<rails.controller.class>[^#]+)#(?<rails.controller.action>\w+)
# this will often be the only line:
RAILS3HEAD (?m)Started %{WORD:http.request.method} "%{URIPATHPARAM:url.original}" for %{IPORHOST:source.address} at (?<timestamp>%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:%{MINUTE}:%{SECOND} %{ISO8601_TIMEZONE})
# for some a strange reason, params are stripped of {} - not sure that's a good idea.
RPROCESSING \W*Processing by %{RCONTROLLER} as (?<rails.request.format>\S+)(?:\W*Parameters: {%{DATA:rails.request.params}}\W*)?
RAILS3FOOT Completed %{POSINT:http.response.status_code:int}%{DATA} in %{NUMBER:rails.request.duration.total:float}ms %{RAILS3PROFILE}%{GREEDYDATA}
RAILS3PROFILE (?:\(Views: %{NUMBER:rails.request.duration.view:float}ms \| ActiveRecord: %{NUMBER:rails.request.duration.active_record:float}ms|\(ActiveRecord: %{NUMBER:rails.request.duration.active_record:float}ms)?
# putting it all together
RAILS3 %{RAILS3HEAD}(?:%{RPROCESSING})?(?<rails.request.explain.original>(?:%{DATA}\n)*)(?:%{RAILS3FOOT})?

3
libs/grok/src/main/resources/patterns/ecs-v1/redis Normal file
View File

@ -0,0 +1,3 @@
REDISTIMESTAMP %{MONTHDAY} %{MONTH} %{TIME}
REDISLOG \[%{POSINT:process.pid:int}\] %{REDISTIMESTAMP:timestamp} \*
REDISMONLOG %{NUMBER:timestamp} \[%{INT:redis.database.id} %{IP:client.ip}:%{POSINT:client.port:int}\] "%{WORD:redis.command.name}"\s?%{GREEDYDATA:redis.command.args}

2
libs/grok/src/main/resources/patterns/ecs-v1/ruby Normal file
View File

@ -0,0 +1,2 @@
RUBY_LOGLEVEL (?:DEBUG|FATAL|ERROR|WARN|INFO)
RUBY_LOGGER [DFEWI], \[%{TIMESTAMP_ISO8601:timestamp} #%{POSINT:process.pid:int}\] *%{RUBY_LOGLEVEL:log.level} -- +%{DATA:process.name}: %{GREEDYDATA:message}

6
libs/grok/src/main/resources/patterns/ecs-v1/squid Normal file
View File

@ -0,0 +1,6 @@
# Pattern squid3
# Documentation of squid3 logs formats can be found at the following link:
# http://wiki.squid-cache.org/Features/LogFormat
SQUID3_STATUS (?:%{POSINT:http.response.status_code:int}|0|000)
SQUID3 %{NUMBER:timestamp}\s+%{NUMBER:squid.request.duration:int}\s%{IP:source.ip}\s%{WORD:event.action}/%{SQUID3_STATUS}\s%{INT:http.response.bytes:long}\s%{WORD:http.request.method}\s%{NOTSPACE:url.original}\s(?:-|%{NOTSPACE:user.name})\s%{WORD:squid.hierarchy_code}/(?:-|%{IPORHOST:destination.address})\s(?:-|%{NOTSPACE:http.response.mime_type})
# :long - %{INT:http.response.bytes:int}

33
libs/grok/src/main/resources/patterns/ecs-v1/zeek Normal file
View File

@ -0,0 +1,33 @@
# updated Zeek log matching, for legacy matching see the patters/ecs-v1/bro
ZEEK_BOOL [TF]
ZEEK_DATA [^\t]+
# http.log - the 'new' format (compared to BRO_HTTP)
# has *version* and *origin* fields added and *filename* replaced with *orig_filenames* + *resp_filenames*
ZEEK_HTTP %{NUMBER:timestamp}\t%{NOTSPACE:zeek.session_id}\t%{IP:source.ip}\t%{INT:source.port:int}\t%{IP:destination.ip}\t%{INT:destination.port:int}\t%{INT:zeek.http.trans_depth:int}\t(?:-|%{WORD:http.request.method})\t(?:-|%{ZEEK_DATA:url.domain})\t(?:-|%{ZEEK_DATA:url.original})\t(?:-|%{ZEEK_DATA:http.request.referrer})\t(?:-|%{NUMBER:http.version})\t(?:-|%{ZEEK_DATA:user_agent.original})\t(?:-|%{ZEEK_DATA:zeek.http.origin})\t(?:-|%{NUMBER:http.request.body.bytes:long})\t(?:-|%{NUMBER:http.response.body.bytes:long})\t(?:-|%{POSINT:http.response.status_code:int})\t(?:-|%{DATA:zeek.http.status_msg})\t(?:-|%{POSINT:zeek.http.info_code:int})\t(?:-|%{DATA:zeek.http.info_msg})\t(?:\(empty\)|%{ZEEK_DATA:zeek.http.tags})\t(?:-|%{ZEEK_DATA:url.username})\t(?:-|%{ZEEK_DATA:url.password})\t(?:-|%{ZEEK_DATA:zeek.http.proxied})\t(?:-|%{ZEEK_DATA:zeek.http.orig_fuids})\t(?:-|%{ZEEK_DATA:zeek.http.orig_filenames})\t(?:-|%{ZEEK_DATA:http.request.mime_type})\t(?:-|%{ZEEK_DATA:zeek.http.resp_fuids})\t(?:-|%{ZEEK_DATA:zeek.http.resp_filenames})\t(?:-|%{ZEEK_DATA:http.response.mime_type})
# :long - %{NUMBER:http.request.body.bytes:int}
# :long - %{NUMBER:http.response.body.bytes:int}
# dns.log - 'updated' BRO_DNS format (added *zeek.dns.rtt*)
ZEEK_DNS %{NUMBER:timestamp}\t%{NOTSPACE:zeek.session_id}\t%{IP:source.ip}\t%{INT:source.port:int}\t%{IP:destination.ip}\t%{INT:destination.port:int}\t%{WORD:network.transport}\t(?:-|%{INT:dns.id:int})\t(?:-|%{NUMBER:zeek.dns.rtt:float})\t(?:-|%{ZEEK_DATA:dns.question.name})\t(?:-|%{INT:zeek.dns.qclass:int})\t(?:-|%{ZEEK_DATA:zeek.dns.qclass_name})\t(?:-|%{INT:zeek.dns.qtype:int})\t(?:-|%{ZEEK_DATA:dns.question.type})\t(?:-|%{INT:zeek.dns.rcode:int})\t(?:-|%{ZEEK_DATA:dns.response_code})\t%{ZEEK_BOOL:zeek.dns.AA}\t%{ZEEK_BOOL:zeek.dns.TC}\t%{ZEEK_BOOL:zeek.dns.RD}\t%{ZEEK_BOOL:zeek.dns.RA}\t%{NONNEGINT:zeek.dns.Z:int}\t(?:-|%{ZEEK_DATA:zeek.dns.answers})\t(?:-|%{DATA:zeek.dns.TTLs})\t(?:-|%{ZEEK_BOOL:zeek.dns.rejected})
# conn.log - the 'new' format (requires *zeek.connection.local_resp*, handles `(empty)` as `-` for tunnel_parents, and optional mac adresses)
ZEEK_CONN %{NUMBER:timestamp}\t%{NOTSPACE:zeek.session_id}\t%{IP:source.ip}\t%{INT:source.port:int}\t%{IP:destination.ip}\t%{INT:destination.port:int}\t%{WORD:network.transport}\t(?:-|%{ZEEK_DATA:network.protocol})\t(?:-|%{NUMBER:zeek.connection.duration:float})\t(?:-|%{INT:zeek.connection.orig_bytes:long})\t(?:-|%{INT:zeek.connection.resp_bytes:long})\t(?:-|%{ZEEK_DATA:zeek.connection.state})\t(?:-|%{ZEEK_BOOL:zeek.connection.local_orig})\t(?:-|%{ZEEK_BOOL:zeek.connection.local_resp})\t(?:-|%{INT:zeek.connection.missed_bytes:long})\t(?:-|%{ZEEK_DATA:zeek.connection.history})\t(?:-|%{INT:source.packets:long})\t(?:-|%{INT:source.bytes:long})\t(?:-|%{INT:destination.packets:long})\t(?:-|%{INT:destination.bytes:long})\t(?:-|%{ZEEK_DATA:zeek.connection.tunnel_parents})(?:\t(?:-|%{COMMONMAC:source.mac})\t(?:-|%{COMMONMAC:destination.mac}))?
# :long - %{INT:zeek.connection.orig_bytes:int}
# :long - %{INT:zeek.connection.resp_bytes:int}
# :long - %{INT:zeek.connection.missed_bytes:int}
# :long - %{INT:source.packets:int}
# :long - %{INT:source.bytes:int}
# :long - %{INT:destination.packets:int}
# :long - %{INT:destination.bytes:int}
# files.log - updated BRO_FILES format (2 new fields added at the end)
ZEEK_FILES_TX_HOSTS (?:-|%{IP:server.ip})|(?<zeek.files.tx_hosts>%{IP:server.ip}(?:\s,%{IP})+)
ZEEK_FILES_RX_HOSTS (?:-|%{IP:client.ip})|(?<zeek.files.rx_hosts>%{IP:client.ip}(?:\s,%{IP})+)
ZEEK_FILES %{NUMBER:timestamp}\t%{NOTSPACE:zeek.files.fuid}\t%{ZEEK_FILES_TX_HOSTS}\t%{ZEEK_FILES_RX_HOSTS}\t(?:-|%{ZEEK_DATA:zeek.files.session_ids})\t(?:-|%{ZEEK_DATA:zeek.files.source})\t(?:-|%{INT:zeek.files.depth:int})\t(?:-|%{ZEEK_DATA:zeek.files.analyzers})\t(?:-|%{ZEEK_DATA:file.mime_type})\t(?:-|%{ZEEK_DATA:file.name})\t(?:-|%{NUMBER:zeek.files.duration:float})\t(?:-|%{ZEEK_DATA:zeek.files.local_orig})\t(?:-|%{ZEEK_BOOL:zeek.files.is_orig})\t(?:-|%{INT:zeek.files.seen_bytes:long})\t(?:-|%{INT:file.size:long})\t(?:-|%{INT:zeek.files.missing_bytes:long})\t(?:-|%{INT:zeek.files.overflow_bytes:long})\t(?:-|%{ZEEK_BOOL:zeek.files.timedout})\t(?:-|%{ZEEK_DATA:zeek.files.parent_fuid})\t(?:-|%{ZEEK_DATA:file.hash.md5})\t(?:-|%{ZEEK_DATA:file.hash.sha1})\t(?:-|%{ZEEK_DATA:file.hash.sha256})\t(?:-|%{ZEEK_DATA:zeek.files.extracted})(?:\t(?:-|%{ZEEK_BOOL:zeek.files.extracted_cutoff})\t(?:-|%{INT:zeek.files.extracted_size:long}))?
# :long - %{INT:zeek.files.seen_bytes:int}
# :long - %{INT:file.size:int}
# :long - %{INT:zeek.files.missing_bytes:int}
# :long - %{INT:zeek.files.overflow_bytes:int}
# :long - %{INT:zeek.files.extracted_size:int}

0
libs/grok/src/main/resources/patterns/aws → libs/grok/src/main/resources/patterns/legacy/aws
View File

0
libs/grok/src/main/resources/patterns/bacula → libs/grok/src/main/resources/patterns/legacy/bacula
View File

0
libs/grok/src/main/resources/patterns/bind → libs/grok/src/main/resources/patterns/legacy/bind
View File

0
libs/grok/src/main/resources/patterns/bro → libs/grok/src/main/resources/patterns/legacy/bro
View File

0
libs/grok/src/main/resources/patterns/exim → libs/grok/src/main/resources/patterns/legacy/exim
View File

0
libs/grok/src/main/resources/patterns/firewalls → libs/grok/src/main/resources/patterns/legacy/firewalls
View File

0
libs/grok/src/main/resources/patterns/grok-patterns → libs/grok/src/main/resources/patterns/legacy/grok-patterns
View File

0
libs/grok/src/main/resources/patterns/haproxy → libs/grok/src/main/resources/patterns/legacy/haproxy
View File

0
libs/grok/src/main/resources/patterns/httpd → libs/grok/src/main/resources/patterns/legacy/httpd
View File

0
libs/grok/src/main/resources/patterns/java → libs/grok/src/main/resources/patterns/legacy/java
View File

0
libs/grok/src/main/resources/patterns/junos → libs/grok/src/main/resources/patterns/legacy/junos
View File

0
libs/grok/src/main/resources/patterns/linux-syslog → libs/grok/src/main/resources/patterns/legacy/linux-syslog
View File

1
libs/grok/src/main/resources/patterns/legacy/maven Normal file
View File

@ -0,0 +1 @@
MAVEN_VERSION (?:(\d+)\.)?(?:(\d+)\.)?(\*|\d+)(?:[.-](RELEASE|SNAPSHOT))?

0
libs/grok/src/main/resources/patterns/mcollective-patterns → libs/grok/src/main/resources/patterns/legacy/mcollective-patterns
View File

0
libs/grok/src/main/resources/patterns/mongodb → libs/grok/src/main/resources/patterns/legacy/mongodb
View File

0
libs/grok/src/main/resources/patterns/nagios → libs/grok/src/main/resources/patterns/legacy/nagios
View File

0
libs/grok/src/main/resources/patterns/postgresql → libs/grok/src/main/resources/patterns/legacy/postgresql
View File

0
libs/grok/src/main/resources/patterns/rails → libs/grok/src/main/resources/patterns/legacy/rails
View File

0
libs/grok/src/main/resources/patterns/redis → libs/grok/src/main/resources/patterns/legacy/redis
View File

0
libs/grok/src/main/resources/patterns/ruby → libs/grok/src/main/resources/patterns/legacy/ruby
View File

0
libs/grok/src/main/resources/patterns/squid → libs/grok/src/main/resources/patterns/legacy/squid
View File

450
libs/grok/src/test/java/org/elasticsearch/grok/GrokTests.java
View File

@ -8,10 +8,12 @@
package org.elasticsearch.grok;
import org.elasticsearch.core.Tuple;
import org.elasticsearch.grok.GrokCaptureConfig.NativeExtracterMap;
import org.elasticsearch.test.ESTestCase;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
@ -27,6 +29,7 @@ import java.util.function.Function;
import java.util.function.IntConsumer;
import java.util.function.LongConsumer;
import static org.elasticsearch.core.Tuple.tuple;
import static org.elasticsearch.grok.GrokCaptureType.BOOLEAN;
import static org.elasticsearch.grok.GrokCaptureType.DOUBLE;
import static org.elasticsearch.grok.GrokCaptureType.FLOAT;
@ -40,15 +43,26 @@ import static org.hamcrest.Matchers.nullValue;
public class GrokTests extends ESTestCase {
public void testMatchWithoutCaptures() {
Grok grok = new Grok(Grok.BUILTIN_PATTERNS, "value", logger::warn);
testMatchWithoutCaptures(false);
testMatchWithoutCaptures(true);
}
private void testMatchWithoutCaptures(boolean ecsCompatibility) {
Grok grok = new Grok(Grok.getBuiltinPatterns(ecsCompatibility), "value", logger::warn);
assertThat(grok.captures("value"), equalTo(Map.of()));
assertThat(grok.captures("prefix_value"), equalTo(Map.of()));
assertThat(grok.captures("no_match"), nullValue());
}
public void testCaputuresBytes() {
Grok grok = new Grok(Grok.BUILTIN_PATTERNS, "%{NUMBER:n:int}", logger::warn);
public void testCapturesBytes() {
testCapturesBytes(false);
testCapturesBytes(true);
}
private void testCapturesBytes(boolean ecsCompatibility) {
Grok grok = new Grok(Grok.getBuiltinPatterns(ecsCompatibility), "%{NUMBER:n:int}", logger::warn);
byte[] utf8 = "10".getBytes(StandardCharsets.UTF_8);
assertThat(captureBytes(grok, utf8, 0, utf8.length), equalTo(Map.of("n", 10)));
assertThat(captureBytes(grok, utf8, 0, 1), equalTo(Map.of("n", 1)));
@ -72,79 +86,171 @@ public class GrokTests extends ESTestCase {
}
public void testSimpleSyslogLine() {
String line = "Mar 16 00:01:25 evita postfix/smtpd[1713]: connect from camomile.cloud9.net[168.100.1.3]";
Grok grok = new Grok(Grok.BUILTIN_PATTERNS, "%{SYSLOGLINE}", logger::warn);
assertCaptureConfig(
grok,
Map.ofEntries(
Map.entry("facility", STRING),
Map.entry("logsource", STRING),
Map.entry("message", STRING),
Map.entry("pid", STRING),
Map.entry("priority", STRING),
Map.entry("program", STRING),
Map.entry("timestamp", STRING),
Map.entry("timestamp8601", STRING)
)
final String logSource = "evita";
final String timestamp = "Mar 16 00:01:25";
final String message = "connect from camomile.cloud9.net[168.100.1.3]";
final String program = "postfix/smtpd";
testSimpleSyslogLine(
false,
tuple(Map.entry("facility", STRING), null),
tuple(Map.entry("logsource", STRING), logSource),
tuple(Map.entry("message", STRING), message),
tuple(Map.entry("pid", STRING), "1713"),
tuple(Map.entry("priority", STRING), null),
tuple(Map.entry("program", STRING), program),
tuple(Map.entry("timestamp", STRING), timestamp),
tuple(Map.entry("timestamp8601", STRING), null),
List.of()
);
testSimpleSyslogLine(
true,
tuple(Map.entry("log.syslog.facility.code", INTEGER), null),
tuple(Map.entry("host.hostname", STRING), logSource),
tuple(Map.entry("message", STRING), message),
tuple(Map.entry("process.pid", INTEGER), 1713),
tuple(Map.entry("log.syslog.priority", INTEGER), null),
tuple(Map.entry("process.name", STRING), program),
tuple(Map.entry("timestamp", STRING), timestamp),
null,
List.of("timestamp")
);
}
private void testSimpleSyslogLine(
boolean ecsCompatibility,
Tuple<Map.Entry<String, GrokCaptureType>, Object> facility,
Tuple<Map.Entry<String, GrokCaptureType>, Object> logSource,
Tuple<Map.Entry<String, GrokCaptureType>, Object> message,
Tuple<Map.Entry<String, GrokCaptureType>, Object> pid,
Tuple<Map.Entry<String, GrokCaptureType>, Object> priority,
Tuple<Map.Entry<String, GrokCaptureType>, Object> program,
Tuple<Map.Entry<String, GrokCaptureType>, Object> timestamp,
Tuple<Map.Entry<String, GrokCaptureType>, Object> timestamp8601,
List<String> acceptedDuplicates
) {
String line = "Mar 16 00:01:25 evita postfix/smtpd[1713]: connect from camomile.cloud9.net[168.100.1.3]";
Grok grok = new Grok(Grok.getBuiltinPatterns(ecsCompatibility), "%{SYSLOGLINE}", logger::warn);
Map<String, GrokCaptureType> captureTypes = new HashMap<>();
captureTypes.put(facility.v1().getKey(), facility.v1().getValue());
captureTypes.put(logSource.v1().getKey(), logSource.v1().getValue());
captureTypes.put(message.v1().getKey(), message.v1().getValue());
captureTypes.put(pid.v1().getKey(), pid.v1().getValue());
captureTypes.put(priority.v1().getKey(), priority.v1().getValue());
captureTypes.put(program.v1().getKey(), program.v1().getValue());
captureTypes.put(timestamp.v1().getKey(), timestamp.v1().getValue());
if (timestamp8601 != null) {
captureTypes.put(timestamp8601.v1().getKey(), timestamp8601.v1().getValue());
}
assertCaptureConfig(grok, captureTypes, acceptedDuplicates);
Map<String, Object> matches = grok.captures(line);
assertEquals("evita", matches.get("logsource"));
assertEquals("Mar 16 00:01:25", matches.get("timestamp"));
assertEquals("connect from camomile.cloud9.net[168.100.1.3]", matches.get("message"));
assertEquals("postfix/smtpd", matches.get("program"));
assertEquals("1713", matches.get("pid"));
assertEquals(logSource.v2(), matches.get(logSource.v1().getKey()));
assertEquals(timestamp.v2(), matches.get(timestamp.v1().getKey()));
assertEquals(message.v2(), matches.get(message.v1().getKey()));
assertEquals(program.v2(), matches.get(program.v1().getKey()));
assertEquals(pid.v2(), matches.get(pid.v1().getKey()));
String[] logsource = new String[1];
GrokCaptureExtracter logsourceExtracter = namedConfig(grok, "logsource").nativeExtracter(new ThrowingNativeExtracterMap() {
@Override
public GrokCaptureExtracter forString(Function<Consumer<String>, GrokCaptureExtracter> buildExtracter) {
return buildExtracter.apply(str -> logsource[0] = str);
}
});
GrokCaptureExtracter logsourceExtracter =
namedConfig(grok, logSource.v1().getKey())
.nativeExtracter(new ThrowingNativeExtracterMap() {
@Override
public GrokCaptureExtracter forString(Function<Consumer<String>, GrokCaptureExtracter> buildExtracter) {
return buildExtracter.apply(str -> logsource[0] = str);
}
});
assertThat(specificCapture(grok, line, logsourceExtracter), is(true));
assertThat(logsource[0], equalTo("evita"));
assertThat(logsource[0], equalTo(logSource.v2()));
}
public void testSyslog5424Line() {
final String ts = "2009-06-30T18:30:00+02:00";
final String host = "paxton.local";
final String app = "grokdebug";
final String sd = "[id1 foo=\\\"bar\\\"][id2 baz=\\\"something\\\"]";
final String msg = "Hello, syslog.";
final String ver = "1";
testSyslog5424Line(
false,
tuple(Map.entry("syslog5424_app", STRING), app),
tuple(Map.entry("syslog5424_host", STRING), host),
tuple(Map.entry("syslog5424_msg", STRING), msg),
tuple(Map.entry("syslog5424_msgid", STRING), null),
tuple(Map.entry("syslog5424_pri", STRING), "191"),
tuple(Map.entry("syslog5424_proc", STRING), "4123"),
tuple(Map.entry("syslog5424_sd", STRING), sd),
tuple(Map.entry("syslog5424_ts", STRING), ts),
tuple(Map.entry("syslog5424_ver", STRING), ver)
);
testSyslog5424Line(
true,
tuple(Map.entry("process.name", STRING), app),
tuple(Map.entry("host.hostname", STRING), host),
tuple(Map.entry("message", STRING), msg),
tuple(Map.entry("event.code", STRING), null),
tuple(Map.entry("log.syslog.priority", INTEGER), 191),
tuple(Map.entry("process.pid", INTEGER), 4123),
tuple(Map.entry("system.syslog.structured_data", STRING), sd),
tuple(Map.entry("timestamp", STRING), ts),
tuple(Map.entry("system.syslog.version", STRING), ver)
);
}
private void testSyslog5424Line(
boolean ecsCompatibility,
Tuple<Map.Entry<String, GrokCaptureType>, Object> app,
Tuple<Map.Entry<String, GrokCaptureType>, Object> host,
Tuple<Map.Entry<String, GrokCaptureType>, Object> msg,
Tuple<Map.Entry<String, GrokCaptureType>, Object> msgid,
Tuple<Map.Entry<String, GrokCaptureType>, Object> pri,
Tuple<Map.Entry<String, GrokCaptureType>, Object> proc,
Tuple<Map.Entry<String, GrokCaptureType>, Object> sd,
Tuple<Map.Entry<String, GrokCaptureType>, Object> ts,
Tuple<Map.Entry<String, GrokCaptureType>, Object> ver
) {
String line = "<191>1 2009-06-30T18:30:00+02:00 paxton.local grokdebug 4123 - [id1 foo=\\\"bar\\\"][id2 baz=\\\"something\\\"] " +
"Hello, syslog.";
Grok grok = new Grok(Grok.BUILTIN_PATTERNS, "%{SYSLOG5424LINE}", logger::warn);
"Hello, syslog.";
Grok grok = new Grok(Grok.getBuiltinPatterns(ecsCompatibility), "%{SYSLOG5424LINE}", logger::warn);
assertCaptureConfig(
grok,
Map.ofEntries(
Map.entry("syslog5424_app", STRING),
Map.entry("syslog5424_host", STRING),
Map.entry("syslog5424_msg", STRING),
Map.entry("syslog5424_msgid", STRING),
Map.entry("syslog5424_pri", STRING),
Map.entry("syslog5424_proc", STRING),
Map.entry("syslog5424_sd", STRING),
Map.entry("syslog5424_ts", STRING),
Map.entry("syslog5424_ver", STRING)
)
Map.ofEntries(app.v1(), host.v1(), msg.v1(), msgid.v1(), pri.v1(), proc.v1(), sd.v1(), ts.v1(), ver.v1())
);
Map<String, Object> matches = grok.captures(line);
assertEquals("191", matches.get("syslog5424_pri"));
assertEquals("1", matches.get("syslog5424_ver"));
assertEquals("2009-06-30T18:30:00+02:00", matches.get("syslog5424_ts"));
assertEquals("paxton.local", matches.get("syslog5424_host"));
assertEquals("grokdebug", matches.get("syslog5424_app"));
assertEquals("4123", matches.get("syslog5424_proc"));
assertEquals(null, matches.get("syslog5424_msgid"));
assertEquals("[id1 foo=\\\"bar\\\"][id2 baz=\\\"something\\\"]", matches.get("syslog5424_sd"));
assertEquals("Hello, syslog.", matches.get("syslog5424_msg"));
assertEquals(pri.v2(), matches.get(pri.v1().getKey()));
assertEquals(ver.v2(), matches.get(ver.v1().getKey()));
assertEquals(ts.v2(), matches.get(ts.v1().getKey()));
assertEquals(host.v2(), matches.get(host.v1().getKey()));
assertEquals(app.v2(), matches.get(app.v1().getKey()));
assertEquals(proc.v2(), matches.get(proc.v1().getKey()));
assertEquals(msgid.v2(), matches.get(msgid.v1().getKey()));
assertEquals(sd.v2(), matches.get(sd.v1().getKey()));
assertEquals(msg.v2(), matches.get(msg.v1().getKey()));
}
public void testDatePattern() {
testDatePattern(false);
testDatePattern(true);
}
private void testDatePattern(boolean ecsCompatibility) {
String line = "fancy 12-12-12 12:12:12";
Grok grok = new Grok(Grok.BUILTIN_PATTERNS, "(?<timestamp>%{DATE_EU} %{TIME})", logger::warn);
Grok grok = new Grok(Grok.getBuiltinPatterns(ecsCompatibility), "(?<timestamp>%{DATE_EU} %{TIME})", logger::warn);
assertCaptureConfig(grok, Map.of("timestamp", STRING));
Map<String, Object> matches = grok.captures(line);
assertEquals("12-12-12 12:12:12", matches.get("timestamp"));
}
public void testNilCoercedValues() {
Grok grok = new Grok(Grok.BUILTIN_PATTERNS, "test (N/A|%{BASE10NUM:duration:float}ms)", logger::warn);
testNilCoercedValues(false);
testNilCoercedValues(true);
}
private void testNilCoercedValues(boolean ecsCompatibility) {
Grok grok = new Grok(Grok.getBuiltinPatterns(ecsCompatibility), "test (N/A|%{BASE10NUM:duration:float}ms)", logger::warn);
assertCaptureConfig(grok, Map.of("duration", FLOAT));
Map<String, Object> matches = grok.captures("test 28.4ms");
assertEquals(28.4f, matches.get("duration"));
@ -153,7 +259,12 @@ public class GrokTests extends ESTestCase {
}
public void testNilWithNoCoercion() {
Grok grok = new Grok(Grok.BUILTIN_PATTERNS, "test (N/A|%{BASE10NUM:duration}ms)", logger::warn);
testNilWithNoCoercion(false);
testNilWithNoCoercion(true);
}
private void testNilWithNoCoercion(boolean ecsCompatibility) {
Grok grok = new Grok(Grok.getBuiltinPatterns(ecsCompatibility), "test (N/A|%{BASE10NUM:duration}ms)", logger::warn);
assertCaptureConfig(grok, Map.of("duration", STRING));
Map<String, Object> matches = grok.captures("test 28.4ms");
assertEquals("28.4", matches.get("duration"));
@ -162,9 +273,17 @@ public class GrokTests extends ESTestCase {
}
public void testUnicodeSyslog() {
Grok grok = new Grok(Grok.BUILTIN_PATTERNS, "<%{POSINT:syslog_pri}>%{SPACE}%{SYSLOGTIMESTAMP:syslog_timestamp} " +
testUnicodeSyslog(false);
testUnicodeSyslog(true);
}
private void testUnicodeSyslog(boolean ecsCompatibility) {
Grok grok = new Grok(
Grok.getBuiltinPatterns(ecsCompatibility),
"<%{POSINT:syslog_pri}>%{SPACE}%{SYSLOGTIMESTAMP:syslog_timestamp} " +
"%{SYSLOGHOST:syslog_hostname} %{PROG:syslog_program}(:?)(?:\\[%{GREEDYDATA:syslog_pid}\\])?(:?) " +
"%{GREEDYDATA:syslog_message}", logger::warn);
"%{GREEDYDATA:syslog_message}", logger::warn
);
assertCaptureConfig(
grok,
Map.ofEntries(
@ -185,21 +304,36 @@ public class GrokTests extends ESTestCase {
}
public void testNamedFieldsWithWholeTextMatch() {
Grok grok = new Grok(Grok.BUILTIN_PATTERNS, "%{DATE_EU:stimestamp}", logger::warn);
testNamedFieldsWithWholeTextMatch(false);
testNamedFieldsWithWholeTextMatch(true);
}
private void testNamedFieldsWithWholeTextMatch(boolean ecsCompatibility) {
Grok grok = new Grok(Grok.getBuiltinPatterns(ecsCompatibility), "%{DATE_EU:stimestamp}", logger::warn);
assertCaptureConfig(grok, Map.of("stimestamp", STRING));
Map<String, Object> matches = grok.captures("11/01/01");
assertThat(matches.get("stimestamp"), equalTo("11/01/01"));
}
public void testWithOniguramaNamedCaptures() {
Grok grok = new Grok(Grok.BUILTIN_PATTERNS, "(?<foo>\\w+)", logger::warn);
testWithOniguramaNamedCaptures(false);
testWithOniguramaNamedCaptures(true);
}
private void testWithOniguramaNamedCaptures(boolean ecsCompatibility) {
Grok grok = new Grok(Grok.getBuiltinPatterns(ecsCompatibility), "(?<foo>\\w+)", logger::warn);
assertCaptureConfig(grok, Map.of("foo", STRING));
Map<String, Object> matches = grok.captures("hello world");
assertThat(matches.get("foo"), equalTo("hello"));
}
public void testISO8601() {
Grok grok = new Grok(Grok.BUILTIN_PATTERNS, "^%{TIMESTAMP_ISO8601}$", logger::warn);
testISO8601(false);
testISO8601(true);
}
private void testISO8601(boolean ecsCompatibility) {
Grok grok = new Grok(Grok.getBuiltinPatterns(ecsCompatibility), "^%{TIMESTAMP_ISO8601}$", logger::warn);
assertCaptureConfig(grok, Map.of());
List<String> timeMessages = Arrays.asList(
"2001-01-01T00:00:00",
@ -224,7 +358,12 @@ public class GrokTests extends ESTestCase {
}
public void testNotISO8601() {
Grok grok = new Grok(Grok.BUILTIN_PATTERNS, "^%{TIMESTAMP_ISO8601}$", logger::warn);
testNotISO8601(false, List.of("2001-01-01T0:00:00")); // legacy patterns do not permit single-digit hours
testNotISO8601(true, List.of());
}
private void testNotISO8601(boolean ecsCompatibility, List<String> additionalCases) {
Grok grok = new Grok(Grok.getBuiltinPatterns(ecsCompatibility), "^%{TIMESTAMP_ISO8601}$", logger::warn);
assertCaptureConfig(grok, Map.of());
List<String> timeMessages = Arrays.asList(
"2001-13-01T00:00:00", // invalid month
@ -234,7 +373,6 @@ public class GrokTests extends ESTestCase {
"2001-01-aT00:00:00", // invalid day
"2001-01-1aT00:00:00", // invalid day
"2001-01-01Ta0:00:00", // invalid hour
"2001-01-01T0:00:00", // invalid hour
"2001-01-01T25:00:00", // invalid hour
"2001-01-01T01:60:00", // invalid minute
"2001-01-01T00:aa:00", // invalid minute
@ -250,7 +388,9 @@ public class GrokTests extends ESTestCase {
"2001-01-01T00:00:00-2500", // invalid timezone
"2001-01-01T00:00:00-00:61" // invalid timezone
);
for (String msg : timeMessages) {
List<String> timesToTest = new ArrayList<>(timeMessages);
timesToTest.addAll(additionalCases);
for (String msg : timesToTest) {
assertThat(grok.match(msg), is(false));
}
}
@ -350,8 +490,13 @@ public class GrokTests extends ESTestCase {
}
public void testBooleanCaptures() {
testBooleanCaptures(false);
testBooleanCaptures(true);
}
private void testBooleanCaptures(boolean ecsCompatibility) {
String pattern = "%{WORD:name}=%{WORD:status:boolean}";
Grok g = new Grok(Grok.BUILTIN_PATTERNS, pattern, logger::warn);
Grok g = new Grok(Grok.getBuiltinPatterns(ecsCompatibility), pattern, logger::warn);
assertCaptureConfig(g, Map.of("name", STRING, "status", BOOLEAN));
String text = "active=true";
@ -451,42 +596,101 @@ public class GrokTests extends ESTestCase {
}
public void testApacheLog() {
final String agent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.12785 " +
"YaBrowser/13.12.1599.12785 Safari/537.36";
final String clientIp = "31.184.238.164";
final String timestamp = "24/Jul/2014:05:35:37 +0530";
final String verb = "GET";
final String request = "/logs/access.log";
final String httpVersion = "1.0";
final String referrer = "http://8rursodiol.enjin.com";
testApacheLog(
false,
tuple(Map.entry("agent", STRING), "\"" + agent + "\""),
tuple(Map.entry("auth", STRING), "-"),
tuple(Map.entry("bytes", STRING), "69849"),
tuple(Map.entry("clientip", STRING), clientIp),
tuple(Map.entry("httpversion", STRING), httpVersion),
tuple(Map.entry("ident", STRING), "-"),
tuple(Map.entry("rawrequest", STRING), null),
tuple(Map.entry("referrer", STRING), "\"" + referrer + "\""),
tuple(Map.entry("request", STRING), request),
tuple(Map.entry("timestamp", STRING), timestamp),
tuple(Map.entry("verb", STRING), verb),
List.of(tuple(Map.entry("response", STRING), "200"))
);
testApacheLog(
true,
tuple(Map.entry("user_agent.original", STRING), agent),
tuple(Map.entry("user.name", STRING), null),
tuple(Map.entry("http.response.body.bytes", LONG), 69849L),
tuple(Map.entry("source.address", STRING), clientIp),
tuple(Map.entry("http.version", STRING), httpVersion),
tuple(Map.entry("apache.access.user.identity", STRING), null),
tuple(Map.entry("http.response.status_code", INTEGER), 200),
tuple(Map.entry("http.request.referrer", STRING), referrer),
tuple(Map.entry("url.original", STRING), request),
tuple(Map.entry("timestamp", STRING), timestamp),
tuple(Map.entry("http.request.method", STRING), verb),
List.of()
);
}
public void testApacheLog(
boolean ecsCompatibility,
Tuple<Map.Entry<String, GrokCaptureType>, Object> agent,
Tuple<Map.Entry<String, GrokCaptureType>, Object> auth,
Tuple<Map.Entry<String, GrokCaptureType>, Object> bytes,
Tuple<Map.Entry<String, GrokCaptureType>, Object> clientIp,
Tuple<Map.Entry<String, GrokCaptureType>, Object> httpVersion,
Tuple<Map.Entry<String, GrokCaptureType>, Object> ident,
Tuple<Map.Entry<String, GrokCaptureType>, Object> rawRequest,
Tuple<Map.Entry<String, GrokCaptureType>, Object> referrer,
Tuple<Map.Entry<String, GrokCaptureType>, Object> request,
Tuple<Map.Entry<String, GrokCaptureType>, Object> timestamp,
Tuple<Map.Entry<String, GrokCaptureType>, Object> verb,
List<Tuple<Map.Entry<String, GrokCaptureType>, Object>> additionalFields
) {
String logLine = "31.184.238.164 - - [24/Jul/2014:05:35:37 +0530] \"GET /logs/access.log HTTP/1.0\" 200 69849 " +
"\"http://8rursodiol.enjin.com\" \"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) " +
"Chrome/30.0.1599.12785 YaBrowser/13.12.1599.12785 Safari/537.36\" \"www.dlwindianrailways.com\"";
Grok grok = new Grok(Grok.BUILTIN_PATTERNS, "%{COMBINEDAPACHELOG}", logger::warn);
assertCaptureConfig(
grok,
Map.ofEntries(
Map.entry("agent", STRING),
Map.entry("auth", STRING),
Map.entry("bytes", STRING),
Map.entry("clientip", STRING),
Map.entry("httpversion", STRING),
Map.entry("ident", STRING),
Map.entry("rawrequest", STRING),
Map.entry("referrer", STRING),
Map.entry("request", STRING),
Map.entry("response", STRING),
Map.entry("timestamp", STRING),
Map.entry("verb", STRING)
)
);
Grok grok = new Grok(Grok.getBuiltinPatterns(ecsCompatibility), "%{COMBINEDAPACHELOG}", logger::warn);
Map<String, GrokCaptureType> captureTypes = new HashMap<>();
captureTypes.put(agent.v1().getKey(), agent.v1().getValue());
captureTypes.put(auth.v1().getKey(), auth.v1().getValue());
captureTypes.put(bytes.v1().getKey(), bytes.v1().getValue());
captureTypes.put(clientIp.v1().getKey(), clientIp.v1().getValue());
captureTypes.put(httpVersion.v1().getKey(), httpVersion.v1().getValue());
captureTypes.put(ident.v1().getKey(), ident.v1().getValue());
captureTypes.put(rawRequest.v1().getKey(), rawRequest.v1().getValue());
captureTypes.put(referrer.v1().getKey(), referrer.v1().getValue());
captureTypes.put(request.v1().getKey(), request.v1().getValue());
captureTypes.put(timestamp.v1().getKey(), timestamp.v1().getValue());
captureTypes.put(verb.v1().getKey(), verb.v1().getValue());
for (var additionalField : additionalFields) {
captureTypes.put(additionalField.v1().getKey(), additionalField.v1().getValue());
}
assertCaptureConfig(grok, captureTypes);
Map<String, Object> matches = grok.captures(logLine);
assertEquals("31.184.238.164", matches.get("clientip"));
assertEquals("-", matches.get("ident"));
assertEquals("-", matches.get("auth"));
assertEquals("24/Jul/2014:05:35:37 +0530", matches.get("timestamp"));
assertEquals("GET", matches.get("verb"));
assertEquals("/logs/access.log", matches.get("request"));
assertEquals("1.0", matches.get("httpversion"));
assertEquals("200", matches.get("response"));
assertEquals("69849", matches.get("bytes"));
assertEquals("\"http://8rursodiol.enjin.com\"", matches.get("referrer"));
assertEquals(clientIp.v2(), matches.get(clientIp.v1().getKey()));
assertEquals(ident.v2(), matches.get(ident.v1().getKey()));
assertEquals(auth.v2(), matches.get(auth.v1().getKey()));
assertEquals(timestamp.v2(), matches.get(timestamp.v1().getKey()));
assertEquals(verb.v2(), matches.get(verb.v1().getKey()));
assertEquals(request.v2(), matches.get(request.v1().getKey()));
assertEquals(httpVersion.v2(), matches.get(httpVersion.v1().getKey()));
assertEquals(bytes.v2(), matches.get(bytes.v1().getKey()));
assertEquals(referrer.v2(), matches.get(referrer.v1().getKey()));
assertEquals(null, matches.get("port"));
assertEquals("\"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.12785 " +
"YaBrowser/13.12.1599.12785 Safari/537.36\"", matches.get("agent"));
assertEquals(agent.v2(), matches.get(agent.v1().getKey()));
assertEquals(rawRequest.v2(), matches.get(rawRequest.v1().getKey()));
for (var additionalField : additionalFields) {
assertEquals(additionalField.v2(), matches.get(additionalField.v1().getKey()));
}
}
public void testComplete() {
@ -587,6 +791,11 @@ public class GrokTests extends ESTestCase {
}
public void testExponentialExpressions() {
testExponentialExpressions(false);
testExponentialExpressions(true);
}
private void testExponentialExpressions(boolean ecsCompatibility) {
AtomicBoolean run = new AtomicBoolean(true); // to avoid a lingering thread when test has completed
String grokPattern = "Bonsuche mit folgender Anfrage: Belegart->\\[%{WORD:param2},(?<param5>(\\s*%{NOTSPACE})*)\\] " +
@ -606,8 +815,12 @@ public class GrokTests extends ESTestCase {
});
t.start();
};
Grok grok = new Grok(Grok.BUILTIN_PATTERNS, grokPattern, MatcherWatchdog.newInstance(10, 200, System::currentTimeMillis, scheduler),
logger::warn);
Grok grok = new Grok(
Grok.getBuiltinPatterns(ecsCompatibility),
grokPattern,
MatcherWatchdog.newInstance(10, 200, System::currentTimeMillis, scheduler),
logger::warn
);
Exception e = expectThrows(RuntimeException.class, () -> grok.captures(logLine));
run.set(false);
assertThat(e.getMessage(), equalTo("grok pattern matching was interrupted after [200] ms"));
@ -647,24 +860,44 @@ public class GrokTests extends ESTestCase {
}
public void testUnsupportedBracketsInFieldName() {
Grok grok = new Grok(Grok.BUILTIN_PATTERNS, "%{WORD:unsuppo(r)ted}", logger::warn);
testUnsupportedBracketsInFieldName(false);
testUnsupportedBracketsInFieldName(true);
}
private void testUnsupportedBracketsInFieldName(boolean ecsCompatibility) {
Grok grok = new Grok(Grok.getBuiltinPatterns(ecsCompatibility), "%{WORD:unsuppo(r)ted}", logger::warn);
Map<String, Object> matches = grok.captures("line");
assertNull(matches);
}
public void testJavaClassPatternWithUnderscore() {
Grok grok = new Grok(Grok.BUILTIN_PATTERNS, "%{JAVACLASS}", logger::warn);
testJavaClassPatternWithUnderscore(false);
testJavaClassPatternWithUnderscore(true);
}
private void testJavaClassPatternWithUnderscore(boolean ecsCompatibility) {
Grok grok = new Grok(Grok.getBuiltinPatterns(ecsCompatibility), "%{JAVACLASS}", logger::warn);
assertThat(grok.match("Test_Class.class"), is(true));
}
public void testJavaFilePatternWithSpaces() {
Grok grok = new Grok(Grok.BUILTIN_PATTERNS, "%{JAVAFILE}", logger::warn);
testJavaFilePatternWithSpaces(false);
testJavaFilePatternWithSpaces(true);
}
private void testJavaFilePatternWithSpaces(boolean ecsCompatibility) {
Grok grok = new Grok(Grok.getBuiltinPatterns(ecsCompatibility), "%{JAVAFILE}", logger::warn);
assertThat(grok.match("Test Class.java"), is(true));
}
public void testLogCallBack(){
public void testLogCallBack() {
testLogCallBack(false);
testLogCallBack(true);
}
private void testLogCallBack(boolean ecsCompatibility) {
AtomicReference<String> message = new AtomicReference<>();
Grok grok = new Grok(Grok.BUILTIN_PATTERNS, ".*\\[.*%{SPACE}*\\].*", message::set);
Grok grok = new Grok(Grok.getBuiltinPatterns(ecsCompatibility), ".*\\[.*%{SPACE}*\\].*", message::set);
grok.match("[foo]");
//this message comes from Joni, so updates to Joni may change the expectation
assertThat(message.get(), containsString("regular expression has redundant nested repeat operator"));
@ -672,16 +905,25 @@ public class GrokTests extends ESTestCase {
private void assertGrokedField(String fieldName) {
String line = "foo";
Grok grok = new Grok(Grok.BUILTIN_PATTERNS, "%{WORD:" + fieldName + "}", logger::warn);
Map<String, Object> matches = grok.captures(line);
assertEquals(line, matches.get(fieldName));
// test both with and without ECS compatibility
for (boolean ecsCompatibility : new boolean[]{false, true}) {
Grok grok = new Grok(Grok.getBuiltinPatterns(ecsCompatibility), "%{WORD:" + fieldName + "}", logger::warn);
Map<String, Object> matches = grok.captures(line);
assertEquals(line, matches.get(fieldName));
}
}
private void assertCaptureConfig(Grok grok, Map<String, GrokCaptureType> nameToType) {
assertCaptureConfig(grok, nameToType, List.of());
}
private void assertCaptureConfig(Grok grok, Map<String, GrokCaptureType> nameToType, List<String> acceptedDuplicates) {
Map<String, GrokCaptureType> fromGrok = new TreeMap<>();
for (GrokCaptureConfig config : grok.captureConfig()) {
Object old = fromGrok.put(config.name(), config.type());
assertThat("duplicates not allowed", old, nullValue());
if (acceptedDuplicates.contains(config.name()) == false) {
assertThat("duplicates not allowed", old, nullValue());
}
}
assertThat(fromGrok, equalTo(new TreeMap<>(nameToType)));
}

15
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java
View File

@ -26,6 +26,8 @@ import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationExcept
public final class GrokProcessor extends AbstractProcessor {
public static final String TYPE = "grok";
public static final String DEFAULT_ECS_COMPATIBILITY_MODE = Grok.ECS_COMPATIBILITY_MODES[0];
private static final String PATTERN_MATCH_KEY = "_ingest._grok_match_index";
private static final Logger logger = LogManager.getLogger(GrokProcessor.class);
@ -127,11 +129,9 @@ public final class GrokProcessor extends AbstractProcessor {
public static final class Factory implements Processor.Factory {
private final Map<String, String> builtinPatterns;
private final MatcherWatchdog matcherWatchdog;
public Factory(Map<String, String> builtinPatterns, MatcherWatchdog matcherWatchdog) {
this.builtinPatterns = builtinPatterns;
public Factory(MatcherWatchdog matcherWatchdog) {
this.matcherWatchdog = matcherWatchdog;
}
@ -142,12 +142,19 @@ public final class GrokProcessor extends AbstractProcessor {
List<String> matchPatterns = ConfigurationUtils.readList(TYPE, processorTag, config, "patterns");
boolean traceMatch = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "trace_match", false);
boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
String ecsCompatibility =
ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "ecs_compatibility", DEFAULT_ECS_COMPATIBILITY_MODE);
if (Grok.isValidEcsCompatibilityMode(ecsCompatibility) == false) {
throw newConfigurationException(TYPE, processorTag, "ecs_compatibility", "unsupported mode '" + ecsCompatibility + "'");
}
if (matchPatterns.isEmpty()) {
throw newConfigurationException(TYPE, processorTag, "patterns", "List of patterns must not be empty");
}
Map<String, String> customPatternBank = ConfigurationUtils.readOptionalMap(TYPE, processorTag, config, "pattern_definitions");
Map<String, String> patternBank = new HashMap<>(builtinPatterns);
Map<String, String> patternBank = new HashMap<>(
Grok.getBuiltinPatterns(ecsCompatibility)
);
if (customPatternBank != null) {
patternBank.putAll(customPatternBank);
}

48
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessorGetAction.java
View File

@ -7,6 +7,7 @@
*/
package org.elasticsearch.ingest.common;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.action.ActionRequestValidationException;
@ -46,14 +47,19 @@ public class GrokProcessorGetAction extends ActionType<GrokProcessorGetAction.Re
public static class Request extends ActionRequest {
private final boolean sorted;
private final String ecsCompatibility;
public Request(boolean sorted) {
public Request(boolean sorted, String ecsCompatibility) {
this.sorted = sorted;
this.ecsCompatibility = ecsCompatibility;
}
Request(StreamInput in) throws IOException {
super(in);
this.sorted = in.readBoolean();
this.ecsCompatibility = in.getVersion().onOrAfter(Version.V_8_0_0)
? in.readString()
: GrokProcessor.DEFAULT_ECS_COMPATIBILITY_MODE;
}
@Override
@ -65,11 +71,18 @@ public class GrokProcessorGetAction extends ActionType<GrokProcessorGetAction.Re
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeBoolean(sorted);
if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
out.writeString(ecsCompatibility);
}
}
public boolean sorted() {
return sorted;
}
public String getEcsCompatibility() {
return ecsCompatibility;
}
}
public static class Response extends ActionResponse implements ToXContentObject {
@ -105,25 +118,38 @@ public class GrokProcessorGetAction extends ActionType<GrokProcessorGetAction.Re
public static class TransportAction extends HandledTransportAction<Request, Response> {
private final Map<String, String> grokPatterns;
private final Map<String, String> sortedGrokPatterns;
private final Map<String, String> legacyGrokPatterns;
private final Map<String, String> sortedLegacyGrokPatterns;
private final Map<String, String> ecsV1GrokPatterns;
private final Map<String, String> sortedEcsV1GrokPatterns;
@Inject
public TransportAction(TransportService transportService, ActionFilters actionFilters) {
this(transportService, actionFilters, Grok.BUILTIN_PATTERNS);
this(transportService, actionFilters, Grok.getBuiltinPatterns(false), Grok.getBuiltinPatterns(true));
}
// visible for testing
TransportAction(TransportService transportService, ActionFilters actionFilters, Map<String, String> grokPatterns) {
TransportAction(
TransportService transportService,
ActionFilters actionFilters,
Map<String, String> legacyGrokPatterns,
Map<String, String> ecsV1GrokPatterns) {
super(NAME, transportService, actionFilters, Request::new);
this.grokPatterns = grokPatterns;
this.sortedGrokPatterns = new TreeMap<>(this.grokPatterns);
this.legacyGrokPatterns = legacyGrokPatterns;
this.sortedLegacyGrokPatterns = new TreeMap<>(this.legacyGrokPatterns);
this.ecsV1GrokPatterns = ecsV1GrokPatterns;
this.sortedEcsV1GrokPatterns = new TreeMap<>(this.ecsV1GrokPatterns);
}
@Override
protected void doExecute(Task task, Request request, ActionListener<Response> listener) {
try {
listener.onResponse(new Response(request.sorted() ? sortedGrokPatterns : grokPatterns));
listener.onResponse(new Response(
request.getEcsCompatibility().equals(Grok.ECS_COMPATIBILITY_MODES[0])
? request.sorted() ? sortedLegacyGrokPatterns : legacyGrokPatterns
: request.sorted() ? sortedEcsV1GrokPatterns : ecsV1GrokPatterns
)
);
} catch (Exception e) {
listener.onFailure(e);
}
@ -145,7 +171,11 @@ public class GrokProcessorGetAction extends ActionType<GrokProcessorGetAction.Re
@Override
protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) {
boolean sorted = request.paramAsBoolean("s", false);
Request grokPatternsRequest = new Request(sorted);
String ecsCompatibility = request.param("ecs_compatibility", GrokProcessor.DEFAULT_ECS_COMPATIBILITY_MODE);
if (Grok.isValidEcsCompatibilityMode(ecsCompatibility) == false) {
throw new IllegalArgumentException("unsupported ECS compatibility mode [" + ecsCompatibility + "]");
}
Request grokPatternsRequest = new Request(sorted, ecsCompatibility);
return channel -> client.executeLocally(INSTANCE, grokPatternsRequest, new RestToXContentListener<>(channel));
}
}

3
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java
View File

@ -18,7 +18,6 @@ import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.SettingsFilter;
import org.elasticsearch.core.TimeValue;
import org.elasticsearch.grok.Grok;
import org.elasticsearch.grok.MatcherWatchdog;
import org.elasticsearch.ingest.DropProcessor;
import org.elasticsearch.ingest.PipelineProcessor;
@ -66,7 +65,7 @@ public class IngestCommonPlugin extends Plugin implements ActionPlugin, IngestPl
entry(ForEachProcessor.TYPE, new ForEachProcessor.Factory(parameters.scriptService)),
entry(DateIndexNameProcessor.TYPE, new DateIndexNameProcessor.Factory(parameters.scriptService)),
entry(SortProcessor.TYPE, new SortProcessor.Factory()),
entry(GrokProcessor.TYPE, new GrokProcessor.Factory(Grok.BUILTIN_PATTERNS, createGrokThreadWatchdog(parameters))),
entry(GrokProcessor.TYPE, new GrokProcessor.Factory(createGrokThreadWatchdog(parameters))),
entry(ScriptProcessor.TYPE, new ScriptProcessor.Factory(parameters.scriptService)),
entry(DotExpanderProcessor.TYPE, new DotExpanderProcessor.Factory()),
entry(JsonProcessor.TYPE, new JsonProcessor.Factory()),

27
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorFactoryTests.java
View File

@ -23,7 +23,7 @@ import static org.hamcrest.Matchers.notNullValue;
public class GrokProcessorFactoryTests extends ESTestCase {
public void testBuild() throws Exception {
GrokProcessor.Factory factory = new GrokProcessor.Factory(Collections.emptyMap(), MatcherWatchdog.noop());
GrokProcessor.Factory factory = new GrokProcessor.Factory(MatcherWatchdog.noop());
Map<String, Object> config = new HashMap<>();
config.put("field", "_field");
@ -37,7 +37,7 @@ public class GrokProcessorFactoryTests extends ESTestCase {
}
public void testBuildWithIgnoreMissing() throws Exception {
GrokProcessor.Factory factory = new GrokProcessor.Factory(Collections.emptyMap(), MatcherWatchdog.noop());
GrokProcessor.Factory factory = new GrokProcessor.Factory(MatcherWatchdog.noop());
Map<String, Object> config = new HashMap<>();
config.put("field", "_field");
@ -52,7 +52,7 @@ public class GrokProcessorFactoryTests extends ESTestCase {
}
public void testBuildMissingField() throws Exception {
GrokProcessor.Factory factory = new GrokProcessor.Factory(Collections.emptyMap(), MatcherWatchdog.noop());
GrokProcessor.Factory factory = new GrokProcessor.Factory(MatcherWatchdog.noop());
Map<String, Object> config = new HashMap<>();
config.put("patterns", Collections.singletonList("(?<foo>\\w+)"));
ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config));
@ -60,7 +60,7 @@ public class GrokProcessorFactoryTests extends ESTestCase {
}
public void testBuildMissingPatterns() throws Exception {
GrokProcessor.Factory factory = new GrokProcessor.Factory(Collections.emptyMap(), MatcherWatchdog.noop());
GrokProcessor.Factory factory = new GrokProcessor.Factory(MatcherWatchdog.noop());
Map<String, Object> config = new HashMap<>();
config.put("field", "foo");
ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config));
@ -68,7 +68,7 @@ public class GrokProcessorFactoryTests extends ESTestCase {
}
public void testBuildEmptyPatternsList() throws Exception {
GrokProcessor.Factory factory = new GrokProcessor.Factory(Collections.emptyMap(), MatcherWatchdog.noop());
GrokProcessor.Factory factory = new GrokProcessor.Factory(MatcherWatchdog.noop());
Map<String, Object> config = new HashMap<>();
config.put("field", "foo");
config.put("patterns", Collections.emptyList());
@ -77,7 +77,7 @@ public class GrokProcessorFactoryTests extends ESTestCase {
}
public void testCreateWithCustomPatterns() throws Exception {
GrokProcessor.Factory factory = new GrokProcessor.Factory(Collections.emptyMap(), MatcherWatchdog.noop());
GrokProcessor.Factory factory = new GrokProcessor.Factory(MatcherWatchdog.noop());
Map<String, Object> config = new HashMap<>();
config.put("field", "_field");
@ -90,7 +90,7 @@ public class GrokProcessorFactoryTests extends ESTestCase {
}
public void testCreateWithInvalidPattern() throws Exception {
GrokProcessor.Factory factory = new GrokProcessor.Factory(Collections.emptyMap(), MatcherWatchdog.noop());
GrokProcessor.Factory factory = new GrokProcessor.Factory(MatcherWatchdog.noop());
Map<String, Object> config = new HashMap<>();
config.put("field", "_field");
config.put("patterns", Collections.singletonList("["));
@ -99,7 +99,7 @@ public class GrokProcessorFactoryTests extends ESTestCase {
}
public void testCreateWithInvalidPatternDefinition() throws Exception {
GrokProcessor.Factory factory = new GrokProcessor.Factory(Collections.emptyMap(), MatcherWatchdog.noop());
GrokProcessor.Factory factory = new GrokProcessor.Factory(MatcherWatchdog.noop());
Map<String, Object> config = new HashMap<>();
config.put("field", "_field");
config.put("patterns", Collections.singletonList("%{MY_PATTERN:name}!"));
@ -108,4 +108,15 @@ public class GrokProcessorFactoryTests extends ESTestCase {
assertThat(e.getMessage(),
equalTo("[patterns] Invalid regex pattern found in: [%{MY_PATTERN:name}!]. premature end of char-class"));
}
public void testCreateWithInvalidEcsCompatibilityMode() throws Exception {
GrokProcessor.Factory factory = new GrokProcessor.Factory(MatcherWatchdog.noop());
Map<String, Object> config = new HashMap<>();
config.put("field", "_field");
config.put("patterns", Collections.singletonList("(?<foo>\\w+)"));
String invalidEcsMode = randomAlphaOfLength(3);
config.put("ecs_compatibility", invalidEcsMode);
ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config));
assertThat(e.getMessage(), equalTo("[ecs_compatibility] unsupported mode '" + invalidEcsMode + "'"));
}
}

83
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorGetActionTests.java
View File

@ -17,6 +17,7 @@ import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.grok.Grok;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.transport.TransportService;
@ -33,10 +34,11 @@ import static org.hamcrest.core.IsNull.nullValue;
import static org.mockito.Mockito.mock;
public class GrokProcessorGetActionTests extends ESTestCase {
private static final Map<String, String> TEST_PATTERNS = Map.of("PATTERN2", "foo2", "PATTERN1", "foo1");
private static final Map<String, String> LEGACY_TEST_PATTERNS = Map.of("PATTERN2", "foo2", "PATTERN1", "foo1");
private static final Map<String, String> ECS_TEST_PATTERNS = Map.of("ECS_PATTERN2", "foo2", "ECS_PATTERN1", "foo1");
public void testRequest() throws Exception {
GrokProcessorGetAction.Request request = new GrokProcessorGetAction.Request(false);
GrokProcessorGetAction.Request request = new GrokProcessorGetAction.Request(false, GrokProcessor.DEFAULT_ECS_COMPATIBILITY_MODE);
BytesStreamOutput out = new BytesStreamOutput();
request.writeTo(out);
StreamInput streamInput = out.bytes().streamInput();
@ -45,55 +47,96 @@ public class GrokProcessorGetActionTests extends ESTestCase {
}
public void testResponseSerialization() throws Exception {
GrokProcessorGetAction.Response response = new GrokProcessorGetAction.Response(TEST_PATTERNS);
GrokProcessorGetAction.Response response = new GrokProcessorGetAction.Response(LEGACY_TEST_PATTERNS);
BytesStreamOutput out = new BytesStreamOutput();
response.writeTo(out);
StreamInput streamInput = out.bytes().streamInput();
GrokProcessorGetAction.Response otherResponse = new GrokProcessorGetAction.Response(streamInput);
assertThat(response.getGrokPatterns(), equalTo(TEST_PATTERNS));
assertThat(response.getGrokPatterns(), equalTo(LEGACY_TEST_PATTERNS));
assertThat(response.getGrokPatterns(), equalTo(otherResponse.getGrokPatterns()));
}
public void testResponseSorting() {
List<String> sortedKeys = new ArrayList<>(TEST_PATTERNS.keySet());
List<String> sortedKeys = new ArrayList<>(LEGACY_TEST_PATTERNS.keySet());
Collections.sort(sortedKeys);
GrokProcessorGetAction.TransportAction transportAction =
new GrokProcessorGetAction.TransportAction(mock(TransportService.class), mock(ActionFilters.class), TEST_PATTERNS);
GrokProcessorGetAction.TransportAction transportAction = new GrokProcessorGetAction.TransportAction(
mock(TransportService.class),
mock(ActionFilters.class),
LEGACY_TEST_PATTERNS,
ECS_TEST_PATTERNS
);
GrokProcessorGetAction.Response[] receivedResponse = new GrokProcessorGetAction.Response[1];
transportAction.doExecute(null, new GrokProcessorGetAction.Request(true), new ActionListener<>() {
@Override
public void onResponse(GrokProcessorGetAction.Response response) {
transportAction.doExecute(
null,
new GrokProcessorGetAction.Request(true, GrokProcessor.DEFAULT_ECS_COMPATIBILITY_MODE),
new ActionListener<>() {
@Override
public void onResponse(GrokProcessorGetAction.Response response) {
receivedResponse[0] = response;
}
@Override
public void onFailure(Exception e) {
@Override
public void onFailure(Exception e) {
fail();
}
});
}
);
assertThat(receivedResponse[0], notNullValue());
assertThat(receivedResponse[0].getGrokPatterns().keySet().toArray(), equalTo(sortedKeys.toArray()));
GrokProcessorGetAction.Response firstResponse = receivedResponse[0];
transportAction.doExecute(null, new GrokProcessorGetAction.Request(true), new ActionListener<>() {
@Override
public void onResponse(GrokProcessorGetAction.Response response) {
transportAction.doExecute(
null,
new GrokProcessorGetAction.Request(true, GrokProcessor.DEFAULT_ECS_COMPATIBILITY_MODE),
new ActionListener<>() {
@Override
public void onResponse(GrokProcessorGetAction.Response response) {
receivedResponse[0] = response;
}
@Override
public void onFailure(Exception e) {
@Override
public void onFailure(Exception e) {
fail();
}
});
}
);
assertThat(receivedResponse[0], notNullValue());
assertThat(receivedResponse[0], not(sameInstance(firstResponse)));
assertThat(receivedResponse[0].getGrokPatterns(), sameInstance(firstResponse.getGrokPatterns()));
}
public void testEcsCompatibilityMode() {
List<String> sortedKeys = new ArrayList<>(ECS_TEST_PATTERNS.keySet());
Collections.sort(sortedKeys);
GrokProcessorGetAction.TransportAction transportAction = new GrokProcessorGetAction.TransportAction(
mock(TransportService.class),
mock(ActionFilters.class),
LEGACY_TEST_PATTERNS,
ECS_TEST_PATTERNS
);
GrokProcessorGetAction.Response[] receivedResponse = new GrokProcessorGetAction.Response[1];
transportAction.doExecute(
null,
new GrokProcessorGetAction.Request(true, Grok.ECS_COMPATIBILITY_MODES[1]),
new ActionListener<>() {
@Override
public void onResponse(GrokProcessorGetAction.Response response) {
receivedResponse[0] = response;
}
@Override
public void onFailure(Exception e) {
fail();
}
}
);
assertThat(receivedResponse[0], notNullValue());
assertThat(receivedResponse[0].getGrokPatterns().keySet().toArray(), equalTo(sortedKeys.toArray()));
}
@SuppressWarnings("unchecked")
public void testResponseToXContent() throws Exception {
GrokProcessorGetAction.Response response = new GrokProcessorGetAction.Response(TEST_PATTERNS);
GrokProcessorGetAction.Response response = new GrokProcessorGetAction.Response(LEGACY_TEST_PATTERNS);
try (XContentBuilder builder = JsonXContent.contentBuilder()) {
response.toXContent(builder, ToXContent.EMPTY_PARAMS);
Map<String, Object> converted = XContentHelper.convertToMap(BytesReference.bytes(builder), false, builder.contentType()).v2();

4
modules/runtime-fields-common/src/main/java/org/elasticsearch/runtimefields/NamedGroupExtractor.java
View File

@ -99,13 +99,13 @@ public interface NamedGroupExtractor {
try {
// Try to collect warnings up front and refuse to compile the expression if there are any
List<String> warnings = new ArrayList<>();
new Grok(Grok.BUILTIN_PATTERNS, pattern, watchdog, warnings::add).match("__nomatch__");
new Grok(Grok.getBuiltinPatterns(false), pattern, watchdog, warnings::add).match("__nomatch__");
if (false == warnings.isEmpty()) {
throw new IllegalArgumentException("emitted warnings: " + warnings);
}
return new Grok(
Grok.BUILTIN_PATTERNS,
Grok.getBuiltinPatterns(false),
pattern,
watchdog,
w -> { throw new IllegalArgumentException("grok [" + pattern + "] emitted a warning: " + w); }

57
qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/20_combine_processors.yml
View File

@ -67,6 +67,63 @@
- match: { _source.geoip.continent_name: "North America" }
- match: { _source.geoip.country_iso_code: "US" }
---
"Test with date processor and ECS-v1":
- do:
ingest.put_pipeline:
id: "_id"
body: >
{
"processors": [
{
"grok" : {
"field" : "log",
"ecs_compatibility": "v1",
"patterns": ["%{COMBINEDAPACHELOG}"]
}
},
{
"date" : {
"field" : "timestamp",
"target_field" : "timestamp",
"formats" : ["dd/MMM/yyyy:HH:mm:ss xx"]
}
},
{
"geoip" : {
"field" : "source.address"
}
}
]
}
- match: { acknowledged: true }
- do:
index:
index: test
id: 1
pipeline: "_id"
body: {
log: "70.193.17.92 - - [08/Sep/2014:02:54:42 +0000] \"GET /presentations/logstash-scale11x/images/ahhh___rage_face_by_samusmmx-d5g5zap.png HTTP/1.1\" 200 175208 \"http://mobile.rivals.com/board_posts.asp?SID=880&mid=198829575&fid=2208&tid=198829575&Team=&TeamId=&SiteId=\" \"Mozilla/5.0 (Linux; Android 4.2.2; VS980 4G Build/JDQ39B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.135 Mobile Safari/537.36\""
}
- do:
get:
index: test
id: 1
- length: { _source: 7 }
- match: { _source.url.original: "/presentations/logstash-scale11x/images/ahhh___rage_face_by_samusmmx-d5g5zap.png" }
- match: { _source.user_agent.original: "Mozilla/5.0 (Linux; Android 4.2.2; VS980 4G Build/JDQ39B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.135 Mobile Safari/537.36" }
- match: { _source.http.request.method: "GET" }
- match: { _source.http.request.referrer: "http://mobile.rivals.com/board_posts.asp?SID=880&mid=198829575&fid=2208&tid=198829575&Team=&TeamId=&SiteId=" }
- match: { _source.http.response.status_code: 200 }
- match: { _source.http.response.body.bytes: 175208 }
- match: { _source.source.address: "70.193.17.92" }
- match: { _source.http.version: "1.1" }
- match: { _source.timestamp: "2014-09-08T02:54:42.000Z" }
- match: { _source.geoip.continent_name: "North America" }
- match: { _source.geoip.country_iso_code: "US" }
---
"Test mutate":
- do:

2
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/categorization/GrokPatternCreator.java
View File

@ -280,7 +280,7 @@ public final class GrokPatternCreator {
GrokPatternCandidate(String grokPatternName, String fieldName, String preBreak, String postBreak) {
this.grokPatternName = grokPatternName;
this.fieldName = fieldName;
this.grok = new Grok(Grok.BUILTIN_PATTERNS, "%{DATA:" + PREFACE + "}" + preBreak + "%{" + grokPatternName + ":this}" +
this.grok = new Grok(Grok.getBuiltinPatterns(false), "%{DATA:" + PREFACE + "}" + preBreak + "%{" + grokPatternName + ":this}" +
postBreak + "%{GREEDYDATA:" + EPILOGUE + "}", logger::warn);
}
}

14
x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/structurefinder/GrokPatternCreator.java
View File

@ -156,9 +156,9 @@ public final class GrokPatternCreator {
this.mappings = mappings;
this.fieldStats = fieldStats;
if (customGrokPatternDefinitions.isEmpty()) {
grokPatternDefinitions = Grok.BUILTIN_PATTERNS;
grokPatternDefinitions = Grok.getBuiltinPatterns(false);
} else {
grokPatternDefinitions = new HashMap<>(Grok.BUILTIN_PATTERNS);
grokPatternDefinitions = new HashMap<>(Grok.getBuiltinPatterns(false));
grokPatternDefinitions.putAll(customGrokPatternDefinitions);
}
this.timeoutChecker = Objects.requireNonNull(timeoutChecker);
@ -509,7 +509,7 @@ public final class GrokPatternCreator {
fieldName,
"\\b",
"\\b",
Grok.BUILTIN_PATTERNS
Grok.getBuiltinPatterns(false)
);
}
@ -549,7 +549,7 @@ public final class GrokPatternCreator {
fieldName,
preBreak,
postBreak,
Grok.BUILTIN_PATTERNS
Grok.getBuiltinPatterns(false)
);
}
@ -699,7 +699,7 @@ public final class GrokPatternCreator {
throw new IllegalStateException("Cannot process KV matches until a field name has been determined");
}
Grok grok = new Grok(
Grok.BUILTIN_PATTERNS,
Grok.getBuiltinPatterns(false),
"(?m)%{DATA:" + PREFACE + "}\\b" + fieldName + "=%{USER:" + VALUE + "}%{GREEDYDATA:" + EPILOGUE + "}",
TimeoutChecker.watchdog,
logger::warn
@ -768,7 +768,7 @@ public final class GrokPatternCreator {
private final Grok grok;
static FullMatchGrokPatternCandidate fromGrokPatternName(String grokPatternName, String timeField) {
return new FullMatchGrokPatternCandidate("%{" + grokPatternName + "}", timeField, Grok.BUILTIN_PATTERNS);
return new FullMatchGrokPatternCandidate("%{" + grokPatternName + "}", timeField, Grok.getBuiltinPatterns(false));
}
static FullMatchGrokPatternCandidate fromGrokPatternName(
@ -780,7 +780,7 @@ public final class GrokPatternCreator {
}
static FullMatchGrokPatternCandidate fromGrokPattern(String grokPattern, String timeField) {
return new FullMatchGrokPatternCandidate(grokPattern, timeField, Grok.BUILTIN_PATTERNS);
return new FullMatchGrokPatternCandidate(grokPattern, timeField, Grok.getBuiltinPatterns(false));
}
static FullMatchGrokPatternCandidate fromGrokPattern(

6
x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/structurefinder/TextStructureUtils.java
View File

@ -59,19 +59,19 @@ public final class TextStructureUtils {
"(?:%{WKT_POINT}|%{WKT_LINESTRING}|%{WKT_MULTIPOINT}|%{WKT_POLYGON}|%{WKT_MULTILINESTRING}|%{WKT_MULTIPOLYGON}|%{WKT_BBOX})"
);
patterns.put("WKT_GEOMETRYCOLLECTION", "GEOMETRYCOLLECTION \\(%{WKT_ANY}(?:, %{WKT_ANY})\\)");
patterns.putAll(Grok.BUILTIN_PATTERNS);
patterns.putAll(Grok.getBuiltinPatterns(false));
EXTENDED_PATTERNS = Collections.unmodifiableMap(patterns);
}
private static final int NUM_TOP_HITS = 10;
// NUMBER Grok pattern doesn't support scientific notation, so we extend it
private static final Grok NUMBER_GROK = new Grok(
Grok.BUILTIN_PATTERNS,
Grok.getBuiltinPatterns(false),
"^%{NUMBER}(?:[eE][+-]?[0-3]?[0-9]{1,2})?$",
TimeoutChecker.watchdog,
logger::warn
);
private static final Grok IP_GROK = new Grok(Grok.BUILTIN_PATTERNS, "^%{IP}$", TimeoutChecker.watchdog, logger::warn);
private static final Grok IP_GROK = new Grok(Grok.getBuiltinPatterns(false), "^%{IP}$", TimeoutChecker.watchdog, logger::warn);
private static final Grok GEO_POINT_WKT = new Grok(EXTENDED_PATTERNS, "^%{WKT_POINT}$", TimeoutChecker.watchdog, logger::warn);
private static final Grok GEO_WKT = new Grok(
EXTENDED_PATTERNS,

4
x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/structurefinder/TimestampFormatFinder.java
View File

@ -1706,13 +1706,13 @@ public final class TimestampFormatFinder {
this.strictGrokPattern = Objects.requireNonNull(strictGrokPattern);
// The (?m) here has the Ruby meaning, which is equivalent to (?s) in Java
this.strictSearchGrok = new Grok(
Grok.BUILTIN_PATTERNS,
Grok.getBuiltinPatterns(false),
"(?m)%{DATA:" + PREFACE + "}" + strictGrokPattern + "%{GREEDYDATA:" + EPILOGUE + "}",
TimeoutChecker.watchdog,
logger::warn
);
this.strictFullMatchGrok = new Grok(
Grok.BUILTIN_PATTERNS,
Grok.getBuiltinPatterns(false),
"^" + strictGrokPattern + "$",
TimeoutChecker.watchdog,
logger::warn

2
x-pack/plugin/text-structure/src/test/java/org/elasticsearch/xpack/textstructure/structurefinder/TimeoutCheckerTests.java
View File

@ -81,7 +81,7 @@ public class TimeoutCheckerTests extends TextStructureTestCase {
}
public void testGrokCaptures() throws Exception {
Grok grok = new Grok(Grok.BUILTIN_PATTERNS, "{%DATA:data}{%GREEDYDATA:greedydata}", TimeoutChecker.watchdog, logger::warn);
Grok grok = new Grok(Grok.getBuiltinPatterns(false), "{%DATA:data}{%GREEDYDATA:greedydata}", TimeoutChecker.watchdog, logger::warn);
TimeValue timeout = TimeValue.timeValueMillis(1);
try (TimeoutChecker timeoutChecker = new TimeoutChecker("grok captures test", timeout, scheduler)) {