Extract device type from user agent info (#69322)

This commit is contained in:
Shahzad 2021-03-29 23:34:53 +02:00 committed by GitHub
parent 06d5f9487a
commit f7efa3eaba
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 1044 additions and 17 deletions

View File

@ -53,7 +53,7 @@ the processors as follows:
[options="header"]
|====
| Processor type | Field | Additional options | Description
| Processor type | Field | Additional options | Description
| <<date-processor,**Date**>>
| `@timestamp`
@ -247,7 +247,8 @@ The API returns:
},
"name": "Chrome",
"device": {
"name": "Mac"
"name": "Mac",
"type": "Desktop"
},
"version": "52.0.2743.116"
}

View File

@ -69,7 +69,8 @@ Which returns
"full": "Mac OS X 10.10.5"
},
"device" : {
"name" : "Mac"
"name" : "Mac",
"type" : "Desktop"
},
}
}

View File

@ -0,0 +1,179 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
package org.elasticsearch.ingest.useragent;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static org.elasticsearch.ingest.useragent.UserAgentParser.readParserConfigurations;
import static org.elasticsearch.ingest.useragent.UserAgentParser.VersionedName;
public class DeviceTypeParser {
private static final String OS_PARSERS = "os_parsers";
private static final String BROWSER_PARSER = "browser_parsers";
private static final String DEVICE_PARSER = "device_parsers";
private static final String AGENT_STRING_PARSER = "agent_string_parsers";
private static final String robot = "Robot", tablet = "Tablet", desktop = "Desktop", phone = "Phone";
private final List<String> patternListKeys = List.of(OS_PARSERS, BROWSER_PARSER, DEVICE_PARSER, AGENT_STRING_PARSER);
private final HashMap<String, ArrayList<DeviceTypeSubPattern>> deviceTypePatterns = new HashMap<>();
public void init(InputStream regexStream) throws IOException {
// EMPTY is safe here because we don't use namedObject
XContentParser yamlParser = XContentFactory.xContent(XContentType.YAML).createParser(NamedXContentRegistry.EMPTY,
LoggingDeprecationHandler.INSTANCE, regexStream);
XContentParser.Token token = yamlParser.nextToken();
if (token == XContentParser.Token.START_OBJECT) {
token = yamlParser.nextToken();
for (; token != null; token = yamlParser.nextToken()) {
String currentName = yamlParser.currentName();
if (token == XContentParser.Token.FIELD_NAME && patternListKeys.contains(currentName)) {
List<Map<String, String>> parserConfigurations = readParserConfigurations(yamlParser);
ArrayList<DeviceTypeSubPattern> subPatterns = new ArrayList<>();
for (Map<String, String> map : parserConfigurations) {
subPatterns.add(new DeviceTypeSubPattern(Pattern.compile((map.get("regex"))),
map.get("replacement")));
}
deviceTypePatterns.put(currentName, subPatterns);
}
}
}
if (patternListKeys.size() != deviceTypePatterns.size()) {
throw new ElasticsearchParseException("not a valid regular expression file");
}
}
public String findDeviceType(String agentString, VersionedName userAgent, VersionedName os, VersionedName device) {
if (deviceTypePatterns.isEmpty()) {
return null;
}
if (agentString != null) {
String deviceType = findMatch(deviceTypePatterns.get(AGENT_STRING_PARSER), agentString);
if (deviceType != null) {
return deviceType;
}
}
return findDeviceType(userAgent, os, device);
}
public String findDeviceType(VersionedName userAgent, VersionedName os, VersionedName device) {
if (deviceTypePatterns.isEmpty()) {
return null;
}
ArrayList<String> extractedDeviceTypes = new ArrayList<>();
for (String patternKey : patternListKeys) {
String deviceType = null;
switch (patternKey) {
case OS_PARSERS:
if (os != null && os.name != null) {
deviceType = findMatch(deviceTypePatterns.get(patternKey), os.name);
}
break;
case BROWSER_PARSER:
if (userAgent != null && userAgent.name != null) {
deviceType = findMatch(deviceTypePatterns.get(patternKey), userAgent.name);
}
break;
case DEVICE_PARSER:
if (device != null && device.name != null) {
deviceType = findMatch(deviceTypePatterns.get(patternKey), device.name);
}
break;
default:
break;
}
if (deviceType != null) {
extractedDeviceTypes.add(deviceType);
}
}
if (extractedDeviceTypes.contains(robot)) {
return robot;
}
if (extractedDeviceTypes.contains(tablet)) {
return tablet;
}
if (extractedDeviceTypes.contains(phone)) {
return phone;
}
if (extractedDeviceTypes.contains(desktop)) {
return desktop;
}
return "Other";
}
private String findMatch(List<DeviceTypeSubPattern> possiblePatterns, String matchString) {
String name;
for (DeviceTypeSubPattern pattern : possiblePatterns) {
name = pattern.match(matchString);
if (name != null) {
return name;
}
}
return null;
}
static final class DeviceTypeSubPattern {
private final Pattern pattern;
private final String nameReplacement;
DeviceTypeSubPattern(Pattern pattern, String nameReplacement) {
this.pattern = pattern;
this.nameReplacement = nameReplacement;
}
public String match(String matchString) {
String name = null;
Matcher matcher = pattern.matcher(matchString);
if (matcher.find() == false) {
return null;
}
int groupCount = matcher.groupCount();
if (nameReplacement != null) {
if (nameReplacement.contains("$1") && groupCount >= 1 && matcher.group(1) != null) {
name = nameReplacement.replaceFirst("\\$1", Matcher.quoteReplacement(matcher.group(1)));
} else {
name = nameReplacement;
}
}
return name;
}
}
}

View File

@ -55,7 +55,8 @@ public class IngestUserAgentPlugin extends Plugin implements IngestPlugin {
Map<String, UserAgentParser> userAgentParsers = new HashMap<>();
UserAgentParser defaultParser = new UserAgentParser(DEFAULT_PARSER_NAME,
IngestUserAgentPlugin.class.getResourceAsStream("/regexes.yml"), cache);
IngestUserAgentPlugin.class.getResourceAsStream("/regexes.yml"),
IngestUserAgentPlugin.class.getResourceAsStream("/device_type_regexes.yml"), cache);
userAgentParsers.put(DEFAULT_PARSER_NAME, defaultParser);
if (Files.exists(userAgentConfigDirectory) && Files.isDirectory(userAgentConfigDirectory)) {
@ -66,8 +67,9 @@ public class IngestUserAgentPlugin extends Plugin implements IngestPlugin {
Iterable<Path> iterable = regexFiles::iterator;
for (Path path : iterable) {
String parserName = path.getFileName().toString();
try (InputStream regexStream = Files.newInputStream(path, StandardOpenOption.READ)) {
userAgentParsers.put(parserName, new UserAgentParser(parserName, regexStream, cache));
try (InputStream regexStream = Files.newInputStream(path, StandardOpenOption.READ);
InputStream deviceTypeRegexStream = IngestUserAgentPlugin.class.getResourceAsStream("/device_type_regexes.yml")) {
userAgentParsers.put(parserName, new UserAgentParser(parserName, regexStream, deviceTypeRegexStream, cache));
}
}
}

View File

@ -26,17 +26,21 @@ import java.util.regex.Pattern;
final class UserAgentParser {
private final UserAgentCache cache;
private final DeviceTypeParser deviceTypeParser = new DeviceTypeParser();
private final List<UserAgentSubpattern> uaPatterns = new ArrayList<>();
private final List<UserAgentSubpattern> osPatterns = new ArrayList<>();
private final List<UserAgentSubpattern> devicePatterns = new ArrayList<>();
private final String name;
UserAgentParser(String name, InputStream regexStream, UserAgentCache cache) {
UserAgentParser(String name, InputStream regexStream, InputStream deviceTypeRegexStream, UserAgentCache cache) {
this.name = name;
this.cache = cache;
try {
init(regexStream);
if (deviceTypeRegexStream != null) {
deviceTypeParser.init(deviceTypeRegexStream);
}
} catch (IOException e) {
throw new ElasticsearchParseException("error parsing regular expression file", e);
}
@ -96,8 +100,8 @@ final class UserAgentParser {
}
}
private List<Map<String, String>> readParserConfigurations(XContentParser yamlParser) throws IOException {
List <Map<String, String>> patternList = new ArrayList<>();
static List<Map<String, String>> readParserConfigurations(XContentParser yamlParser) throws IOException {
List<Map<String, String>> patternList = new ArrayList<>();
XContentParser.Token token = yamlParser.nextToken();
if (token != XContentParser.Token.START_ARRAY) {
@ -156,9 +160,8 @@ final class UserAgentParser {
VersionedName userAgent = findMatch(uaPatterns, agentString);
VersionedName operatingSystem = findMatch(osPatterns, agentString);
VersionedName device = findMatch(devicePatterns, agentString);
details = new Details(userAgent, operatingSystem, device);
String deviceType = deviceTypeParser.findDeviceType(agentString, userAgent, operatingSystem, device);
details = new Details(userAgent, operatingSystem, device, deviceType);
cache.put(name, agentString, details);
}
@ -182,11 +185,13 @@ final class UserAgentParser {
public final VersionedName userAgent;
public final VersionedName operatingSystem;
public final VersionedName device;
public final String deviceType;
Details(VersionedName userAgent, VersionedName operatingSystem, VersionedName device) {
Details(VersionedName userAgent, VersionedName operatingSystem, VersionedName device, String deviceType) {
this.userAgent = userAgent;
this.operatingSystem = operatingSystem;
this.device = device;
this.deviceType = deviceType;
}
}

View File

@ -125,8 +125,14 @@ public class UserAgentProcessor extends AbstractProcessor {
Map<String, String> deviceDetails = new HashMap<>(1);
if (uaClient.device != null && uaClient.device.name != null) {
deviceDetails.put("name", uaClient.device.name);
deviceDetails.put("type", uaClient.deviceType);
} else {
deviceDetails.put("name", "Other");
if (uaClient.deviceType != null) {
deviceDetails.put("type", uaClient.deviceType);
} else {
deviceDetails.put("type", "Other");
}
}
uaDetails.put("device", deviceDetails);
break;

View File

@ -0,0 +1,67 @@
# Apache License, Version 2.0
# ===========================
#
# Copyright 2009 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
## Custom parser being added to support device types
os_parsers:
# Robot
- regex: 'Bot|bot|spider|Spider|Crawler|crawler|AppEngine-Google'
replacement: 'Robot'
# Desktop OS, Most Common
- regex: '^(Windows$|Windows NT$|Mac OS X|Linux$|Chrome OS|Fedora$|Ubuntu$)'
replacement: 'Desktop'
# Phone OS
- regex: '^(Android$|iOS|Windows Phone|Firefox OS|BlackBerry OS|KaiOS|Sailfish$|Maemo)'
replacement: 'Phone'
# Desktop OS, Not Common
- regex: '^(Windows XP|Windows 7|Windows 10|FreeBSD|OpenBSD|Arch Linux|Solaris|NetBSD|SUSE|SunOS|BeOS\/Haiku)'
replacement: 'Desktop'
- regex: 'Tablet|BlackBerry Tablet OS|iPad|FireOS|Crosswalk'
replacement: 'Tablet'
browser_parsers:
# Robot
- regex: 'Bot|bot|spider|Spider|Crawler|crawler|AppEngine-Google'
replacement: 'Robot'
# Desktop Browsers
- regex: '^(Chrome$|Chromium$|Edge$|Firefox$|IE$|Maxthon$|Opera$|Safari$|SeaMonkey$|Vivaldi$|Yandex Browser$)'
replacement: 'Desktop'
# Phone Browsers, Most Common
- regex: '^(Chrome Mobile$|Chrome Mobile iOS|Firefox Mobile|Firefox iOS|Edge Mobile|Android|Facebook|Instagram|IE Mobile)'
replacement: 'Phone'
# Phone Browsers, Not Common
- regex: '^(BlackBerry WebKit|OktaMobile|Sailfish Browser|Amazon Silk|Pinterest|Flipboard)'
replacement: 'Phone'
- regex: 'Tablet|BlackBerry Tablet OS|iPad|FireOS|Crosswalk'
replacement: 'Tablet'
device_parsers:
- regex: 'Tablet|BlackBerry Tablet OS|iPad|FireOS|Crosswalk|Kindle'
replacement: 'Tablet'
# Samsung tablets
- regex: 'SM-T\d+|SM-P\d+|GT-P\d+'
replacement: 'Tablet'
# other tablets
- regex: 'Asus Nexus \d+|Lenovo TB'
replacement: 'Tablet'
agent_string_parsers:
- regex: 'Synthetic|Scanner|Crawler|Site24x7|PagePeeker|SpeedCurve|RuxitSynthetic|Google Web Preview|Synthetic|SiteChecker|Parser'
replacement: 'Robot'
- regex: 'Tablet'
replacement: 'Tablet'

View File

@ -0,0 +1,217 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
package org.elasticsearch.ingest.useragent;
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.test.ESTestCase;
import org.junit.BeforeClass;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.elasticsearch.ingest.useragent.UserAgentParser.VersionedName;
import static org.elasticsearch.ingest.useragent.UserAgentParser.readParserConfigurations;
import static org.hamcrest.Matchers.is;
public class DeviceTypeParserTests extends ESTestCase {
private static DeviceTypeParser deviceTypeParser;
private ArrayList<HashMap<String, String>> readTestDevices(InputStream regexStream, String keyName) throws IOException {
XContentParser yamlParser = XContentFactory.xContent(XContentType.YAML).createParser(NamedXContentRegistry.EMPTY,
LoggingDeprecationHandler.INSTANCE, regexStream);
XContentParser.Token token = yamlParser.nextToken();
ArrayList<HashMap<String, String>> testDevices = new ArrayList<>();
if (token == XContentParser.Token.START_OBJECT) {
token = yamlParser.nextToken();
for (; token != null; token = yamlParser.nextToken()) {
String currentName = yamlParser.currentName();
if (token == XContentParser.Token.FIELD_NAME && currentName.equals(keyName)) {
List<Map<String, String>> parserConfigurations = readParserConfigurations(yamlParser);
for (Map<String, String> map : parserConfigurations) {
HashMap<String, String> testDevice = new HashMap<>();
testDevice.put("type", map.get("type"));
testDevice.put("os", map.get("os"));
testDevice.put("browser", map.get("browser"));
testDevice.put("device", map.get("device"));
testDevices.add(testDevice);
}
}
}
}
return testDevices;
}
private static VersionedName getVersionName(String name){
return new VersionedName(name, null, null, null, null);
}
@BeforeClass
public static void setupDeviceParser() throws IOException {
InputStream deviceTypeRegexStream = UserAgentProcessor.class.getResourceAsStream("/device_type_regexes.yml");
assertNotNull(deviceTypeRegexStream);
assertNotNull(deviceTypeRegexStream);
deviceTypeParser = new DeviceTypeParser();
deviceTypeParser.init(deviceTypeRegexStream);
}
@SuppressWarnings("unchecked")
public void testMacDesktop() throws Exception {
VersionedName os = getVersionName("Mac OS X");
VersionedName userAgent = getVersionName("Chrome");
String deviceType = deviceTypeParser.findDeviceType(userAgent, os, null);
assertThat(deviceType, is("Desktop"));
}
@SuppressWarnings("unchecked")
public void testAndroidMobile() throws Exception {
VersionedName os = getVersionName("iOS");
VersionedName userAgent = getVersionName("Safari");
String deviceType = deviceTypeParser.findDeviceType(userAgent, os, null);
assertThat(deviceType, is("Phone"));
}
@SuppressWarnings("unchecked")
public void testIPadTablet() throws Exception {
VersionedName os = getVersionName("iOS");
VersionedName userAgent = getVersionName("Safari");
VersionedName device = getVersionName("iPad");
String deviceType = deviceTypeParser.findDeviceType(userAgent, os, device);
assertThat(deviceType, is("Tablet"));
}
@SuppressWarnings("unchecked")
public void testWindowDesktop() throws Exception {
VersionedName os = getVersionName("Mac OS X");
VersionedName userAgent = getVersionName("Chrome");
String deviceType = deviceTypeParser.findDeviceType(userAgent, os, null);
assertThat(deviceType, is("Desktop"));
}
@SuppressWarnings("unchecked")
public void testRobotAgentString() throws Exception {
String deviceType = deviceTypeParser.findDeviceType(
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:63.0.247) Gecko/20100101 Firefox/63.0.247 Site24x7", null, null, null);
assertThat(deviceType, is("Robot"));
}
@SuppressWarnings("unchecked")
public void testRobotDevices() throws Exception {
InputStream testRobotDevices = IngestUserAgentPlugin.class.getResourceAsStream("/test-robot-devices.yml");
ArrayList<HashMap<String, String>> testDevices = readTestDevices(testRobotDevices, "robot_devices");
for (HashMap<String, String> testDevice : testDevices) {
VersionedName os = getVersionName(testDevice.get("os"));
VersionedName userAgent = getVersionName(testDevice.get("browser"));
String deviceType = deviceTypeParser.findDeviceType(userAgent, os, null);
assertThat(deviceType, is("Robot"));
}
}
@SuppressWarnings("unchecked")
public void testDesktopDevices() throws Exception {
InputStream testDesktopDevices = IngestUserAgentPlugin.class.getResourceAsStream("/test-desktop-devices.yml");
ArrayList<HashMap<String, String>> testDevices = readTestDevices(testDesktopDevices, "desktop_devices");
for (HashMap<String, String> testDevice : testDevices) {
VersionedName os = getVersionName(testDevice.get("os"));
VersionedName userAgent = getVersionName(testDevice.get("browser"));
String deviceType = deviceTypeParser.findDeviceType(userAgent, os, null);
assertThat(deviceType, is("Desktop"));
}
}
@SuppressWarnings("unchecked")
public void testMobileDevices() throws Exception {
InputStream testMobileDevices = IngestUserAgentPlugin.class.getResourceAsStream("/test-mobile-devices.yml");
ArrayList<HashMap<String, String>> testDevices = readTestDevices(testMobileDevices, "mobile_devices");
for (HashMap<String, String> testDevice : testDevices) {
VersionedName os = getVersionName(testDevice.get("os"));
VersionedName userAgent = getVersionName(testDevice.get("browser"));
String deviceType = deviceTypeParser.findDeviceType(userAgent, os, null);
assertThat(deviceType, is("Phone"));
}
}
@SuppressWarnings("unchecked")
public void testTabletDevices() throws Exception {
InputStream testTabletDevices = IngestUserAgentPlugin.class.getResourceAsStream("/test-tablet-devices.yml");
ArrayList<HashMap<String, String>> testDevices = readTestDevices(testTabletDevices, "tablet_devices");
for (HashMap<String, String> testDevice : testDevices) {
VersionedName os = getVersionName(testDevice.get("os"));
VersionedName userAgent = getVersionName(testDevice.get("browser"));
VersionedName device = getVersionName(testDevice.get("device"));
String deviceType = deviceTypeParser.findDeviceType(userAgent, os, device);
assertThat(deviceType, is("Tablet"));
}
}
}

View File

@ -32,9 +32,12 @@ public class UserAgentProcessorTests extends ESTestCase {
@BeforeClass
public static void setupProcessor() throws IOException {
InputStream regexStream = UserAgentProcessor.class.getResourceAsStream("/regexes.yml");
assertNotNull(regexStream);
InputStream deviceTypeRegexStream = UserAgentProcessor.class.getResourceAsStream("/device_type_regexes.yml");
UserAgentParser parser = new UserAgentParser(randomAlphaOfLength(10), regexStream, new UserAgentCache(1000));
assertNotNull(regexStream);
assertNotNull(deviceTypeRegexStream);
UserAgentParser parser = new UserAgentParser(randomAlphaOfLength(10), regexStream, deviceTypeRegexStream, new UserAgentCache(1000));
processor = new UserAgentProcessor(randomAlphaOfLength(10), null, "source_field", "target_field", parser,
EnumSet.allOf(UserAgentProcessor.Property.class), false);
@ -101,6 +104,34 @@ public class UserAgentProcessorTests extends ESTestCase {
assertThat(target.get("os"), is(os));
Map<String, String> device = new HashMap<>();
device.put("name", "Mac");
device.put("type", "Desktop");
assertThat(target.get("device"), is(device));
}
@SuppressWarnings("unchecked")
public void testWindowsOS() throws Exception {
Map<String, Object> document = new HashMap<>();
document.put("source_field",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36");
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
processor.execute(ingestDocument);
Map<String, Object> data = ingestDocument.getSourceAndMetadata();
assertThat(data, hasKey("target_field"));
Map<String, Object> target = (Map<String, Object>) data.get("target_field");
assertThat(target.get("name"), is("Chrome"));
assertThat(target.get("version"), is("87.0.4280.141"));
Map<String, String> os = new HashMap<>();
os.put("name", "Windows");
os.put("version", "10");
os.put("full", "Windows 10");
assertThat(target.get("os"), is(os));
Map<String, String> device = new HashMap<>();
device.put("name", "Other");
device.put("type", "Desktop");
assertThat(target.get("device"), is(device));
}
@ -129,6 +160,7 @@ public class UserAgentProcessorTests extends ESTestCase {
Map<String, String> device = new HashMap<>();
device.put("name", "Motorola Xoom");
device.put("type", "Phone");
assertThat(target.get("device"), is(device));
}
@ -152,6 +184,37 @@ public class UserAgentProcessorTests extends ESTestCase {
Map<String, String> device = new HashMap<>();
device.put("name", "Spider");
device.put("type", "Robot");
assertThat(target.get("device"), is(device));
}
@SuppressWarnings("unchecked")
public void testTablet() throws Exception {
Map<String, Object> document = new HashMap<>();
document.put("source_field",
"Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) " +
"Version/12.1 Mobile/15E148 Safari/604.1");
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
processor.execute(ingestDocument);
Map<String, Object> data = ingestDocument.getSourceAndMetadata();
assertThat(data, hasKey("target_field"));
Map<String, Object> target = (Map<String, Object>) data.get("target_field");
assertThat(target.get("name"), is("Mobile Safari"));
assertThat(target.get("version"), is("12.1"));
Map<String, String> os = new HashMap<>();
os.put("name", "iOS");
os.put("version", "12.2");
os.put("full", "iOS 12.2");
assertThat(target.get("os"), is(os));
Map<String, String> device = new HashMap<>();
device.put("name", "iPad");
device.put("type", "Tablet");
assertThat(target.get("device"), is(device));
}
@ -177,6 +240,7 @@ public class UserAgentProcessorTests extends ESTestCase {
assertNull(target.get("os"));
Map<String, String> device = new HashMap<>();
device.put("name", "Other");
device.put("type", "Other");
assertThat(target.get("device"), is(device));
}
}

View File

@ -0,0 +1,177 @@
# Apache License, Version 2.0
# ===========================
#
# Copyright 2009 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
desktop_devices:
- type: Desktop
os: Windows
browser: Chrome
- type: Desktop
os: Mac OS X
browser: Chrome
- type: Desktop
os: Linux
browser: Cypress
- type: Desktop
os: Ubuntu
browser: Firefox
- type: Desktop
os: Chrome OS
browser: Chrome
- type: Desktop
os: Fedora
browser: Firefox
- type: Desktop
os: FreeBSD
browser: Chrome
- type: Desktop
os: OpenBSD
browser: Firefox
- type: Desktop
os: Arch Linux
browser: Firefox
- type: Desktop
os: Solaris
browser: Firefox
- type: Desktop
os: NetBSD
browser: Firefox
- type: Desktop
os: SUSE
browser: Epiphany
- type: Desktop
browser: Chrome
os: Mac OS X
- type: Desktop
browser: Firefox
os: Windows NT
- type: Desktop
browser: Edge
os: Windows NT
- type: Desktop
browser: HeadlessChrome
os: Linux
- type: Desktop
browser: Safari
os: Mac OS X
- type: Desktop
browser: Electron
os: Linux
- type: Desktop
browser: Opera
os: Linux
- type: Desktop
browser: Samsung Internet
os: Linux
- type: Desktop
browser: Chromium
os: Ubuntu
- type: Desktop
browser: Yandex Browser
os: Windows NT
- type: Desktop
browser: Whale
os: Windows NT
- type: Desktop
browser: Sogou Explorer
os: Windows NT
- type: Desktop
browser: QQ Browser
os: Windows NT
- type: Desktop
browser: IE
os: Windows NT
- type: Desktop
browser: Yeti
os: Windows NT
- type: Desktop
browser: Apple Mail
os: Mac OS X
- type: Desktop
browser: Coc Coc
os: Windows NT
- type: Desktop
browser: Maxthon
os: Windows NT
- type: Desktop
browser: Waterfox
os: Linux
- type: Desktop
browser: Iron
os: Mac OS X
- type: Desktop
browser: UC Browser
os: Windows NT
- type: Desktop
browser: Pale Moon
os: Linux
- type: Desktop
browser: WordPress
os: Linux
- type: Desktop
browser: Vivaldi
os: Windows NT
- type: Desktop
browser: Dragon
os: Windows NT
- type: Desktop
browser: SeaMonkey
os: Windows NT
- type: Desktop
browser: Sleipnir
os: Windows NT
- type: Desktop
browser: Thunderbird
os: Linux
- type: Desktop
browser: Epiphany
os: Linux
- type: Desktop
browser: Datanyze
os: Linux
- type: Desktop
browser: Basilisk
os: Windows NT
- type: Desktop
browser: Swiftfox
os: Linux
- type: Desktop
browser: Netscape
os: SunOS
- type: Desktop
browser: Puffin
os: Linux
- type: Desktop
browser: Seznam prohlížeč
os: Windows NT
- type: Desktop
browser: iCab
os: Mac OS X
- type: Desktop
browser: Opera Neon
os: Windows NT
- type: Desktop
browser: Mail.ru Chromium Browser
os: Windows NT
- type: Desktop
browser: Otter
os: BeOS/Haiku
- type: Desktop
browser: Iceweasel
os: Linux
- type: Desktop
browser: Chrome Mobile WebView
os: Linux

View File

@ -0,0 +1,124 @@
# Apache License, Version 2.0
# ===========================
#
# Copyright 2009 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
mobile_devices:
- type: Phone
os: Android
browser: Chrome
- type: Phone
os: iOS
browser: Firefox
# - type: Phone
# os: Windows Phone
# browser: Edge
- type: Phone
os: KaiOS
browser: Firefox
- type: Phone
os: Sailfish
browser: SailfishBrowser
- type: Phone
os: Maemo
browser: Fennec
- type: Phone
os: BlackBerry OS
browser: Mobile Safari
- type: Phone
browser: Chrome Mobile
os: Android
- type: Phone
browser: Mobile Safari
os: iOS
- type: Phone
browser: Chrome Mobile WebView
os: Android
- type: Phone
browser: Firefox Mobile
os: Android
- type: Phone
browser: Chrome Mobile iOS
os: iOS
- type: Phone
browser: Facebook
os: Android
- type: Phone
browser: Mobile Safari UI/WKWebView
os: iOS
- type: Phone
browser: Firefox iOS
os: iOS
- type: Phone
browser: Opera Mobile
os: Android
- type: Phone
browser: MiuiBrowser
os: Android
- type: Phone
browser: Edge Mobile
os: Android
- type: Phone
browser: Android
os: Android
- type: Phone
browser: LINE
os: iOS
- type: Phone
browser: QQ Browser Mobile
os: Android
- type: Phone
browser: Flipboard
os: Android
- type: Phone
browser: Instagram
os: iOS
- type: Phone
browser: Pinterest
os: iOS
- type: Phone
browser: OktaMobile
os: iOS
- type: Phone
browser: Twitter
os: Android
- type: Phone
browser: Mint Browser
os: Android
- type: Phone
browser: Snapchat
os: iOS
- type: Phone
browser: IE Mobile
os: Windows Phone
- type: Phone
browser: Sailfish Browser
os: Linux
- type: Phone
browser: MobileIron
os: iOS
- type: Phone
browser: charlotte
os: Android
- type: Phone
browser: BlackBerry WebKit
os: BlackBerry
- type: Phone
browser: YandexSearch
os: Android
- type: Phone
browser: Salesforce
os: iOS

View File

@ -0,0 +1,22 @@
# Apache License, Version 2.0
# ===========================
#
# Copyright 2009 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
robot_devices:
- type: Desktop
os: Tizen
browser: AppleWebKit

View File

@ -0,0 +1,123 @@
# Apache License, Version 2.0
# ===========================
#
# Copyright 2009 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
robot_devices:
- type: Robot
os: CentOS
browser: AdsBot-Naver
- type: Robot
browser: iSec_Bot
os: Cloud
- type: Robot
browser: moatbot
os: Cloud
- type: Robot
browser: Baiduspider-render
os: Cloud
- type: Robot
browser: AhrefsBot
os: Cloud
- type: Robot
browser: Applebot
os: Cloud
- type: Robot
browser: Seekport Crawler
os: Cloud
- type: Robot
browser: Linespider
os: Cloud
- type: Robot
browser: pingbot
os: Cloud
- type: Robot
browser: YisouSpider
os: Cloud
- type: Robot
browser: HubSpot Crawler
os: Cloud
- type: Robot
browser: AdsBot
os: Cloud
- type: Robot
browser: net/bot
os: Cloud
- type: Robot
browser: Investment Crawler
os: Cloud
- type: Robot
browser: Bytespider
os: Cloud
- type: Robot
browser: IBM-Crawler
os: Cloud
- type: Robot
browser: BublupBot
os: Cloud
- type: Robot
browser: AppEngine-Google
os: Google Cloud
- type: Robot
browser: YandexBot
os: Cloud
- type: Robot
browser: Slackbot-LinkExpanding
os: Cloud
- type: Robot
browser: WebPageTest.org bot
os: Cloud
- type: Robot
browser: Baiduspider-image
os: Cloud
- type: Robot
browser: Pinterestbot
os: Cloud
- type: Robot
browser: YandexAccessibilityBot
os: Cloud
- type: Robot
browser: FacebookBot
os: Cloud
- type: Robot
browser: BLEXBot
os: Cloud
- type: Robot
browser: SuperBot
os: Cloud
- type: Robot
browser: Googlebot-News
os: Cloud
- type: Robot
browser: SMTBot
os: Cloud
- type: Robot
browser: GooglePlusBot
os: Cloud
- type: Robot
browser: niocBot
os: Cloud
- type: Robot
browser: SpiderWeb
os: Cloud
- type: Robot
browser: facebot
os: Cloud
- type: Robot
browser: MJ12bot
os: Cloud
- type: Robot
browser: ethical-bugbot
os: Linux

View File

@ -0,0 +1,39 @@
# Apache License, Version 2.0
# ===========================
#
# Copyright 2009 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
tablet_devices:
- type: Tablet
os: BlackBerry Tablet OS
browser: Edg
- type: Tablet
browser: Amazon Silk
os: FireOS
- type: Tablet
browser: Crosswalk
os: Android
- type: Tablet
browser: Chrome Mobile WebView
os: Android
device: Samsung SM-T590
- type: Tablet
browser: Amazon Silk
os: Linux
device: Kindle
- type: Tablet
browser: Chrome
os: Android
device: Samsung SM-T307U

View File

@ -32,7 +32,7 @@
- match: { _source.user_agent.original: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" }
- match: { _source.user_agent.os: {"name":"Mac OS X", "version":"10.9.2", "full":"Mac OS X 10.9.2"} }
- match: { _source.user_agent.version: "33.0.1750.149" }
- match: { _source.user_agent.device: {"name": "Mac" }}
- match: { _source.user_agent.device: {"name": "Mac", type: "Desktop" }}
---
"Test user agent processor with parameters":

View File

@ -30,6 +30,6 @@
id: 1
- match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" }
- match: { _source.user_agent.name: "Test" }
- match: { _source.user_agent.device: {"name": "Other" }}
- match: { _source.user_agent.device: {"name": "Other", "type": "Other" }}
- is_false: _source.user_agent.os
- is_false: _source.user_agent.version