Support types table in lookup join docs (#130410)
* Support types table in lookup join docs * Don't show a results column in the join types * Make LOOKUP JOIN types table more compact * Update docs/reference/query-languages/esql/esql-lookup-join.md Co-authored-by: Liam Thompson <32779855+leemthompo@users.noreply.github.com> Co-authored-by: Alexander Spies <alexander.spies@elastic.co>
This commit is contained in:
Craig Taverner
GitHub
parent
ee1bbafca7
commit
efd1aaf46d
|
|
@ -42,6 +42,9 @@ results, the output will contain one row for each matching combination.
|
|||
For important information about using `LOOKUP JOIN`, refer to [Usage notes](../../../../esql/esql-lookup-join.md#usage-notes).
|
||||
::::
|
||||
|
||||
:::{include} ../types/lookup-join.md
|
||||
:::
|
||||
|
||||
**Examples**
|
||||
|
||||
**IP Threat correlation**: This query would allow you to see if any source
|
||||
|
|
|
|||
|
|
@ -0,0 +1,21 @@
|
|||
% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it.
|
||||
|
||||
**Supported types**
|
||||
|
||||
| field from the left index | field from the lookup index |
|
||||
| --- | --- |
|
||||
| boolean | boolean |
|
||||
| byte | half_float, float, double, scaled_float, byte, short, integer, long |
|
||||
| date | date |
|
||||
| date_nanos | date_nanos |
|
||||
| double | half_float, float, double, scaled_float, byte, short, integer, long |
|
||||
| float | half_float, float, double, scaled_float, byte, short, integer, long |
|
||||
| half_float | half_float, float, double, scaled_float, byte, short, integer, long |
|
||||
| integer | half_float, float, double, scaled_float, byte, short, integer, long |
|
||||
| ip | ip |
|
||||
| keyword | keyword |
|
||||
| long | half_float, float, double, scaled_float, byte, short, integer, long |
|
||||
| scaled_float | half_float, float, double, scaled_float, byte, short, integer, long |
|
||||
| short | half_float, float, double, scaled_float, byte, short, integer, long |
|
||||
| text | keyword |
|
||||
|
||||
|
|
@ -142,19 +142,38 @@ Refer to the examples section of the [`LOOKUP JOIN`](/reference/query-languages/
|
|||
|
||||
## Prerequisites [esql-lookup-join-prereqs]
|
||||
|
||||
To use `LOOKUP JOIN`, the following requirements must be met:
|
||||
### Index configuration
|
||||
|
||||
* Indices used for lookups must be configured with the [`lookup` index mode](/reference/elasticsearch/index-settings/index-modules.md#index-mode-setting)
|
||||
* **Compatible data types**: The join key and join field in the lookup index must have compatible data types. This means:
|
||||
* The data types must either be identical or be internally represented as the same type in {{esql}}
|
||||
* Numeric types follow these compatibility rules:
|
||||
* `short` and `byte` are compatible with `integer` (all represented as `int`)
|
||||
* `float`, `half_float`, and `scaled_float` are compatible with `double` (all represented as `double`)
|
||||
* For text fields: You can only use text fields as the join key on the left-hand side of the join and only if they have a `.keyword` subfield
|
||||
Indices used for lookups must be configured with the [`lookup` index mode](/reference/elasticsearch/index-settings/index-modules.md#index-mode-setting).
|
||||
|
||||
### Data type compatibility
|
||||
|
||||
Join keys must have compatible data types between the source and lookup indices. Types within the same compatibility group can be joined together:
|
||||
|
||||
| Compatibility group | Types | Notes |
|
||||
|------------------------|-------------------------------------------------------------------------------------|----------------------------------------------------------------------------------|
|
||||
| **Numeric family** | `byte`, `short`, `integer`, `long`, `half_float`, `float`, `scaled_float`, `double` | All compatible |
|
||||
| **Keyword family** | `keyword`, `text.keyword` | Text fields only as join key on left-hand side and must have `.keyword` subfield |
|
||||
| **Date (Exact)** | `date` | Must match exactly |
|
||||
| **Date Nanos (Exact)** | `date_nanos` | Must match exactly |
|
||||
| **Boolean** | `boolean` | Must match exactly |
|
||||
|
||||
```{tip}
|
||||
To obtain a join key with a compatible type, use a [conversion function](/reference/query-languages/esql/functions-operators/type-conversion-functions.md) if needed.
|
||||
```
|
||||
|
||||
For a complete list of supported data types and their internal representations, see the [Supported Field Types documentation](/reference/query-languages/esql/limitations.md#_supported_types).
|
||||
### Unsupported Types
|
||||
|
||||
In addition to the [{{esql}} unsupported field types](/reference/query-languages/esql/limitations.md#_unsupported_types), `LOOKUP JOIN` does not support:
|
||||
|
||||
* `VERSION`
|
||||
* `UNSIGNED_LONG`
|
||||
* Spatial types like `GEO_POINT`, `GEO_SHAPE`
|
||||
* Temporal intervals like `DURATION`, `PERIOD`
|
||||
|
||||
```{note}
|
||||
For a complete list of all types supported in `LOOKUP JOIN`, refer to the [`LOOKUP JOIN` supported types table](/reference/query-languages/esql/commands/processing-commands.md#esql-lookup-join).
|
||||
```
|
||||
|
||||
## Usage notes
|
||||
|
||||
|
|
|
|||
|
|
@ -226,6 +226,47 @@ tasks.named("test").configure {
|
|||
}
|
||||
}
|
||||
|
||||
// This is similar to the test task above, but needed for the LookupJoinTypesIT which runs in the internalClusterTest task
|
||||
// and generates a types table for the LOOKUP JOIN command. It is possible in future we might have move tests that do this.
|
||||
tasks.named("internalClusterTest").configure {
|
||||
if (buildParams.ci == false) {
|
||||
systemProperty 'generateDocs', true
|
||||
def injected = project.objects.newInstance(Injected)
|
||||
// Define the folder to delete and recreate
|
||||
def tempDir = file("build/testrun/internalClusterTest/temp/esql")
|
||||
doFirst {
|
||||
injected.fs.delete {
|
||||
it.delete(tempDir)
|
||||
}
|
||||
// Re-create this folder so we can save a table of generated examples to extract from csv-spec tests
|
||||
tempDir.mkdirs() // Recreate the folder
|
||||
}
|
||||
File snippetsFolder = file("build/testrun/internalClusterTest/temp/esql/_snippets")
|
||||
def snippetsDocFolder = file("${rootDir}/docs/reference/query-languages/esql/_snippets")
|
||||
def snippetsTree = fileTree(snippetsFolder).matching {
|
||||
include "**/types/*.md" // Recursively include all types/*.md files (effectively counting functions and operators)
|
||||
}
|
||||
|
||||
doLast {
|
||||
def snippets = snippetsTree.files.collect { it.name }
|
||||
int countSnippets = snippets.size()
|
||||
if (countSnippets == 0) {
|
||||
logger.quiet("ESQL Docs: No function/operator snippets created. Skipping sync.")
|
||||
} else {
|
||||
logger.quiet("ESQL Docs: Found $countSnippets generated function/operator snippets to patch into docs")
|
||||
injected.fs.sync {
|
||||
from snippetsFolder
|
||||
into snippetsDocFolder
|
||||
include '**/*.md'
|
||||
preserve {
|
||||
include '**/*.md'
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/****************************************************************
|
||||
* Enable QA/rest integration tests for snapshot builds only *
|
||||
* TODO: Enable for all builds upon this feature release *
|
||||
|
|
|
|||
|
|
@ -19,6 +19,8 @@ import org.elasticsearch.xcontent.XContentType;
|
|||
import org.elasticsearch.xpack.core.esql.action.ColumnInfo;
|
||||
import org.elasticsearch.xpack.esql.VerificationException;
|
||||
import org.elasticsearch.xpack.esql.core.type.DataType;
|
||||
import org.elasticsearch.xpack.esql.expression.function.DocsV3Support;
|
||||
import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
|
||||
import org.elasticsearch.xpack.esql.plan.logical.join.Join;
|
||||
import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
|
||||
import org.elasticsearch.xpack.spatial.SpatialPlugin;
|
||||
|
|
@ -36,6 +38,7 @@ import java.util.Locale;
|
|||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.function.Supplier;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE;
|
||||
|
|
@ -265,6 +268,22 @@ public class LookupJoinTypesIT extends ESIntegTestCase {
|
|||
return existing.stream().anyMatch(c -> c.exists(indexName));
|
||||
}
|
||||
|
||||
/** This test generates documentation for the supported output types of the lookup join. */
|
||||
public void testOutputSupportedTypes() throws Exception {
|
||||
Map<List<DataType>, DataType> signatures = new LinkedHashMap<>();
|
||||
for (TestConfigs configs : testConfigurations.values()) {
|
||||
if (configs.group.equals("unsupported") || configs.group.equals("union-types")) {
|
||||
continue;
|
||||
}
|
||||
for (TestConfig config : configs.configs.values()) {
|
||||
if (config instanceof TestConfigPasses) {
|
||||
signatures.put(List.of(config.mainType(), config.lookupType()), null);
|
||||
}
|
||||
}
|
||||
}
|
||||
saveJoinTypes(() -> signatures);
|
||||
}
|
||||
|
||||
public void testLookupJoinStrings() {
|
||||
testLookupJoinTypes("strings");
|
||||
}
|
||||
|
|
@ -747,4 +766,18 @@ public class LookupJoinTypesIT extends ESIntegTestCase {
|
|||
private boolean isValidDataType(DataType dataType) {
|
||||
return UNDER_CONSTRUCTION.get(dataType) == null || UNDER_CONSTRUCTION.get(dataType).isEnabled();
|
||||
}
|
||||
|
||||
private static void saveJoinTypes(Supplier<Map<List<DataType>, DataType>> signatures) throws Exception {
|
||||
ArrayList<EsqlFunctionRegistry.ArgSignature> args = new ArrayList<>();
|
||||
args.add(new EsqlFunctionRegistry.ArgSignature("field from the left index", null, null, false, false));
|
||||
args.add(new EsqlFunctionRegistry.ArgSignature("field from the lookup index", null, null, false, false));
|
||||
DocsV3Support.CommandsDocsSupport docs = new DocsV3Support.CommandsDocsSupport(
|
||||
"lookup-join",
|
||||
LookupJoinTypesIT.class,
|
||||
null,
|
||||
args,
|
||||
signatures
|
||||
);
|
||||
docs.renderDocs();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -62,7 +62,9 @@ import java.util.HashMap;
|
|||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.TreeMap;
|
||||
import java.util.function.Function;
|
||||
import java.util.function.Supplier;
|
||||
import java.util.regex.Matcher;
|
||||
|
|
@ -310,7 +312,7 @@ public abstract class DocsV3Support {
|
|||
protected final String name;
|
||||
protected final FunctionDefinition definition;
|
||||
protected final Logger logger;
|
||||
private final Supplier<Map<List<DataType>, DataType>> signatures;
|
||||
protected final Supplier<Map<List<DataType>, DataType>> signatures;
|
||||
private TempFileWriter tempFileWriter;
|
||||
private final LicenseRequirementChecker licenseChecker;
|
||||
|
||||
|
|
@ -859,9 +861,11 @@ public abstract class DocsV3Support {
|
|||
/** Command specific docs generating, currently very empty since we only render kibana definition files */
|
||||
public static class CommandsDocsSupport extends DocsV3Support {
|
||||
private final LogicalPlan command;
|
||||
private List<EsqlFunctionRegistry.ArgSignature> args;
|
||||
private final XPackLicenseState licenseState;
|
||||
private final ObservabilityTier observabilityTier;
|
||||
|
||||
/** Used in CommandLicenseTests to generate Kibana docs with licensing information for commands */
|
||||
public CommandsDocsSupport(
|
||||
String name,
|
||||
Class<?> testClass,
|
||||
|
|
@ -875,6 +879,21 @@ public abstract class DocsV3Support {
|
|||
this.observabilityTier = observabilityTier;
|
||||
}
|
||||
|
||||
/** Used in LookupJoinTypesIT to generate table of supported types for join field */
|
||||
public CommandsDocsSupport(
|
||||
String name,
|
||||
Class<?> testClass,
|
||||
LogicalPlan command,
|
||||
List<EsqlFunctionRegistry.ArgSignature> args,
|
||||
Supplier<Map<List<DataType>, DataType>> signatures
|
||||
) {
|
||||
super("commands", name, testClass, signatures);
|
||||
this.command = command;
|
||||
this.args = args;
|
||||
this.licenseState = null;
|
||||
this.observabilityTier = null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void renderSignature() throws IOException {
|
||||
// Unimplemented until we make command docs dynamically generated
|
||||
|
|
@ -882,8 +901,14 @@ public abstract class DocsV3Support {
|
|||
|
||||
@Override
|
||||
public void renderDocs() throws Exception {
|
||||
// Currently we only render kibana definition files, but we could expand to rendering much more if we decide to
|
||||
renderKibanaCommandDefinition();
|
||||
// Currently we only render either signatures or kibana definition files,
|
||||
// but we could expand to rendering much more if we decide to
|
||||
if (args != null) {
|
||||
renderTypes(name, args);
|
||||
}
|
||||
if (licenseState != null) {
|
||||
renderKibanaCommandDefinition();
|
||||
}
|
||||
}
|
||||
|
||||
void renderKibanaCommandDefinition() throws Exception {
|
||||
|
|
@ -906,6 +931,47 @@ public abstract class DocsV3Support {
|
|||
writeToTempKibanaDir("definition", "json", rendered);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
void renderTypes(String name, List<EsqlFunctionRegistry.ArgSignature> args) throws IOException {
|
||||
assert args.size() == 2;
|
||||
StringBuilder header = new StringBuilder("| ");
|
||||
StringBuilder separator = new StringBuilder("| ");
|
||||
List<String> argNames = args.stream().map(EsqlFunctionRegistry.ArgSignature::name).toList();
|
||||
for (String arg : argNames) {
|
||||
header.append(arg).append(" | ");
|
||||
separator.append("---").append(" | ");
|
||||
}
|
||||
|
||||
Map<String, List<String>> compactedTable = new TreeMap<>();
|
||||
for (Map.Entry<List<DataType>, DataType> sig : this.signatures.get().entrySet()) {
|
||||
if (shouldHideSignature(sig.getKey(), sig.getValue())) {
|
||||
continue;
|
||||
}
|
||||
String mainType = sig.getKey().getFirst().esNameIfPossible();
|
||||
String secondaryType = sig.getKey().get(1).esNameIfPossible();
|
||||
List<String> secondaryTypes = compactedTable.computeIfAbsent(mainType, (k) -> new ArrayList<>());
|
||||
secondaryTypes.add(secondaryType);
|
||||
}
|
||||
|
||||
List<String> table = new ArrayList<>();
|
||||
for (Map.Entry<String, List<String>> sig : compactedTable.entrySet()) {
|
||||
String row = "| " + sig.getKey() + " | " + String.join(", ", sig.getValue()) + " |";
|
||||
table.add(row);
|
||||
}
|
||||
Collections.sort(table);
|
||||
if (table.isEmpty()) {
|
||||
logger.info("Warning: No table of types generated for [{}]", name);
|
||||
return;
|
||||
}
|
||||
|
||||
String rendered = DOCS_WARNING + """
|
||||
**Supported types**
|
||||
|
||||
""" + header + "\n" + separator + "\n" + String.join("\n", table) + "\n\n";
|
||||
logger.info("Writing function types for [{}]:\n{}", name, rendered);
|
||||
writeToTempSnippetsDir("types", rendered);
|
||||
}
|
||||
}
|
||||
|
||||
protected String buildFunctionSignatureSvg() throws IOException {
|
||||
|
|
@ -927,6 +993,7 @@ public abstract class DocsV3Support {
|
|||
}
|
||||
|
||||
void renderTypes(String name, List<EsqlFunctionRegistry.ArgSignature> args) throws IOException {
|
||||
boolean showResultColumn = signatures.get().values().stream().anyMatch(Objects::nonNull);
|
||||
StringBuilder header = new StringBuilder("| ");
|
||||
StringBuilder separator = new StringBuilder("| ");
|
||||
List<String> argNames = args.stream().map(EsqlFunctionRegistry.ArgSignature::name).toList();
|
||||
|
|
@ -934,8 +1001,10 @@ public abstract class DocsV3Support {
|
|||
header.append(arg).append(" | ");
|
||||
separator.append("---").append(" | ");
|
||||
}
|
||||
header.append("result |");
|
||||
separator.append("--- |");
|
||||
if (showResultColumn) {
|
||||
header.append("result |");
|
||||
separator.append("--- |");
|
||||
}
|
||||
|
||||
List<String> table = new ArrayList<>();
|
||||
for (Map.Entry<List<DataType>, DataType> sig : this.signatures.get().entrySet()) { // TODO flip to using sortedSignatures
|
||||
|
|
@ -945,7 +1014,7 @@ public abstract class DocsV3Support {
|
|||
if (sig.getKey().size() > argNames.size()) { // skip variadic [test] cases (but not those with optional parameters)
|
||||
continue;
|
||||
}
|
||||
table.add(getTypeRow(args, sig, argNames));
|
||||
table.add(getTypeRow(args, sig, argNames, showResultColumn));
|
||||
}
|
||||
Collections.sort(table);
|
||||
if (table.isEmpty()) {
|
||||
|
|
@ -964,7 +1033,8 @@ public abstract class DocsV3Support {
|
|||
private static String getTypeRow(
|
||||
List<EsqlFunctionRegistry.ArgSignature> args,
|
||||
Map.Entry<List<DataType>, DataType> sig,
|
||||
List<String> argNames
|
||||
List<String> argNames,
|
||||
boolean showResultColumn
|
||||
) {
|
||||
StringBuilder b = new StringBuilder("| ");
|
||||
for (int i = 0; i < sig.getKey().size(); i++) {
|
||||
|
|
@ -978,8 +1048,10 @@ public abstract class DocsV3Support {
|
|||
b.append(" | ");
|
||||
}
|
||||
b.append("| ".repeat(argNames.size() - sig.getKey().size()));
|
||||
b.append(sig.getValue().esNameIfPossible());
|
||||
b.append(" |");
|
||||
if (showResultColumn) {
|
||||
b.append(sig.getValue().esNameIfPossible());
|
||||
b.append(" |");
|
||||
}
|
||||
return b.toString();
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue