diff --git a/docs/reference/query-languages/esql/_snippets/commands/layout/lookup-join.md b/docs/reference/query-languages/esql/_snippets/commands/layout/lookup-join.md index da99cc69c031..7b4f2d794ac2 100644 --- a/docs/reference/query-languages/esql/_snippets/commands/layout/lookup-join.md +++ b/docs/reference/query-languages/esql/_snippets/commands/layout/lookup-join.md @@ -42,6 +42,9 @@ results, the output will contain one row for each matching combination. For important information about using `LOOKUP JOIN`, refer to [Usage notes](../../../../esql/esql-lookup-join.md#usage-notes). :::: +:::{include} ../types/lookup-join.md +::: + **Examples** **IP Threat correlation**: This query would allow you to see if any source diff --git a/docs/reference/query-languages/esql/_snippets/commands/types/lookup-join.md b/docs/reference/query-languages/esql/_snippets/commands/types/lookup-join.md new file mode 100644 index 000000000000..3e54f0ad6627 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/commands/types/lookup-join.md @@ -0,0 +1,21 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Supported types** + +| field from the left index | field from the lookup index | +| --- | --- | +| boolean | boolean | +| byte | half_float, float, double, scaled_float, byte, short, integer, long | +| date | date | +| date_nanos | date_nanos | +| double | half_float, float, double, scaled_float, byte, short, integer, long | +| float | half_float, float, double, scaled_float, byte, short, integer, long | +| half_float | half_float, float, double, scaled_float, byte, short, integer, long | +| integer | half_float, float, double, scaled_float, byte, short, integer, long | +| ip | ip | +| keyword | keyword | +| long | half_float, float, double, scaled_float, byte, short, integer, long | +| scaled_float | half_float, float, double, scaled_float, byte, short, integer, long | +| short | half_float, float, double, scaled_float, byte, short, integer, long | +| text | keyword | + diff --git a/docs/reference/query-languages/esql/esql-lookup-join.md b/docs/reference/query-languages/esql/esql-lookup-join.md index d57437833c1b..0b6764834e1d 100644 --- a/docs/reference/query-languages/esql/esql-lookup-join.md +++ b/docs/reference/query-languages/esql/esql-lookup-join.md @@ -142,19 +142,38 @@ Refer to the examples section of the [`LOOKUP JOIN`](/reference/query-languages/ ## Prerequisites [esql-lookup-join-prereqs] -To use `LOOKUP JOIN`, the following requirements must be met: +### Index configuration -* Indices used for lookups must be configured with the [`lookup` index mode](/reference/elasticsearch/index-settings/index-modules.md#index-mode-setting) -* **Compatible data types**: The join key and join field in the lookup index must have compatible data types. This means: - * The data types must either be identical or be internally represented as the same type in {{esql}} - * Numeric types follow these compatibility rules: - * `short` and `byte` are compatible with `integer` (all represented as `int`) - * `float`, `half_float`, and `scaled_float` are compatible with `double` (all represented as `double`) - * For text fields: You can only use text fields as the join key on the left-hand side of the join and only if they have a `.keyword` subfield +Indices used for lookups must be configured with the [`lookup` index mode](/reference/elasticsearch/index-settings/index-modules.md#index-mode-setting). +### Data type compatibility + +Join keys must have compatible data types between the source and lookup indices. Types within the same compatibility group can be joined together: + +| Compatibility group | Types | Notes | +|------------------------|-------------------------------------------------------------------------------------|----------------------------------------------------------------------------------| +| **Numeric family** | `byte`, `short`, `integer`, `long`, `half_float`, `float`, `scaled_float`, `double` | All compatible | +| **Keyword family** | `keyword`, `text.keyword` | Text fields only as join key on left-hand side and must have `.keyword` subfield | +| **Date (Exact)** | `date` | Must match exactly | +| **Date Nanos (Exact)** | `date_nanos` | Must match exactly | +| **Boolean** | `boolean` | Must match exactly | + +```{tip} To obtain a join key with a compatible type, use a [conversion function](/reference/query-languages/esql/functions-operators/type-conversion-functions.md) if needed. +``` -For a complete list of supported data types and their internal representations, see the [Supported Field Types documentation](/reference/query-languages/esql/limitations.md#_supported_types). +### Unsupported Types + +In addition to the [{{esql}} unsupported field types](/reference/query-languages/esql/limitations.md#_unsupported_types), `LOOKUP JOIN` does not support: + +* `VERSION` +* `UNSIGNED_LONG` +* Spatial types like `GEO_POINT`, `GEO_SHAPE` +* Temporal intervals like `DURATION`, `PERIOD` + +```{note} +For a complete list of all types supported in `LOOKUP JOIN`, refer to the [`LOOKUP JOIN` supported types table](/reference/query-languages/esql/commands/processing-commands.md#esql-lookup-join). +``` ## Usage notes diff --git a/x-pack/plugin/esql/build.gradle b/x-pack/plugin/esql/build.gradle index 0db0e558cd14..447ceb53489f 100644 --- a/x-pack/plugin/esql/build.gradle +++ b/x-pack/plugin/esql/build.gradle @@ -226,6 +226,47 @@ tasks.named("test").configure { } } +// This is similar to the test task above, but needed for the LookupJoinTypesIT which runs in the internalClusterTest task +// and generates a types table for the LOOKUP JOIN command. It is possible in future we might have move tests that do this. +tasks.named("internalClusterTest").configure { + if (buildParams.ci == false) { + systemProperty 'generateDocs', true + def injected = project.objects.newInstance(Injected) + // Define the folder to delete and recreate + def tempDir = file("build/testrun/internalClusterTest/temp/esql") + doFirst { + injected.fs.delete { + it.delete(tempDir) + } + // Re-create this folder so we can save a table of generated examples to extract from csv-spec tests + tempDir.mkdirs() // Recreate the folder + } + File snippetsFolder = file("build/testrun/internalClusterTest/temp/esql/_snippets") + def snippetsDocFolder = file("${rootDir}/docs/reference/query-languages/esql/_snippets") + def snippetsTree = fileTree(snippetsFolder).matching { + include "**/types/*.md" // Recursively include all types/*.md files (effectively counting functions and operators) + } + + doLast { + def snippets = snippetsTree.files.collect { it.name } + int countSnippets = snippets.size() + if (countSnippets == 0) { + logger.quiet("ESQL Docs: No function/operator snippets created. Skipping sync.") + } else { + logger.quiet("ESQL Docs: Found $countSnippets generated function/operator snippets to patch into docs") + injected.fs.sync { + from snippetsFolder + into snippetsDocFolder + include '**/*.md' + preserve { + include '**/*.md' + } + } + } + } + } +} + /**************************************************************** * Enable QA/rest integration tests for snapshot builds only * * TODO: Enable for all builds upon this feature release * diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java index df021f27a31f..731b2976f88d 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java @@ -19,6 +19,8 @@ import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.esql.action.ColumnInfo; import org.elasticsearch.xpack.esql.VerificationException; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.DocsV3Support; +import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; import org.elasticsearch.xpack.esql.plan.logical.join.Join; import org.elasticsearch.xpack.esql.plugin.EsqlPlugin; import org.elasticsearch.xpack.spatial.SpatialPlugin; @@ -36,6 +38,7 @@ import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.function.Consumer; +import java.util.function.Supplier; import java.util.stream.Collectors; import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE; @@ -265,6 +268,22 @@ public class LookupJoinTypesIT extends ESIntegTestCase { return existing.stream().anyMatch(c -> c.exists(indexName)); } + /** This test generates documentation for the supported output types of the lookup join. */ + public void testOutputSupportedTypes() throws Exception { + Map, DataType> signatures = new LinkedHashMap<>(); + for (TestConfigs configs : testConfigurations.values()) { + if (configs.group.equals("unsupported") || configs.group.equals("union-types")) { + continue; + } + for (TestConfig config : configs.configs.values()) { + if (config instanceof TestConfigPasses) { + signatures.put(List.of(config.mainType(), config.lookupType()), null); + } + } + } + saveJoinTypes(() -> signatures); + } + public void testLookupJoinStrings() { testLookupJoinTypes("strings"); } @@ -747,4 +766,18 @@ public class LookupJoinTypesIT extends ESIntegTestCase { private boolean isValidDataType(DataType dataType) { return UNDER_CONSTRUCTION.get(dataType) == null || UNDER_CONSTRUCTION.get(dataType).isEnabled(); } + + private static void saveJoinTypes(Supplier, DataType>> signatures) throws Exception { + ArrayList args = new ArrayList<>(); + args.add(new EsqlFunctionRegistry.ArgSignature("field from the left index", null, null, false, false)); + args.add(new EsqlFunctionRegistry.ArgSignature("field from the lookup index", null, null, false, false)); + DocsV3Support.CommandsDocsSupport docs = new DocsV3Support.CommandsDocsSupport( + "lookup-join", + LookupJoinTypesIT.class, + null, + args, + signatures + ); + docs.renderDocs(); + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3Support.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3Support.java index 322418aa8c94..f577e24f1251 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3Support.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3Support.java @@ -62,7 +62,9 @@ import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Optional; +import java.util.TreeMap; import java.util.function.Function; import java.util.function.Supplier; import java.util.regex.Matcher; @@ -310,7 +312,7 @@ public abstract class DocsV3Support { protected final String name; protected final FunctionDefinition definition; protected final Logger logger; - private final Supplier, DataType>> signatures; + protected final Supplier, DataType>> signatures; private TempFileWriter tempFileWriter; private final LicenseRequirementChecker licenseChecker; @@ -859,9 +861,11 @@ public abstract class DocsV3Support { /** Command specific docs generating, currently very empty since we only render kibana definition files */ public static class CommandsDocsSupport extends DocsV3Support { private final LogicalPlan command; + private List args; private final XPackLicenseState licenseState; private final ObservabilityTier observabilityTier; + /** Used in CommandLicenseTests to generate Kibana docs with licensing information for commands */ public CommandsDocsSupport( String name, Class testClass, @@ -875,6 +879,21 @@ public abstract class DocsV3Support { this.observabilityTier = observabilityTier; } + /** Used in LookupJoinTypesIT to generate table of supported types for join field */ + public CommandsDocsSupport( + String name, + Class testClass, + LogicalPlan command, + List args, + Supplier, DataType>> signatures + ) { + super("commands", name, testClass, signatures); + this.command = command; + this.args = args; + this.licenseState = null; + this.observabilityTier = null; + } + @Override public void renderSignature() throws IOException { // Unimplemented until we make command docs dynamically generated @@ -882,8 +901,14 @@ public abstract class DocsV3Support { @Override public void renderDocs() throws Exception { - // Currently we only render kibana definition files, but we could expand to rendering much more if we decide to - renderKibanaCommandDefinition(); + // Currently we only render either signatures or kibana definition files, + // but we could expand to rendering much more if we decide to + if (args != null) { + renderTypes(name, args); + } + if (licenseState != null) { + renderKibanaCommandDefinition(); + } } void renderKibanaCommandDefinition() throws Exception { @@ -906,6 +931,47 @@ public abstract class DocsV3Support { writeToTempKibanaDir("definition", "json", rendered); } } + + @Override + void renderTypes(String name, List args) throws IOException { + assert args.size() == 2; + StringBuilder header = new StringBuilder("| "); + StringBuilder separator = new StringBuilder("| "); + List argNames = args.stream().map(EsqlFunctionRegistry.ArgSignature::name).toList(); + for (String arg : argNames) { + header.append(arg).append(" | "); + separator.append("---").append(" | "); + } + + Map> compactedTable = new TreeMap<>(); + for (Map.Entry, DataType> sig : this.signatures.get().entrySet()) { + if (shouldHideSignature(sig.getKey(), sig.getValue())) { + continue; + } + String mainType = sig.getKey().getFirst().esNameIfPossible(); + String secondaryType = sig.getKey().get(1).esNameIfPossible(); + List secondaryTypes = compactedTable.computeIfAbsent(mainType, (k) -> new ArrayList<>()); + secondaryTypes.add(secondaryType); + } + + List table = new ArrayList<>(); + for (Map.Entry> sig : compactedTable.entrySet()) { + String row = "| " + sig.getKey() + " | " + String.join(", ", sig.getValue()) + " |"; + table.add(row); + } + Collections.sort(table); + if (table.isEmpty()) { + logger.info("Warning: No table of types generated for [{}]", name); + return; + } + + String rendered = DOCS_WARNING + """ + **Supported types** + + """ + header + "\n" + separator + "\n" + String.join("\n", table) + "\n\n"; + logger.info("Writing function types for [{}]:\n{}", name, rendered); + writeToTempSnippetsDir("types", rendered); + } } protected String buildFunctionSignatureSvg() throws IOException { @@ -927,6 +993,7 @@ public abstract class DocsV3Support { } void renderTypes(String name, List args) throws IOException { + boolean showResultColumn = signatures.get().values().stream().anyMatch(Objects::nonNull); StringBuilder header = new StringBuilder("| "); StringBuilder separator = new StringBuilder("| "); List argNames = args.stream().map(EsqlFunctionRegistry.ArgSignature::name).toList(); @@ -934,8 +1001,10 @@ public abstract class DocsV3Support { header.append(arg).append(" | "); separator.append("---").append(" | "); } - header.append("result |"); - separator.append("--- |"); + if (showResultColumn) { + header.append("result |"); + separator.append("--- |"); + } List table = new ArrayList<>(); for (Map.Entry, DataType> sig : this.signatures.get().entrySet()) { // TODO flip to using sortedSignatures @@ -945,7 +1014,7 @@ public abstract class DocsV3Support { if (sig.getKey().size() > argNames.size()) { // skip variadic [test] cases (but not those with optional parameters) continue; } - table.add(getTypeRow(args, sig, argNames)); + table.add(getTypeRow(args, sig, argNames, showResultColumn)); } Collections.sort(table); if (table.isEmpty()) { @@ -964,7 +1033,8 @@ public abstract class DocsV3Support { private static String getTypeRow( List args, Map.Entry, DataType> sig, - List argNames + List argNames, + boolean showResultColumn ) { StringBuilder b = new StringBuilder("| "); for (int i = 0; i < sig.getKey().size(); i++) { @@ -978,8 +1048,10 @@ public abstract class DocsV3Support { b.append(" | "); } b.append("| ".repeat(argNames.size() - sig.getKey().size())); - b.append(sig.getValue().esNameIfPossible()); - b.append(" |"); + if (showResultColumn) { + b.append(sig.getValue().esNameIfPossible()); + b.append(" |"); + } return b.toString(); }