mirrors
/
elasticsearch
mirror of https://mirror.osredm.com/root/elasticsearch.git
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Upgrade to lucene-9.5.0-snapshot-d19c3e2e0ed (#92957) 

9.5 will include several changes related to vector search. An extensive list is available at https://github.com/apache/lucene/milestone/4 .
This commit is contained in:
Luca Cavanna 2023-01-19 14:07:33 +01:00 committed by GitHub
parent 6ff081beef
commit edd7749164
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
59 changed files with 3655 additions and 2087 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
build-tools-internal/version.properties
View File

@ -1,5 +1,5 @@
elasticsearch = 8.7.0
lucene = 9.4.2
lucene = 9.5.0-snapshot-d19c3e2e0ed
bundled_jdk_vendor = openjdk
bundled_jdk = 19.0.1+10@afdd2e245b014143b62ccb916125e3ce
@ -21,12 +21,12 @@ netty = 4.1.86.Final
commons_lang3 = 3.9
antlr4 = 4.11.1
# when updating this version, you need to ensure compatibility with:
# - modules/ingest-attachment (transitive dependency, check the upstream POM)
# - distribution/tools/plugin-cli
# - x-pack/plugin/security
bouncycastle=1.64
# used by security and idp (need to be in sync due to cross-dependency in testing)
opensaml = 4.0.1

3
distribution/src/config/jvm.options
View File

@ -73,3 +73,6 @@
## GC logging
-Xlog:gc*,gc+age=trace,safepoint:file=@loggc@:utctime,level,pid,tags:filecount=32,filesize=64m
# temporarily disable the Panama-based MMapDirectory
-Dorg.apache.lucene.store.MMapDirectory.enableMemorySegments=false

4
docs/Versions.asciidoc
View File

@ -1,8 +1,8 @@
include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[]
:lucene_version: 9.4.2
:lucene_version_path: 9_4_2
:lucene_version: 9.5.0
:lucene_version_path: 9_5_0
:jdk: 11.0.2
:jdk_major: 11
:build_type: tar

5
docs/changelog/92957.yaml Normal file
View File

@ -0,0 +1,5 @@
pr: 92957
summary: Upgrade to lucene-9.5.0-snapshot-d19c3e2e0ed
area: Search
type: upgrade
issues: []

335
gradle/verification-metadata.xml
View File

@ -759,26 +759,36 @@
<sha256 value="37f5216e14af2772930dff9b8734353f0a80e89ba3f33e065441de6537c5e842" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="com.ibm.icu" name="icu4j" version="61.1">
<artifact name="icu4j-61.1.jar">
<sha256 value="55c98eb1838b2a4bb9a07dc36bd378532d64d0cdcb7ceee914236866a7de4464" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="com.ibm.icu" name="icu4j" version="68.2">
<artifact name="icu4j-68.2.jar">
<sha256 value="9bd7bf869a44ba8aeb0cddd7e6616e88cd4795ba5bfce2230447cb0e185a646c" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="com.jamesmurty.utils" name="java-xmlbuilder" version="0.4">
<artifact name="java-xmlbuilder-0.4.jar">
<sha256 value="681e53c4ffd59fa12068803b259e3a83d43f07a47c112e748a187dee179eb31f" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="com.jcraft" name="jsch" version="0.1.54">
<artifact name="jsch-0.1.54.jar">
<sha256 value="92eb273a3316762478fdd4fe03a0ce1842c56f496c9c12fe1235db80450e1fdb" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="com.ibm.icu" name="icu4j" version="61.1">
<artifact name="icu4j-61.1.jar">
<sha256 value="55c98eb1838b2a4bb9a07dc36bd378532d64d0cdcb7ceee914236866a7de4464"
origin="Generated by Gradle"/>
</artifact>
</component>
<component group="com.ibm.icu" name="icu4j" version="68.2">
<artifact name="icu4j-68.2.jar">
<sha256 value="9bd7bf869a44ba8aeb0cddd7e6616e88cd4795ba5bfce2230447cb0e185a646c"
origin="Generated by Gradle"/>
</artifact>
</component>
<component group="com.ibm.icu" name="icu4j" version="71.1">
<artifact name="icu4j-71.1.jar">
<sha256 value="91c4f8ebf0ceb489547098fe9d5c09a65eb419caea6ed714867f5280800bcf1a"
origin="Generated by Gradle"/>
</artifact>
</component>
<component group="com.jamesmurty.utils" name="java-xmlbuilder" version="0.4">
<artifact name="java-xmlbuilder-0.4.jar">
<sha256 value="681e53c4ffd59fa12068803b259e3a83d43f07a47c112e748a187dee179eb31f"
origin="Generated by Gradle"/>
</artifact>
</component>
<component group="com.jcraft" name="jsch" version="0.1.54">
<artifact name="jsch-0.1.54.jar">
<sha256 value="92eb273a3316762478fdd4fe03a0ce1842c56f496c9c12fe1235db80450e1fdb"
origin="Generated by Gradle"/>
</artifact>
</component>
<component group="com.jcraft" name="jsch" version="0.1.55">
<artifact name="jsch-0.1.55.jar">
<sha256 value="d492b15a6d2ea3f1cc39c422c953c40c12289073dbe8360d98c0f6f9ec74fc44" origin="Generated by Gradle"/>
@ -1678,49 +1688,82 @@
<sha256 value="adb1a33c07b45c39b926bdeeadf800f701be9c3d04e0deb543069e5f09856185" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.abego.treelayout" name="org.abego.treelayout.core" version="1.0.3">
<artifact name="org.abego.treelayout.core-1.0.3.jar">
<sha256 value="fa5e31395c39c2e7d46aca0f81f72060931607b2fa41bd36038eb2cb6fb93326" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.antlr" name="ST4" version="4.3">
<artifact name="ST4-4.3.jar">
<sha256 value="28547dba48cfceb77b6efbfe069aebe9ed3324ae60dbd52093d13a1d636ed069" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.antlr" name="antlr-runtime" version="3.4">
<artifact name="antlr-runtime-3.4.jar">
<sha256 value="5b7cf53b7b30b034023f58030c8147c433f2bee0fe7dec8fae6bebf3708c5a63" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.antlr" name="antlr-runtime" version="3.5.2">
<artifact name="antlr-runtime-3.5.2.jar">
<sha256 value="ce3fc8ecb10f39e9a3cddcbb2ce350d272d9cd3d0b1e18e6fe73c3b9389c8734" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.antlr" name="antlr4" version="4.5.3">
<artifact name="antlr4-4.5.3.jar">
<sha256 value="a32de739cfdf515774e696f91aa9697d2e7731e5cb5045ca8a4b657f8b1b4fb4" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.antlr" name="antlr4" version="4.9.2">
<artifact name="antlr4-4.9.2.jar">
<sha256 value="7d66253762da7c8c7ab6ac05da1471aeeb3cb8e92310ecfb08f939306b4c7dae" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.antlr" name="antlr4-runtime" version="4.10.1">
<artifact name="antlr4-runtime-4.10.1.jar">
<sha256 value="da66be0c98acfb29bc708300d05f1a3269c40f9984a4cb9251cf2ba1898d1334" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.antlr" name="antlr4-runtime" version="4.5.1-1">
<artifact name="antlr4-runtime-4.5.1-1.jar">
<sha256 value="ffca72bc2a25bb2b0c80a58cee60530a78be17da739bb6c91a8c2e3584ca099e" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.antlr" name="antlr4-runtime" version="4.5.3">
<artifact name="antlr4-runtime-4.5.3.jar">
<sha256 value="93bca08ec995caeaaf60bdf80035a0be8507fcdabd3c2618fd8c5aab4444a752" origin="Generated by Gradle"/>
<component group="org.abego.treelayout" name="org.abego.treelayout.core" version="1.0.3">
<artifact name="org.abego.treelayout.core-1.0.3.jar">
<sha256 value="fa5e31395c39c2e7d46aca0f81f72060931607b2fa41bd36038eb2cb6fb93326"
origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.antlr" name="ST4" version="4.3">
<artifact name="ST4-4.3.jar">
<sha256 value="28547dba48cfceb77b6efbfe069aebe9ed3324ae60dbd52093d13a1d636ed069"
origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.antlr" name="ST4" version="4.3.4">
<artifact name="ST4-4.3.4.jar">
<sha256 value="f927ac384c46d749f8b5ec68972a53aed21e00313509299616edb73bfa15ff33"
origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.antlr" name="antlr-runtime" version="3.4">
<artifact name="antlr-runtime-3.4.jar">
<sha256 value="5b7cf53b7b30b034023f58030c8147c433f2bee0fe7dec8fae6bebf3708c5a63"
origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.antlr" name="antlr-runtime" version="3.5.2">
<artifact name="antlr-runtime-3.5.2.jar">
<sha256 value="ce3fc8ecb10f39e9a3cddcbb2ce350d272d9cd3d0b1e18e6fe73c3b9389c8734"
origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.antlr" name="antlr-runtime" version="3.5.3">
<artifact name="antlr-runtime-3.5.3.jar">
<sha256 value="68bf9f5a33dfcb34033495c587e6236bef4e37aa6612919f5b1e843b90669fb9"
origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.antlr" name="antlr4" version="4.11.1">
<artifact name="antlr4-4.11.1.jar">
<sha256 value="e9686e8a663ca512afe3a2eeb6f6ad3f303abb46188991f19ebc6a0fd9c1c14f"
origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.antlr" name="antlr4" version="4.5.3">
<artifact name="antlr4-4.5.3.jar">
<sha256 value="a32de739cfdf515774e696f91aa9697d2e7731e5cb5045ca8a4b657f8b1b4fb4"
origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.antlr" name="antlr4" version="4.9.2">
<artifact name="antlr4-4.9.2.jar">
<sha256 value="7d66253762da7c8c7ab6ac05da1471aeeb3cb8e92310ecfb08f939306b4c7dae"
origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.antlr" name="antlr4-runtime" version="4.10.1">
<artifact name="antlr4-runtime-4.10.1.jar">
<sha256 value="da66be0c98acfb29bc708300d05f1a3269c40f9984a4cb9251cf2ba1898d1334"
origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.antlr" name="antlr4-runtime" version="4.11.1">
<artifact name="antlr4-runtime-4.11.1.jar">
<sha256 value="e06c6553c1ccc14d36052ec4b0fc6f13b808cf957b5b1dc3f61bf401996ada59"
origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.antlr" name="antlr4-runtime" version="4.5.1-1">
<artifact name="antlr4-runtime-4.5.1-1.jar">
<sha256 value="ffca72bc2a25bb2b0c80a58cee60530a78be17da739bb6c91a8c2e3584ca099e"
origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.antlr" name="antlr4-runtime" version="4.5.3">
<artifact name="antlr4-runtime-4.5.3.jar">
<sha256 value="93bca08ec995caeaaf60bdf80035a0be8507fcdabd3c2618fd8c5aab4444a752"
origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.antlr" name="antlr4-runtime" version="4.9.2">
@ -2611,6 +2654,11 @@
<sha256 value="cafa2c66ac81894c860e2213c6ba71d4c5a3ae6a26be698318e7de88efdd4db5" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-analysis-common" version="9.5.0-snapshot-d19c3e2e0ed">
<artifact name="lucene-analysis-common-9.5.0-snapshot-d19c3e2e0ed.jar">
<sha256 value="73f276384d65dc01697617112d960a0f01fcc197c4925f2144f386fcb21d9445" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-analysis-icu" version="9.4.1">
<artifact name="lucene-analysis-icu-9.4.1.jar">
<sha256 value="a5f35d7294ef475e9e3145e0a191551f5841da738212c4923b0a403a027db200" origin="Generated by Gradle"/>
@ -2621,6 +2669,11 @@
<sha256 value="6fc669da198bea83f2c0616a4eb765f30b7dcb729725aa098528151bb36be9d6" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-analysis-icu" version="9.5.0-snapshot-d19c3e2e0ed">
<artifact name="lucene-analysis-icu-9.5.0-snapshot-d19c3e2e0ed.jar">
<sha256 value="946c7756ee4320d09b5d55e7895b6285a5ad29e99c9de54d2aec620c4378fbac" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-analysis-kuromoji" version="9.4.1">
<artifact name="lucene-analysis-kuromoji-9.4.1.jar">
<sha256 value="daaf3c6939a9b234682d45102b8c26872f0b2d33631feefb4795a54a6dddc75a" origin="Generated by Gradle"/>
@ -2631,6 +2684,11 @@
<sha256 value="13928e760951491b66e1b37f3396a0ee4b150f02a4cbe57743b68dbd0d09ab61" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-analysis-kuromoji" version="9.5.0-snapshot-d19c3e2e0ed">
<artifact name="lucene-analysis-kuromoji-9.5.0-snapshot-d19c3e2e0ed.jar">
<sha256 value="0a1a38c2e357c67e3583ef8894f336b389a5a5337795b1ef698d7cb0b826d8e6" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-analysis-morfologik" version="9.4.1">
<artifact name="lucene-analysis-morfologik-9.4.1.jar">
<sha256 value="55cc0c58ef51d62ad0db64d130bbf163ec2bb9431acb898446e9039165c97f5b" origin="Generated by Gradle"/>
@ -2641,6 +2699,11 @@
<sha256 value="2835fb73f8d58ded27e9201a8d9546e0bdc5d028f5ae5f112f6f74cf71a291ee" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-analysis-morfologik" version="9.5.0-snapshot-d19c3e2e0ed">
<artifact name="lucene-analysis-morfologik-9.5.0-snapshot-d19c3e2e0ed.jar">
<sha256 value="7e2cf5326302d55962944d3ba7a18b8a76b9a49d87e15c4c1904a0b4ae6afdac" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-analysis-nori" version="9.4.1">
<artifact name="lucene-analysis-nori-9.4.1.jar">
<sha256 value="f3e3119787e002606181078e2d2d28715ea80222dd82d5f34013ab11a097e786" origin="Generated by Gradle"/>
@ -2651,6 +2714,11 @@
<sha256 value="316776f69a9e493c36c9a1b9b18b1f10de3914a45537e7f324784ab11d748267" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-analysis-nori" version="9.5.0-snapshot-d19c3e2e0ed">
<artifact name="lucene-analysis-nori-9.5.0-snapshot-d19c3e2e0ed.jar">
<sha256 value="4de35194eebdfb226cfe497058cec2dfd5834f7da78251bd5bc5362e0b36f807" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-analysis-phonetic" version="9.4.1">
<artifact name="lucene-analysis-phonetic-9.4.1.jar">
<sha256 value="57b21cf3c13a128d5817c369b43bb843a332ada8569bee99300b2439024daf1a" origin="Generated by Gradle"/>
@ -2661,6 +2729,11 @@
<sha256 value="30c2af3312daae2facbc43c49e4635b8f6426641d2870f7c1750ba2496c889fe" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-analysis-phonetic" version="9.5.0-snapshot-d19c3e2e0ed">
<artifact name="lucene-analysis-phonetic-9.5.0-snapshot-d19c3e2e0ed.jar">
<sha256 value="e65ce76f0ca6dd3ff3befde0a8a04951af46e195284ee6e75eab32ad25aabce3" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-analysis-smartcn" version="9.4.1">
<artifact name="lucene-analysis-smartcn-9.4.1.jar">
<sha256 value="ff65b08a942c4e9b4050f4d4a697cb550fe04e12464af14869c09f02fc75bab5" origin="Generated by Gradle"/>
@ -2671,6 +2744,11 @@
<sha256 value="ac101ee6815499d04a29693f089b141a2dd207a063e47968475bd829caa9bcf2" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-analysis-smartcn" version="9.5.0-snapshot-d19c3e2e0ed">
<artifact name="lucene-analysis-smartcn-9.5.0-snapshot-d19c3e2e0ed.jar">
<sha256 value="ca83467b4d20f00a4dbbf7c0ec35b977ed9dfc74498d563335e5a4d4c7d9474e" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-analysis-stempel" version="9.4.1">
<artifact name="lucene-analysis-stempel-9.4.1.jar">
<sha256 value="d32007019f35bef2f0ff47b192f4d2faac457bafa448336c905e0c1433e929e7" origin="Generated by Gradle"/>
@ -2681,6 +2759,11 @@
<sha256 value="7b8848fa29b4506f87e03c86699feb243cd258fe3ce92f0db8201417689e530a" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-analysis-stempel" version="9.5.0-snapshot-d19c3e2e0ed">
<artifact name="lucene-analysis-stempel-9.5.0-snapshot-d19c3e2e0ed.jar">
<sha256 value="d0ebadf4fee6e727197981f2295dc043ddd47077b58624e76af9d4b84e65464b" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-backward-codecs" version="9.4.1">
<artifact name="lucene-backward-codecs-9.4.1.jar">
<sha256 value="abdbf830fae812b27ddc98d8287e60582c8f9cffd8292a064f1f9aa8ab5e0ff0" origin="Generated by Gradle"/>
@ -2691,6 +2774,11 @@
<sha256 value="8a6dc9181693d61b25ddb057cec31abdb4d6573b12b4a096b9b5739093a45c91" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-backward-codecs" version="9.5.0-snapshot-d19c3e2e0ed">
<artifact name="lucene-backward-codecs-9.5.0-snapshot-d19c3e2e0ed.jar">
<sha256 value="e1ec8f11b9c83c0249eb145729921a08227bbcfe883c409a17309b5ab6a3824e" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-codecs" version="9.4.1">
<artifact name="lucene-codecs-9.4.1.jar">
<sha256 value="e69bcce29b43e9e09c85eb9e8a17128aa90a631c8a1b014b4efc7b8487222f4e" origin="Generated by Gradle"/>
@ -2701,6 +2789,11 @@
<sha256 value="a8bdb8e14bb6813837e5db84b4d163cd5bc6761bb370d5e6ade28561522eb587" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-codecs" version="9.5.0-snapshot-d19c3e2e0ed">
<artifact name="lucene-codecs-9.5.0-snapshot-d19c3e2e0ed.jar">
<sha256 value="3f57ada190b4561df53d0521df67a64204b315720ccc4cc04a5f621cdac21a42" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-core" version="9.4.1">
<artifact name="lucene-core-9.4.1.jar">
<sha256 value="992f847526c953f734a18295901b6fbb8977a1d99275e01ed219e7e57cc0ae34" origin="Generated by Gradle"/>
@ -2711,6 +2804,11 @@
<sha256 value="fd6db1bfe92c51439332230c1e95d2ac63633a9c3067d0355985594ff415431a" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-core" version="9.5.0-snapshot-d19c3e2e0ed">
<artifact name="lucene-core-9.5.0-snapshot-d19c3e2e0ed.jar">
<sha256 value="481c30915303e79477779b21fdf2769cd944dc5b5acef4ea2eac0bae5a5a59da" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-expressions" version="9.4.1">
<artifact name="lucene-expressions-9.4.1.jar">
<sha256 value="b037fbd8131443cf8a2f35f9c07eb68bf4d95cab01be67733a9225e7629a94e7" origin="Generated by Gradle"/>
@ -2721,6 +2819,11 @@
<sha256 value="ed0366731de0a57c77e34113df8ea34c365f93d1e60556ec20dc253a10f516f5" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-expressions" version="9.5.0-snapshot-d19c3e2e0ed">
<artifact name="lucene-expressions-9.5.0-snapshot-d19c3e2e0ed.jar">
<sha256 value="edc4e4d8398bf91bbda2717f835f0c94d816dcd26d7a7e2173a3563e8d25a719" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-grouping" version="9.4.1">
<artifact name="lucene-grouping-9.4.1.jar">
<sha256 value="dd88245285b240246280a4f922780a68831ed79df40a4884650d17902a33e361" origin="Generated by Gradle"/>
@ -2731,6 +2834,11 @@
<sha256 value="0312ba56249fb252ae8842fd148bc231bf4f3c20569a66f92105f1739815381d" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-grouping" version="9.5.0-snapshot-d19c3e2e0ed">
<artifact name="lucene-grouping-9.5.0-snapshot-d19c3e2e0ed.jar">
<sha256 value="91ac493aae11d211f2ffbe8457cf2f0c7aba68fcf2ddddb0d4da53b4d6b471cf" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-highlighter" version="9.4.1">
<artifact name="lucene-highlighter-9.4.1.jar">
<sha256 value="241af5dd27834d0a37e9ab2ef235aa125ca680181a78c51248511ee80c053240" origin="Generated by Gradle"/>
@ -2741,6 +2849,11 @@
<sha256 value="8512892073eb1c5f13a3fb1bf1b245cd66aeb1b45f7acfb47a83968a4141318d" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-highlighter" version="9.5.0-snapshot-d19c3e2e0ed">
<artifact name="lucene-highlighter-9.5.0-snapshot-d19c3e2e0ed.jar">
<sha256 value="ff7eda5d69ef6aaad97fa36472252fc992d0151a6c8c5aad920ac05540be8a19" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-join" version="9.4.1">
<artifact name="lucene-join-9.4.1.jar">
<sha256 value="1dfd06b72ce8ac45e1747a1781bf7f75681de366e17e88756c171e506f0719bb" origin="Generated by Gradle"/>
@ -2751,6 +2864,11 @@
<sha256 value="ae04eaa43bde57407a19aa9ab86038b8ee3b0d5c9d9333ecf63d283c306e6b0e" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-join" version="9.5.0-snapshot-d19c3e2e0ed">
<artifact name="lucene-join-9.5.0-snapshot-d19c3e2e0ed.jar">
<sha256 value="c29e7b0e2de53408c3d04b6562577679a3932a5f03c45b3d26ad22b1147f2132" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-memory" version="9.4.1">
<artifact name="lucene-memory-9.4.1.jar">
<sha256 value="2426d5a2475c9163dcc28b2e6f2624381abc2b83ede3f6140ab3833724ac60e6" origin="Generated by Gradle"/>
@ -2761,6 +2879,11 @@
<sha256 value="a0c9dee125bed499005893dcd9737a02b3aebe3cfa01ce6eb8d28a4e3b5dc5d4" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-memory" version="9.5.0-snapshot-d19c3e2e0ed">
<artifact name="lucene-memory-9.5.0-snapshot-d19c3e2e0ed.jar">
<sha256 value="f9ee01b34d8fa37e9b48426a00cf7e7fa947dd0356330ff9d14cefee345b2255" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-misc" version="9.4.1">
<artifact name="lucene-misc-9.4.1.jar">
<sha256 value="de8335139c1622234fc3c0d4973fcee9a7a7d53b26bc1b9ac72c3ba2c2f59b58" origin="Generated by Gradle"/>
@ -2771,6 +2894,11 @@
<sha256 value="c3a3f2a487cf62bad685527f9ace8d4728aa173ab56054c328e6e397584e1992" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-misc" version="9.5.0-snapshot-d19c3e2e0ed">
<artifact name="lucene-misc-9.5.0-snapshot-d19c3e2e0ed.jar">
<sha256 value="43c258c6016e776d70a248301f4586ab643491800ed1a887bacf1240aaad92a6" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-queries" version="9.4.1">
<artifact name="lucene-queries-9.4.1.jar">
<sha256 value="0fce03ae6070868fb7369763ea4a29aecdcc13c6342c7fc94c90d3ae29860a40" origin="Generated by Gradle"/>
@ -2781,6 +2909,11 @@
<sha256 value="1924525c4c706706baecbac7f9c0e434630b9ce595e55ae6aa63ca6d465fb453" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-queries" version="9.5.0-snapshot-d19c3e2e0ed">
<artifact name="lucene-queries-9.5.0-snapshot-d19c3e2e0ed.jar">
<sha256 value="7b1534514f1766fc6b6851fcd18e24cbece06373a03410cf71e2b76f23c99bc1" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-queryparser" version="9.4.1">
<artifact name="lucene-queryparser-9.4.1.jar">
<sha256 value="c7bf66dea0ae9e7dc840347fd7eaefa9250a91f018bc2503888afed08010adea" origin="Generated by Gradle"/>
@ -2791,6 +2924,11 @@
<sha256 value="dfc681a69bc60dd89e6bbf964a9b310b11e96de212da8597309b7caeb1cf591d" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-queryparser" version="9.5.0-snapshot-d19c3e2e0ed">
<artifact name="lucene-queryparser-9.5.0-snapshot-d19c3e2e0ed.jar">
<sha256 value="b4542f0109a7a53c366ff0f4b2b22c573ab4a2400e928d9f0d065c1f731d6e8a" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-sandbox" version="9.4.1">
<artifact name="lucene-sandbox-9.4.1.jar">
<sha256 value="ddc756a446986101ab17ac15aedfeb14588832afff38b86c07a13cd93e6e4890" origin="Generated by Gradle"/>
@ -2801,6 +2939,11 @@
<sha256 value="c381fe2797766d91b663a7cbee60e952a1c81dc37551445086f48f0dc5d47216" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-sandbox" version="9.5.0-snapshot-d19c3e2e0ed">
<artifact name="lucene-sandbox-9.5.0-snapshot-d19c3e2e0ed.jar">
<sha256 value="6c246106bd4ad6ac2a99ee8e87b76e25c7b17ffad333fbeed6206e3d23fba5ba" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-spatial-extras" version="9.4.1">
<artifact name="lucene-spatial-extras-9.4.1.jar">
<sha256 value="2fe68c8a9141af9f7a7862b576c88b74ad4509f5a2427cb423463d4233e76d4c" origin="Generated by Gradle"/>
@ -2811,6 +2954,11 @@
<sha256 value="c13713f828a03d47d3634e59abc7a53df13c23531d1b6b70c72d190ff64c5cd4" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-spatial-extras" version="9.5.0-snapshot-d19c3e2e0ed">
<artifact name="lucene-spatial-extras-9.5.0-snapshot-d19c3e2e0ed.jar">
<sha256 value="2ac4f5172e74a3772b68e86d5346f1edd88f4627d2fe3ce63e2b3d7717f36ee3" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-spatial3d" version="9.4.1">
<artifact name="lucene-spatial3d-9.4.1.jar">
<sha256 value="5911614baa3f6e75ee55d0ad63aa16286887766b2fda0654507ce6412d16353b" origin="Generated by Gradle"/>
@ -2821,6 +2969,11 @@
<sha256 value="c9688a9ab51d5a827260892de6435595a0d524ff9426288af806eb595a000dbb" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-spatial3d" version="9.5.0-snapshot-d19c3e2e0ed">
<artifact name="lucene-spatial3d-9.5.0-snapshot-d19c3e2e0ed.jar">
<sha256 value="4f4d96e385ddb5187714f5ab274d0e8cb74cab56dedbc5fa1363389c98a617a1" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-suggest" version="9.4.1">
<artifact name="lucene-suggest-9.4.1.jar">
<sha256 value="0bd1662bfaa60d6cba12e69b74f90e162900ed077320fe37fbf9e8f5cd78e59f" origin="Generated by Gradle"/>
@ -2831,6 +2984,11 @@
<sha256 value="0d56f92c5414149562ea50c6863ac50fcbbb51239d4a72c2bafbe852fb7e0a27" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-suggest" version="9.5.0-snapshot-d19c3e2e0ed">
<artifact name="lucene-suggest-9.5.0-snapshot-d19c3e2e0ed.jar">
<sha256 value="94878bf9c520702c0d5293922dd06866e25bedbd6f9bb823e28eb34dac907197" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-test-framework" version="9.4.1">
<artifact name="lucene-test-framework-9.4.1.jar">
<sha256 value="63850af6b753e0802cba7697db7821727b9b2e23eef11177c1ddbfb37ef3eaf2" origin="Generated by Gradle"/>
@ -2841,6 +2999,11 @@
<sha256 value="be318e426e3ce93d704fff41abc8ccd09bb1def69a40615ecc549c21895dd79d" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.lucene" name="lucene-test-framework" version="9.5.0-snapshot-d19c3e2e0ed">
<artifact name="lucene-test-framework-9.5.0-snapshot-d19c3e2e0ed.jar">
<sha256 value="561369612e4602cf99bcc53201b4bf9b95ed09096ba593e755416f1c96ff4bfb" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.maven" name="maven-model" version="3.6.2">
<artifact name="maven-model-3.6.2.jar">
<sha256 value="f4ada31d7217efc11d2264dec3716623cefa3440cfb2b6b1dcc640a825159a7d" origin="Generated by Gradle"/>
@ -3329,24 +3492,34 @@
<sha256 value="6cd91991323dd7b2fb28ca93d7ac12af5a86a2f53279e2b35827b30313fd0b9f" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.fusesource.leveldbjni" name="leveldbjni-all" version="1.8">
<artifact name="leveldbjni-all-1.8.jar">
<sha256 value="c297213b0e6f9392305952753f3099a4c02e70b3656266fe01867e7b6c160ffe" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.glassfish" name="javax.json" version="1.0.4">
<artifact name="javax.json-1.0.4.jar">
<sha256 value="0e1dec40a1ede965941251eda968aeee052cc4f50378bc316cc48e8159bdbeb4" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.hamcrest" name="hamcrest" version="2.1">
<artifact name="hamcrest-2.1.jar">
<sha256 value="ba93b2e3a562322ba432f0a1b53addcc55cb188253319a020ed77f824e692050" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.hamcrest" name="hamcrest" version="2.2">
<artifact name="hamcrest-2.2.jar">
<sha256 value="5e62846a89f05cd78cd9c1a553f340d002458380c320455dd1f8fc5497a8a1c1" origin="Generated by Gradle"/>
<component group="org.fusesource.leveldbjni" name="leveldbjni-all" version="1.8">
<artifact name="leveldbjni-all-1.8.jar">
<sha256 value="c297213b0e6f9392305952753f3099a4c02e70b3656266fe01867e7b6c160ffe"
origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.glassfish" name="javax.json" version="1.0.4">
<artifact name="javax.json-1.0.4.jar">
<sha256 value="0e1dec40a1ede965941251eda968aeee052cc4f50378bc316cc48e8159bdbeb4"
origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.glassfish" name="javax.json" version="1.1.4">
<artifact name="javax.json-1.1.4.jar">
<sha256 value="17fdeb7e22375a7fb40bb0551306f6dcf2b5743078668adcdf6c642c9a9ec955"
origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.hamcrest" name="hamcrest" version="2.1">
<artifact name="hamcrest-2.1.jar">
<sha256 value="ba93b2e3a562322ba432f0a1b53addcc55cb188253319a020ed77f824e692050"
origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.hamcrest" name="hamcrest" version="2.2">
<artifact name="hamcrest-2.2.jar">
<sha256 value="5e62846a89f05cd78cd9c1a553f340d002458380c320455dd1f8fc5497a8a1c1"
origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.hamcrest" name="hamcrest-core" version="1.3">

13
libs/h3/src/test/java/org/elasticsearch/h3/ParentChildNavigationTests.java
View File

@ -21,7 +21,6 @@ package org.elasticsearch.h3;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
import org.apache.lucene.geo.Point;
import org.apache.lucene.spatial3d.geom.GeoArea;
import org.apache.lucene.spatial3d.geom.GeoPoint;
import org.apache.lucene.spatial3d.geom.GeoPolygon;
import org.apache.lucene.spatial3d.geom.GeoPolygonFactory;
@ -133,23 +132,13 @@ public class ParentChildNavigationTests extends ESTestCase {
public void testNoChildrenIntersecting() {
String[] h3Addresses = H3.getStringRes0Cells();
String h3Address = RandomPicks.randomFrom(random(), h3Addresses);
// Once testIssue91915 is fixed, put upper limit of the loop to H3.MAX_H3_RES
for (int i = 1; i <= 10; i++) {
for (int i = 1; i <= H3.MAX_H3_RES; i++) {
h3Addresses = H3.h3ToChildren(h3Address);
assertIntersectingChildren(h3Address, h3Addresses);
h3Address = RandomPicks.randomFrom(random(), h3Addresses);
}
}
public void testIssue91915() {
GeoPolygon polygon1 = getGeoPolygon("8ec82ea0650155f");
GeoPolygon polygon2 = getGeoPolygon("8ec82ea06501447");
// these polygons are disjoint but due to https://github.com/apache/lucene/issues/11883
// they are reported as intersects. Once this is fixed this test will fail, we should adjust
// testNoChildrenIntersecting
assertEquals("see https://github.com/elastic/elasticsearch/issues/91915", GeoArea.OVERLAPS, polygon1.getRelationship(polygon2));
}
private void assertIntersectingChildren(String h3Address, String[] children) {
int size = H3.h3ToNotIntersectingChildrenSize(h3Address);
for (int i = 0; i < size; i++) {

16
libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentBuilder.java
View File

@ -435,6 +435,22 @@ public final class XContentBuilder implements Closeable, Flushable {
return this;
}
public XContentBuilder array(String name, byte[] values) throws IOException {
return field(name).values(values);
}
private XContentBuilder values(byte[] values) throws IOException {
if (values == null) {
return nullValue();
}
startArray();
for (byte b : values) {
value(b);
}
endArray();
return this;
}
////////////////////////////////////////////////////////////////////////////
// Double
//////////////////////////////////

2
modules/lang-expression/build.gradle
View File

@ -17,7 +17,7 @@ esplugin {
dependencies {
api "org.apache.lucene:lucene-expressions:${versions.lucene}"
runtimeOnly "org.apache.lucene:lucene-codecs:${versions.lucene}"
runtimeOnly 'org.antlr:antlr4-runtime:4.5.1-1'
runtimeOnly "org.antlr:antlr4-runtime:${versions.antlr4}"
runtimeOnly 'org.ow2.asm:asm:7.2'
runtimeOnly 'org.ow2.asm:asm-commons:7.2'
runtimeOnly 'org.ow2.asm:asm-tree:7.2'

6
modules/lang-painless/build.gradle
View File

@ -34,7 +34,7 @@ configurations {
}
dependencies {
api 'org.antlr:antlr4-runtime:4.5.3'
api "org.antlr:antlr4-runtime:${versions.antlr4}"
api 'org.ow2.asm:asm-util:7.2'
api 'org.ow2.asm:asm-tree:7.2'
api 'org.ow2.asm:asm-commons:7.2'
@ -174,7 +174,7 @@ configurations {
}
dependencies {
regenerate 'org.antlr:antlr4:4.5.3'
regenerate "org.antlr:antlr4:${versions.antlr4}"
}
String grammarPath = 'src/main/antlr'
@ -274,7 +274,7 @@ configurations {
}
dependencies {
regenerate 'org.antlr:antlr4:4.5.3'
regenerate "org.antlr:antlr4:${versions.antlr4}"
}
String suggestGrammarPath = 'src/main/antlr'

2
modules/lang-painless/src/main/java/module-info.java
View File

@ -13,7 +13,7 @@ module org.elasticsearch.painless {
requires org.elasticsearch.server;
requires org.elasticsearch.xcontent;
requires antlr4.runtime;
requires org.antlr.antlr4.runtime;
requires org.apache.lucene.core;
requires org.objectweb.asm;
requires org.objectweb.asm.commons;

1180
modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java
View File

File diff suppressed because it is too large Load Diff

1317
modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
View File

File diff suppressed because it is too large Load Diff

1
modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java
View File

@ -11,6 +11,7 @@ import org.antlr.v4.runtime.tree.AbstractParseTreeVisitor;
* @param <T> The return type of the visit operation. Use {@link Void} for
* operations with no return type.
*/
@SuppressWarnings("CheckReturnValue")
class PainlessParserBaseVisitor<T> extends AbstractParseTreeVisitor<T> implements PainlessParserVisitor<T> {
/**
* {@inheritDoc}

1152
modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/SuggestLexer.java
View File

File diff suppressed because it is too large Load Diff

3
server/src/main/java/org/elasticsearch/Version.java
View File

@ -131,7 +131,8 @@ public class Version implements Comparable<Version>, ToXContentFragment {
public static final Version V_8_5_4 = new Version(8_05_04_99, TransportVersion.V_8_5_4, org.apache.lucene.util.Version.LUCENE_9_4_2);
public static final Version V_8_6_0 = new Version(8_06_00_99, TransportVersion.V_8_6_0, org.apache.lucene.util.Version.LUCENE_9_4_2);
public static final Version V_8_6_1 = new Version(8_06_01_99, TransportVersion.V_8_6_1, org.apache.lucene.util.Version.LUCENE_9_4_2);
public static final Version V_8_7_0 = new Version(8_07_00_99, TransportVersion.V_8_7_0, org.apache.lucene.util.Version.LUCENE_9_4_2);
public static final Version V_8_7_0 = new Version(8_07_00_99, TransportVersion.V_8_7_0, org.apache.lucene.util.Version.LUCENE_9_5_0);
public static final Version CURRENT = V_8_7_0;
private static final Map<Integer, Version> VERSION_IDS;

55
server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java
View File

@ -20,6 +20,7 @@ import org.apache.lucene.codecs.StoredFieldsReader;
import org.apache.lucene.codecs.TermVectorsReader;
import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.ByteVectorValues;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.FieldInfo;
@ -137,7 +138,7 @@ final class IndexDiskUsageAnalyzer {
final int skipMask = 0x1FF; // 511
while (docID < reader.maxDoc()) {
cancellationChecker.logEvent();
storedFieldsReader.visitDocument(docID, visitor);
storedFieldsReader.document(docID, visitor);
// As we already estimate the size of stored fields, we can trade off the accuracy for the speed of the estimate.
// Here we only visit 1/11 documents instead of all documents. Ideally, we should visit 1 doc then skip 10 docs
// to avoid missing many skew documents. But, documents are stored in chunks in compressed format and a chunk can
@ -525,23 +526,47 @@ final class IndexDiskUsageAnalyzer {
cancellationChecker.checkForCancellation();
directory.resetBytesRead();
if (field.getVectorDimension() > 0) {
iterateDocValues(reader.maxDoc(), () -> vectorReader.getVectorValues(field.name), vectors -> {
cancellationChecker.logEvent();
vectors.vectorValue();
});
switch (field.getVectorEncoding()) {
case BYTE -> {
iterateDocValues(reader.maxDoc(), () -> vectorReader.getByteVectorValues(field.name), vectors -> {
cancellationChecker.logEvent();
vectors.vectorValue();
});
// do a couple of randomized searches to figure out min and max offsets of index file
VectorValues vectorValues = vectorReader.getVectorValues(field.name);
int numDocsToVisit = reader.maxDoc() < 10 ? reader.maxDoc() : 10 * (int) Math.log10(reader.maxDoc());
int skipFactor = Math.max(reader.maxDoc() / numDocsToVisit, 1);
for (int i = 0; i < reader.maxDoc(); i += skipFactor) {
if ((i = vectorValues.advance(i)) == DocIdSetIterator.NO_MORE_DOCS) {
break;
// do a couple of randomized searches to figure out min and max offsets of index file
ByteVectorValues vectorValues = vectorReader.getByteVectorValues(field.name);
int numDocsToVisit = reader.maxDoc() < 10 ? reader.maxDoc() : 10 * (int) Math.log10(reader.maxDoc());
int skipFactor = Math.max(reader.maxDoc() / numDocsToVisit, 1);
for (int i = 0; i < reader.maxDoc(); i += skipFactor) {
if ((i = vectorValues.advance(i)) == DocIdSetIterator.NO_MORE_DOCS) {
break;
}
cancellationChecker.checkForCancellation();
vectorReader.search(field.name, vectorValues.vectorValue(), 100, null, Integer.MAX_VALUE);
}
stats.addKnnVectors(field.name, directory.getBytesRead());
}
case FLOAT32 -> {
iterateDocValues(reader.maxDoc(), () -> vectorReader.getVectorValues(field.name), vectors -> {
cancellationChecker.logEvent();
vectors.vectorValue();
});
// do a couple of randomized searches to figure out min and max offsets of index file
VectorValues vectorValues = vectorReader.getVectorValues(field.name);
int numDocsToVisit = reader.maxDoc() < 10 ? reader.maxDoc() : 10 * (int) Math.log10(reader.maxDoc());
int skipFactor = Math.max(reader.maxDoc() / numDocsToVisit, 1);
for (int i = 0; i < reader.maxDoc(); i += skipFactor) {
if ((i = vectorValues.advance(i)) == DocIdSetIterator.NO_MORE_DOCS) {
break;
}
cancellationChecker.checkForCancellation();
vectorReader.search(field.name, vectorValues.vectorValue(), 100, null, Integer.MAX_VALUE);
}
stats.addKnnVectors(field.name, directory.getBytesRead());
}
cancellationChecker.checkForCancellation();
vectorReader.search(field.name, vectorValues.vectorValue(), 100, null, Integer.MAX_VALUE);
}
stats.addKnnVectors(field.name, directory.getBytesRead());
}
}
}

2
server/src/main/java/org/elasticsearch/common/lucene/Lucene.java
View File

@ -86,7 +86,7 @@ import java.util.List;
import java.util.Map;
public class Lucene {
public static final String LATEST_CODEC = "Lucene94";
public static final String LATEST_CODEC = "Lucene95";
public static final String SOFT_DELETES_FIELD = "__soft_deletes";

24
server/src/main/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQuery.java
View File

@ -299,7 +299,29 @@ public class MultiPhrasePrefixQuery extends Query {
@Override
public void visit(QueryVisitor visitor) {
if (visitor.acceptField(field)) {
visitor.visitLeaf(this); // TODO implement term visiting
visitor = visitor.getSubVisitor(BooleanClause.Occur.MUST, this);
for (int i = 0; i < termArrays.size() - 1; i++) {
if (termArrays.get(i).length == 1) {
visitor.consumeTerms(this, termArrays.get(i)[0]);
} else {
QueryVisitor shouldVisitor = visitor.getSubVisitor(BooleanClause.Occur.SHOULD, this);
shouldVisitor.consumeTerms(this, termArrays.get(i));
}
}
/* We don't report automata here because this breaks the unified highlighter,
which extracts automata separately from phrases. MPPQ gets rewritten to a
SpanMTQQuery by the PhraseHelper in any case, so highlighting is taken
care of there instead. If we extract automata here then the trailing prefix
word will be highlighted wherever it appears in the document, instead of only
as part of a phrase. This can be re-instated once we switch to using Matches
to highlight.
for (Term prefixTerm : termArrays.get(termArrays.size() - 1)) {
visitor.consumeTermsMatching(this, field, () -> {
CompiledAutomaton ca = new CompiledAutomaton(PrefixQuery.toAutomaton(prefixTerm.bytes()));
return ca.runAutomaton;
});
}
*/
}
}
}

10
server/src/main/java/org/elasticsearch/index/codec/CodecService.java
View File

@ -9,7 +9,7 @@
package org.elasticsearch.index.codec;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.lucene94.Lucene94Codec;
import org.apache.lucene.codecs.lucene95.Lucene95Codec;
import org.elasticsearch.common.util.BigArrays;
import org.elasticsearch.core.Nullable;
import org.elasticsearch.index.mapper.MapperService;
@ -35,11 +35,11 @@ public class CodecService {
public CodecService(@Nullable MapperService mapperService, BigArrays bigArrays) {
final var codecs = new HashMap<String, Codec>();
if (mapperService == null) {
codecs.put(DEFAULT_CODEC, new Lucene94Codec());
codecs.put(BEST_COMPRESSION_CODEC, new Lucene94Codec(Lucene94Codec.Mode.BEST_COMPRESSION));
codecs.put(DEFAULT_CODEC, new Lucene95Codec());
codecs.put(BEST_COMPRESSION_CODEC, new Lucene95Codec(Lucene95Codec.Mode.BEST_COMPRESSION));
} else {
codecs.put(DEFAULT_CODEC, new PerFieldMapperCodec(Lucene94Codec.Mode.BEST_SPEED, mapperService, bigArrays));
codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMapperCodec(Lucene94Codec.Mode.BEST_COMPRESSION, mapperService, bigArrays));
codecs.put(DEFAULT_CODEC, new PerFieldMapperCodec(Lucene95Codec.Mode.BEST_SPEED, mapperService, bigArrays));
codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMapperCodec(Lucene95Codec.Mode.BEST_COMPRESSION, mapperService, bigArrays));
}
codecs.put(LUCENE_DEFAULT_CODEC, Codec.getDefault());
for (String codec : Codec.availableCodecs()) {

4
server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java
View File

@ -13,7 +13,7 @@ import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.KnnVectorsFormat;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
import org.apache.lucene.codecs.lucene94.Lucene94Codec;
import org.apache.lucene.codecs.lucene95.Lucene95Codec;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.util.BigArrays;
import org.elasticsearch.index.IndexMode;
@ -37,7 +37,7 @@ import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
* per index in real time via the mapping API. If no specific postings format or vector format is
* configured for a specific field the default postings or vector format is used.
*/
public class PerFieldMapperCodec extends Lucene94Codec {
public class PerFieldMapperCodec extends Lucene95Codec {
private final MapperService mapperService;
private final DocValuesFormat docValuesFormat = new Lucene90DocValuesFormat();

2
server/src/main/java/org/elasticsearch/index/engine/IdStoredFieldLoader.java
View File

@ -35,7 +35,7 @@ final class IdStoredFieldLoader {
private static CheckedBiConsumer<Integer, StoredFieldVisitor, IOException> getStoredFieldsReader(LeafReader in) {
if (in instanceof SequentialStoredFieldsLeafReader) {
return (((SequentialStoredFieldsLeafReader) in).getSequentialStoredFieldsReader())::visitDocument;
return (((SequentialStoredFieldsLeafReader) in).getSequentialStoredFieldsReader())::document;
}
throw new IllegalArgumentException("Requires a SequentialStoredFieldsReader, got " + in.getClass());
}

2
server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java
View File

@ -338,7 +338,7 @@ final class LuceneChangesSnapshot implements Translog.Snapshot {
assert singleConsumer : "Sequential access optimization must not be enabled for multiple consumers";
assert parallelArray.useSequentialStoredFieldsReader;
assert storedFieldsReaderOrd == leaf.ord : storedFieldsReaderOrd + " != " + leaf.ord;
storedFieldsReader.visitDocument(segmentDocID, fields);
storedFieldsReader.document(segmentDocID, fields);
} else {
leaf.reader().document(segmentDocID, fields);
}

10
server/src/main/java/org/elasticsearch/index/engine/RecoverySourcePruneMergePolicy.java
View File

@ -195,8 +195,8 @@ final class RecoverySourcePruneMergePolicy extends OneMergeWrappingMergePolicy {
}
@Override
public void visitDocument(int docID, StoredFieldVisitor visitor) throws IOException {
in.visitDocument(docID, visitor);
public void document(int docID, StoredFieldVisitor visitor) throws IOException {
in.document(docID, visitor);
}
@Override
@ -220,11 +220,11 @@ final class RecoverySourcePruneMergePolicy extends OneMergeWrappingMergePolicy {
}
@Override
public void visitDocument(int docID, StoredFieldVisitor visitor) throws IOException {
public void document(int docID, StoredFieldVisitor visitor) throws IOException {
if (recoverySourceToKeep != null && recoverySourceToKeep.get(docID)) {
super.visitDocument(docID, visitor);
super.document(docID, visitor);
} else {
super.visitDocument(docID, new FilterStoredFieldVisitor(visitor) {
super.document(docID, new FilterStoredFieldVisitor(visitor) {
@Override
public Status needsField(FieldInfo fieldInfo) throws IOException {
if (recoverySourceField.equals(fieldInfo.name)) {

23
server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java
View File

@ -11,6 +11,7 @@ package org.elasticsearch.index.engine;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.index.BaseTermsEnum;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.ByteVectorValues;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.FieldInfo;
@ -30,6 +31,8 @@ import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.StoredFieldVisitor;
import org.apache.lucene.index.StoredFields;
import org.apache.lucene.index.TermVectors;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.VectorEncoding;
@ -349,11 +352,21 @@ final class TranslogDirectoryReader extends DirectoryReader {
return getDelegate().getVectorValues(field);
}
@Override
public ByteVectorValues getByteVectorValues(String field) throws IOException {
return getDelegate().getByteVectorValues(field);
}
@Override
public TopDocs searchNearestVectors(String field, float[] target, int k, Bits acceptDocs, int visitedLimit) throws IOException {
return getDelegate().searchNearestVectors(field, target, k, acceptDocs, visitedLimit);
}
@Override
public TopDocs searchNearestVectors(String field, BytesRef target, int k, Bits acceptDocs, int visitedLimit) throws IOException {
return getDelegate().searchNearestVectors(field, target, k, acceptDocs, visitedLimit);
}
@Override
public FieldInfos getFieldInfos() {
return getDelegate().getFieldInfos();
@ -382,6 +395,16 @@ final class TranslogDirectoryReader extends DirectoryReader {
return getDelegate().getTermVectors(docID);
}
@Override
public TermVectors termVectors() throws IOException {
return getDelegate().termVectors();
}
@Override
public StoredFields storedFields() throws IOException {
return getDelegate().storedFields();
}
@Override
public int numDocs() {
return 1;

2
server/src/main/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoader.java
View File

@ -108,7 +108,7 @@ public abstract class StoredFieldLoader {
private static CheckedBiConsumer<Integer, FieldsVisitor, IOException> sequentialReader(LeafReaderContext ctx) {
LeafReader leafReader = ctx.reader();
if (leafReader instanceof SequentialStoredFieldsLeafReader lf) {
return lf.getSequentialStoredFieldsReader()::visitDocument;
return lf.getSequentialStoredFieldsReader()::document;
}
return leafReader::document;
}

2
server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java
View File

@ -17,8 +17,8 @@ import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.PointValues;
import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.sandbox.search.IndexSortSortedNumericDocValuesRangeQuery;
import org.apache.lucene.search.IndexOrDocValuesQuery;
import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery;
import org.apache.lucene.search.Query;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.Version;

30
server/src/main/java/org/elasticsearch/index/mapper/DocumentLeafReader.java
View File

@ -9,6 +9,7 @@
package org.elasticsearch.index.mapper;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.ByteVectorValues;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
@ -24,6 +25,8 @@ import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.StoredFieldVisitor;
import org.apache.lucene.index.StoredFields;
import org.apache.lucene.index.TermVectors;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.VectorEncoding;
import org.apache.lucene.index.VectorSimilarityFunction;
@ -46,10 +49,9 @@ import java.util.function.Consumer;
/**
* A {@link LeafReader} over a lucene document that exposes doc values and stored fields.
* Note that unlike lucene's {@link MemoryIndex} implementation, this holds no state and
* does not attempt to do any analysis on text fields. It also supports stored
* fields where MemoryIndex does not. It is used to back index-time scripts that
* reference field data and stored fields from a document that has not yet been
* indexed.
* does not attempt to do any analysis on text fields. It is used to back index-time
* scripts that reference field data and stored fields from a document that has not yet
* been indexed.
*/
class DocumentLeafReader extends LeafReader {
@ -175,6 +177,11 @@ class DocumentLeafReader extends LeafReader {
}
}
@Override
public StoredFields storedFields() throws IOException {
throw new UnsupportedOperationException();
}
@Override
public CacheHelper getCoreCacheHelper() {
throw new UnsupportedOperationException();
@ -240,6 +247,21 @@ class DocumentLeafReader extends LeafReader {
throw new UnsupportedOperationException();
}
@Override
public ByteVectorValues getByteVectorValues(String field) {
throw new UnsupportedOperationException();
}
@Override
public TopDocs searchNearestVectors(String field, BytesRef target, int k, Bits acceptDocs, int visitedLimit) {
throw new UnsupportedOperationException();
}
@Override
public TermVectors termVectors() throws IOException {
throw new UnsupportedOperationException();
}
@Override
public CacheHelper getReaderCacheHelper() {
throw new UnsupportedOperationException();

2
server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java
View File

@ -17,8 +17,8 @@ import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.sandbox.document.HalfFloatPoint;
import org.apache.lucene.sandbox.search.IndexSortSortedNumericDocValuesRangeQuery;
import org.apache.lucene.search.IndexOrDocValuesQuery;
import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery;
import org.apache.lucene.search.MatchNoDocsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.util.BytesRef;

317
server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java
View File

@ -9,19 +9,21 @@
package org.elasticsearch.index.mapper.vectors;
import org.apache.lucene.codecs.KnnVectorsFormat;
import org.apache.lucene.codecs.lucene94.Lucene94HnswVectorsFormat;
import org.apache.lucene.codecs.lucene95.Lucene95HnswVectorsFormat;
import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.KnnByteVectorField;
import org.apache.lucene.document.KnnVectorField;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.ByteVectorValues;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.index.VectorValues;
import org.apache.lucene.search.FieldExistsQuery;
import org.apache.lucene.search.KnnByteVectorQuery;
import org.apache.lucene.search.KnnVectorQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.VectorUtil;
import org.elasticsearch.Version;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.index.fielddata.FieldDataContext;
@ -43,6 +45,7 @@ import org.elasticsearch.search.DocValueFormat;
import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
import org.elasticsearch.xcontent.ToXContent;
import org.elasticsearch.xcontent.XContentBuilder;
import org.elasticsearch.xcontent.XContentParser;
import org.elasticsearch.xcontent.XContentParser.Token;
import java.io.IOException;
@ -51,6 +54,7 @@ import java.time.ZoneId;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.function.Function;
import java.util.stream.Stream;
import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
@ -178,13 +182,18 @@ public class DenseVectorFieldMapper extends FieldMapper {
}
@Override
public float readValue(ByteBuffer byteBuffer) {
return byteBuffer.get();
public void readAndWriteValue(ByteBuffer byteBuffer, XContentBuilder b) throws IOException {
b.value(byteBuffer.get());
}
@Override
KnnByteVectorField createKnnVectorField(String name, BytesRef vector, VectorSimilarityFunction function) {
return new KnnByteVectorField(name, vector, function);
}
@Override
KnnVectorField createKnnVectorField(String name, float[] vector, VectorSimilarityFunction function) {
return new KnnVectorField(name, VectorUtil.toBytesRef(vector), function);
throw new IllegalArgumentException("cannot create a float vector field from byte");
}
@Override
@ -245,7 +254,11 @@ public class DenseVectorFieldMapper extends FieldMapper {
}
@Override
void checkVectorMagnitude(VectorSimilarity similarity, float[] vector, float squaredMagnitude) {
void checkVectorMagnitude(
VectorSimilarity similarity,
Function<StringBuilder, StringBuilder> appender,
float squaredMagnitude
) {
StringBuilder errorBuilder = null;
if (similarity == VectorSimilarity.COSINE && Math.sqrt(squaredMagnitude) == 0.0f) {
@ -255,9 +268,91 @@ public class DenseVectorFieldMapper extends FieldMapper {
}
if (errorBuilder != null) {
throw new IllegalArgumentException(appendErrorElements(errorBuilder, vector).toString());
throw new IllegalArgumentException(appender.apply(errorBuilder).toString());
}
}
@Override
public Field parseKnnVector(DocumentParserContext context, DenseVectorFieldMapper fieldMapper) throws IOException {
int index = 0;
byte[] vector = new byte[fieldMapper.dims];
float squaredMagnitude = 0;
for (Token token = context.parser().nextToken(); token != Token.END_ARRAY; token = context.parser().nextToken()) {
fieldMapper.checkDimensionExceeded(index, context);
ensureExpectedToken(Token.VALUE_NUMBER, token, context.parser());
final int value;
if (context.parser().numberType() != XContentParser.NumberType.INT) {
float floatValue = context.parser().floatValue(true);
if (floatValue % 1.0f != 0.0f) {
throw new IllegalArgumentException(
"element_type ["
+ this
+ "] vectors only support non-decimal values but found decimal value ["
+ floatValue
+ "] at dim ["
+ index
+ "];"
);
}
value = (int) floatValue;
} else {
value = context.parser().intValue(true);
}
if (value < Byte.MIN_VALUE || value > Byte.MAX_VALUE) {
throw new IllegalArgumentException(
"element_type ["
+ this
+ "] vectors only support integers between ["
+ Byte.MIN_VALUE
+ ", "
+ Byte.MAX_VALUE
+ "] but found ["
+ value
+ "] at dim ["
+ index
+ "];"
);
}
vector[index++] = (byte) value;
squaredMagnitude += value * value;
}
fieldMapper.checkDimensionMatches(index, context);
BytesRef bytesVector = new BytesRef(vector);
checkVectorMagnitude(fieldMapper.similarity, errorByteElementsAppender(bytesVector), squaredMagnitude);
return createKnnVectorField(fieldMapper.fieldType().name(), bytesVector, fieldMapper.similarity.function);
}
@Override
double parseKnnVectorToByteBuffer(DocumentParserContext context, DenseVectorFieldMapper fieldMapper, ByteBuffer byteBuffer)
throws IOException {
double dotProduct = 0f;
int index = 0;
for (Token token = context.parser().nextToken(); token != Token.END_ARRAY; token = context.parser().nextToken()) {
fieldMapper.checkDimensionExceeded(index, context);
ensureExpectedToken(Token.VALUE_NUMBER, token, context.parser());
int value = context.parser().intValue(true);
if (value < Byte.MIN_VALUE || value > Byte.MAX_VALUE) {
throw new IllegalArgumentException(
"element_type ["
+ this
+ "] vectors only support integers between ["
+ Byte.MIN_VALUE
+ ", "
+ Byte.MAX_VALUE
+ "] but found ["
+ value
+ "] at dim ["
+ index
+ "];"
);
}
byteBuffer.put((byte) value);
dotProduct += value * value;
index++;
}
fieldMapper.checkDimensionMatches(index, context);
return dotProduct;
}
},
FLOAT(4) {
@ -273,8 +368,8 @@ public class DenseVectorFieldMapper extends FieldMapper {
}
@Override
public float readValue(ByteBuffer byteBuffer) {
return byteBuffer.getFloat();
public void readAndWriteValue(ByteBuffer byteBuffer, XContentBuilder b) throws IOException {
b.value(byteBuffer.getFloat());
}
@Override
@ -282,6 +377,11 @@ public class DenseVectorFieldMapper extends FieldMapper {
return new KnnVectorField(name, vector, function);
}
@Override
KnnByteVectorField createKnnVectorField(String name, BytesRef vector, VectorSimilarityFunction function) {
throw new IllegalArgumentException("cannot create a byte vector field from float");
}
@Override
IndexFieldData.Builder fielddataBuilder(DenseVectorFieldType denseVectorFieldType, FieldDataContext fieldDataContext) {
return new VectorIndexFieldData.Builder(
@ -300,7 +400,11 @@ public class DenseVectorFieldMapper extends FieldMapper {
}
@Override
void checkVectorMagnitude(VectorSimilarity similarity, float[] vector, float squaredMagnitude) {
void checkVectorMagnitude(
VectorSimilarity similarity,
Function<StringBuilder, StringBuilder> appender,
float squaredMagnitude
) {
StringBuilder errorBuilder = null;
if (similarity == VectorSimilarity.DOT_PRODUCT && Math.abs(squaredMagnitude - 1.0f) > 1e-4f) {
@ -314,9 +418,48 @@ public class DenseVectorFieldMapper extends FieldMapper {
}
if (errorBuilder != null) {
throw new IllegalArgumentException(appendErrorElements(errorBuilder, vector).toString());
throw new IllegalArgumentException(appender.apply(errorBuilder).toString());
}
}
@Override
public Field parseKnnVector(DocumentParserContext context, DenseVectorFieldMapper fieldMapper) throws IOException {
int index = 0;
float[] vector = new float[fieldMapper.dims];
float squaredMagnitude = 0;
for (Token token = context.parser().nextToken(); token != Token.END_ARRAY; token = context.parser().nextToken()) {
fieldMapper.checkDimensionExceeded(index, context);
ensureExpectedToken(Token.VALUE_NUMBER, token, context.parser());
float value = context.parser().floatValue(true);
vector[index++] = value;
squaredMagnitude += value * value;
}
fieldMapper.checkDimensionMatches(index, context);
checkVectorBounds(vector);
checkVectorMagnitude(fieldMapper.similarity, errorFloatElementsAppender(vector), squaredMagnitude);
return createKnnVectorField(fieldMapper.fieldType().name(), vector, fieldMapper.similarity.function);
}
@Override
double parseKnnVectorToByteBuffer(DocumentParserContext context, DenseVectorFieldMapper fieldMapper, ByteBuffer byteBuffer)
throws IOException {
double dotProduct = 0f;
int index = 0;
float[] vector = new float[fieldMapper.dims];
for (Token token = context.parser().nextToken(); token != Token.END_ARRAY; token = context.parser().nextToken()) {
fieldMapper.checkDimensionExceeded(index, context);
ensureExpectedToken(Token.VALUE_NUMBER, token, context.parser());
float value = context.parser().floatValue(true);
vector[index] = value;
byteBuffer.putFloat(value);
dotProduct += value * value;
index++;
}
fieldMapper.checkDimensionMatches(index, context);
checkVectorBounds(vector);
return dotProduct;
}
};
final int elementBytes;
@ -327,15 +470,26 @@ public class DenseVectorFieldMapper extends FieldMapper {
public abstract void writeValue(ByteBuffer byteBuffer, float value);
public abstract float readValue(ByteBuffer byteBuffer);
public abstract void readAndWriteValue(ByteBuffer byteBuffer, XContentBuilder b) throws IOException;
abstract KnnVectorField createKnnVectorField(String name, float[] vector, VectorSimilarityFunction function);
abstract KnnByteVectorField createKnnVectorField(String name, BytesRef vector, VectorSimilarityFunction function);
abstract IndexFieldData.Builder fielddataBuilder(DenseVectorFieldType denseVectorFieldType, FieldDataContext fieldDataContext);
abstract Field parseKnnVector(DocumentParserContext context, DenseVectorFieldMapper fieldMapper) throws IOException;
abstract double parseKnnVectorToByteBuffer(DocumentParserContext context, DenseVectorFieldMapper fieldMapper, ByteBuffer byteBuffer)
throws IOException;
public abstract void checkVectorBounds(float[] vector);
abstract void checkVectorMagnitude(VectorSimilarity similarity, float[] vector, float squaredMagnitude);
abstract void checkVectorMagnitude(
VectorSimilarity similarity,
Function<StringBuilder, StringBuilder> errorElementsAppender,
float squaredMagnitude
);
void checkNanAndInfinite(float[] vector) {
StringBuilder errorBuilder = null;
@ -384,6 +538,30 @@ public class DenseVectorFieldMapper extends FieldMapper {
errorBuilder.append("]");
return errorBuilder;
}
StringBuilder appendErrorElements(StringBuilder errorBuilder, BytesRef vector) {
// Include the first five elements of the invalid vector in the error message
errorBuilder.append(" Preview of invalid vector: [");
for (int i = vector.offset; i < vector.offset + Math.min(5, vector.length); i++) {
if (i > vector.offset) {
errorBuilder.append(", ");
}
errorBuilder.append(vector.bytes[i]);
}
if (vector.length >= 5) {
errorBuilder.append(", ...");
}
errorBuilder.append("]");
return errorBuilder;
}
Function<StringBuilder, StringBuilder> errorFloatElementsAppender(float[] vector) {
return sb -> appendErrorElements(sb, vector);
}
Function<StringBuilder, StringBuilder> errorByteElementsAppender(BytesRef vector) {
return sb -> appendErrorElements(sb, vector);
}
}
static final Map<String, ElementType> namesToElementType = Map.of(
@ -546,7 +724,7 @@ public class DenseVectorFieldMapper extends FieldMapper {
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support term queries");
}
public KnnVectorQuery createKnnQuery(float[] queryVector, int numCands, Query filter) {
public Query createKnnQuery(BytesRef queryVector, int numCands, Query filter) {
if (isIndexed() == false) {
throw new IllegalArgumentException(
"to perform knn search on field [" + name() + "], its mapping must have [index] set to [true]"
@ -559,6 +737,35 @@ public class DenseVectorFieldMapper extends FieldMapper {
);
}
if (elementType != ElementType.BYTE) {
throw new IllegalArgumentException(
"only [" + ElementType.BYTE + "] elements are supported when querying field [" + name() + "]"
);
}
if (similarity == VectorSimilarity.DOT_PRODUCT || similarity == VectorSimilarity.COSINE) {
float squaredMagnitude = 0.0f;
for (int i = queryVector.offset; i < queryVector.offset + queryVector.length; i++) {
squaredMagnitude += queryVector.bytes[i] * queryVector.bytes[i];
}
elementType.checkVectorMagnitude(similarity, elementType.errorByteElementsAppender(queryVector), squaredMagnitude);
}
return new KnnByteVectorQuery(name(), queryVector, numCands, filter);
}
public Query createKnnQuery(float[] queryVector, int numCands, Query filter) {
if (isIndexed() == false) {
throw new IllegalArgumentException(
"to perform knn search on field [" + name() + "], its mapping must have [index] set to [true]"
);
}
if (queryVector.length != dims) {
throw new IllegalArgumentException(
"the query vector has a different dimension [" + queryVector.length + "] than the index vectors [" + dims + "]"
);
}
elementType.checkVectorBounds(queryVector);
if (similarity == VectorSimilarity.DOT_PRODUCT || similarity == VectorSimilarity.COSINE) {
@ -566,10 +773,18 @@ public class DenseVectorFieldMapper extends FieldMapper {
for (float e : queryVector) {
squaredMagnitude += e * e;
}
elementType.checkVectorMagnitude(similarity, queryVector, squaredMagnitude);
elementType.checkVectorMagnitude(similarity, elementType.errorFloatElementsAppender(queryVector), squaredMagnitude);
}
return new KnnVectorQuery(name(), queryVector, numCands, filter);
return switch (elementType) {
case BYTE -> {
byte[] bytes = new byte[queryVector.length];
for (int i = 0; i < queryVector.length; i++) {
bytes[i] = (byte) queryVector[i];
}
yield new KnnByteVectorQuery(name(), new BytesRef(bytes), numCands, filter);
}
case FLOAT -> new KnnVectorQuery(name(), queryVector, numCands, filter);
};
}
}
@ -628,21 +843,7 @@ public class DenseVectorFieldMapper extends FieldMapper {
}
private Field parseKnnVector(DocumentParserContext context) throws IOException {
float[] vector = new float[dims];
float squaredMagnitude = 0.0f;
int index = 0;
for (Token token = context.parser().nextToken(); token != Token.END_ARRAY; token = context.parser().nextToken()) {
checkDimensionExceeded(index, context);
ensureExpectedToken(Token.VALUE_NUMBER, token, context.parser());
float value = context.parser().floatValue(true);
vector[index++] = value;
squaredMagnitude += value * value;
}
checkDimensionMatches(index, context);
elementType.checkVectorBounds(vector);
elementType.checkVectorMagnitude(similarity, vector, squaredMagnitude);
return elementType.createKnnVectorField(fieldType().name(), vector, similarity.function);
return elementType.parseKnnVector(context, this);
}
private Field parseBinaryDocValuesVector(DocumentParserContext context) throws IOException {
@ -653,22 +854,7 @@ public class DenseVectorFieldMapper extends FieldMapper {
: new byte[dims * elementType.elementBytes];
ByteBuffer byteBuffer = ByteBuffer.wrap(bytes);
double dotProduct = 0f;
int index = 0;
float[] vector = new float[dims];
for (Token token = context.parser().nextToken(); token != Token.END_ARRAY; token = context.parser().nextToken()) {
checkDimensionExceeded(index, context);
ensureExpectedToken(Token.VALUE_NUMBER, token, context.parser());
float value = context.parser().floatValue(true);
vector[index] = value;
elementType.writeValue(byteBuffer, value);
dotProduct += value * value;
index++;
}
checkDimensionMatches(index, context);
elementType.checkVectorBounds(vector);
double dotProduct = elementType.parseKnnVectorToByteBuffer(context, this, byteBuffer);
if (indexCreatedVersion.onOrAfter(Version.V_7_5_0)) {
// encode vector magnitude at the end
float vectorMagnitude = (float) Math.sqrt(dotProduct);
@ -759,7 +945,7 @@ public class DenseVectorFieldMapper extends FieldMapper {
return null; // use default format
} else {
HnswIndexOptions hnswIndexOptions = (HnswIndexOptions) indexOptions;
return new Lucene94HnswVectorsFormat(hnswIndexOptions.m, hnswIndexOptions.efConstruction);
return new Lucene95HnswVectorsFormat(hnswIndexOptions.m, hnswIndexOptions.efConstruction);
}
}
@ -778,6 +964,7 @@ public class DenseVectorFieldMapper extends FieldMapper {
private class IndexedSyntheticFieldLoader implements SourceLoader.SyntheticFieldLoader {
private VectorValues values;
private ByteVectorValues byteVectorValues;
private boolean hasValue;
@Override
@ -788,13 +975,20 @@ public class DenseVectorFieldMapper extends FieldMapper {
@Override
public DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf) throws IOException {
values = leafReader.getVectorValues(name());
if (values == null) {
return null;
if (values != null) {
return docId -> {
hasValue = docId == values.advance(docId);
return hasValue;
};
}
return docId -> {
hasValue = docId == values.advance(docId);
return hasValue;
};
byteVectorValues = leafReader.getByteVectorValues(name());
if (byteVectorValues != null) {
return docId -> {
hasValue = docId == byteVectorValues.advance(docId);
return hasValue;
};
}
return null;
}
@Override
@ -808,8 +1002,15 @@ public class DenseVectorFieldMapper extends FieldMapper {
return;
}
b.startArray(simpleName());
for (float v : values.vectorValue()) {
b.value(v);
if (values != null) {
for (float v : values.vectorValue()) {
b.value(v);
}
} else if (byteVectorValues != null) {
BytesRef vectorValue = byteVectorValues.vectorValue();
for (int i = vectorValue.offset; i < vectorValue.offset + vectorValue.length; i++) {
b.value(vectorValue.bytes[i]);
}
}
b.endArray();
}
@ -850,7 +1051,7 @@ public class DenseVectorFieldMapper extends FieldMapper {
BytesRef ref = values.binaryValue();
ByteBuffer byteBuffer = ByteBuffer.wrap(ref.bytes, ref.offset, ref.length);
for (int dim = 0; dim < dims; dim++) {
b.value(elementType.readValue(byteBuffer));
elementType.readAndWriteValue(byteBuffer, b);
}
b.endArray();
}

11
server/src/main/java/org/elasticsearch/index/mapper/vectors/VectorDVLeafFieldData.java
View File

@ -11,7 +11,6 @@ package org.elasticsearch.index.mapper.vectors;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.VectorValues;
import org.elasticsearch.Version;
import org.elasticsearch.index.fielddata.LeafFieldData;
import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
@ -56,15 +55,9 @@ final class VectorDVLeafFieldData implements LeafFieldData {
public DocValuesScriptFieldFactory getScriptFieldFactory(String name) {
try {
if (indexed) {
VectorValues values = reader.getVectorValues(field);
if (values == VectorValues.EMPTY) {
// There's no way for KnnDenseVectorDocValuesField to reliably differentiate between VectorValues.EMPTY and
// values that can be iterated through. Since VectorValues.EMPTY throws on docID(), pass a null instead.
values = null;
}
return switch (elementType) {
case BYTE -> new ByteKnnDenseVectorDocValuesField(values, name, elementType, dims);
case FLOAT -> new KnnDenseVectorDocValuesField(values, name, elementType, dims);
case BYTE -> new ByteKnnDenseVectorDocValuesField(reader.getByteVectorValues(field), name, dims);
case FLOAT -> new KnnDenseVectorDocValuesField(reader.getVectorValues(field), name, dims);
};
} else {
BinaryDocValues values = DocValues.getBinary(reader, field);

8
server/src/main/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilder.java
View File

@ -407,11 +407,11 @@ public class CombinedFieldsQueryBuilder extends AbstractQueryBuilder<CombinedFie
}
@Override
protected Query newSynonymQuery(TermAndBoost[] terms) {
protected Query newSynonymQuery(String field, TermAndBoost[] terms) {
CombinedFieldQuery.Builder query = new CombinedFieldQuery.Builder();
for (TermAndBoost termAndBoost : terms) {
assert termAndBoost.boost == BoostAttribute.DEFAULT_BOOST;
BytesRef bytes = termAndBoost.term.bytes();
BytesRef bytes = termAndBoost.term;
query.addTerm(bytes);
}
for (FieldAndBoost fieldAndBoost : fields) {
@ -424,8 +424,8 @@ public class CombinedFieldsQueryBuilder extends AbstractQueryBuilder<CombinedFie
@Override
protected Query newTermQuery(Term term, float boost) {
TermAndBoost termAndBoost = new TermAndBoost(term, boost);
return newSynonymQuery(new TermAndBoost[] { termAndBoost });
TermAndBoost termAndBoost = new TermAndBoost(term.bytes(), boost);
return newSynonymQuery(term.field(), new TermAndBoost[] { termAndBoost });
}
@Override

8
server/src/main/java/org/elasticsearch/index/search/MatchQueryParser.java
View File

@ -538,9 +538,9 @@ public class MatchQueryParser {
} else {
// We don't apply prefix on synonyms
final TermAndBoost[] termAndBoosts = current.stream()
.map(t -> new TermAndBoost(t, BoostAttribute.DEFAULT_BOOST))
.map(t -> new TermAndBoost(t.bytes(), BoostAttribute.DEFAULT_BOOST))
.toArray(TermAndBoost[]::new);
q.add(newSynonymQuery(termAndBoosts), operator);
q.add(newSynonymQuery(field, termAndBoosts), operator);
}
}
@ -648,9 +648,9 @@ public class MatchQueryParser {
} else {
// We don't apply prefix on synonyms
final TermAndBoost[] termAndBoosts = Arrays.stream(terms)
.map(t -> new TermAndBoost(t, BoostAttribute.DEFAULT_BOOST))
.map(t -> new TermAndBoost(t.bytes(), BoostAttribute.DEFAULT_BOOST))
.toArray(TermAndBoost[]::new);
queryPos = newSynonymQuery(termAndBoosts);
queryPos = newSynonymQuery(field, termAndBoosts);
}
}
if (queryPos != null) {

4
server/src/main/java/org/elasticsearch/index/search/MultiMatchQueryParser.java
View File

@ -196,10 +196,10 @@ public class MultiMatchQueryParser extends MatchQueryParser {
}
@Override
protected Query newSynonymQuery(TermAndBoost[] terms) {
protected Query newSynonymQuery(String field, TermAndBoost[] terms) {
BytesRef[] values = new BytesRef[terms.length];
for (int i = 0; i < terms.length; i++) {
values[i] = terms[i].term.bytes();
values[i] = terms[i].term;
}
return blendTerms(context, values, tieBreaker, lenient, blendedFields);
}

12
server/src/main/java/org/elasticsearch/script/VectorScoreScriptUtils.java
View File

@ -52,18 +52,18 @@ public class VectorScoreScriptUtils {
public ByteDenseVectorFunction(ScoreScript scoreScript, DenseVectorDocValuesField field, List<Number> queryVector) {
super(scoreScript, field);
DenseVector.checkDimensions(field.get().getDims(), queryVector.size());
float[] vector = new float[queryVector.size()];
this.queryVector = new byte[queryVector.size()];
float[] validateValues = new float[queryVector.size()];
int queryMagnitude = 0;
for (int i = 0; i < queryVector.size(); i++) {
float value = queryVector.get(i).floatValue();
vector[i] = value;
this.queryVector[i] = (byte) value;
final Number number = queryVector.get(i);
byte value = number.byteValue();
this.queryVector[i] = value;
queryMagnitude += value * value;
validateValues[i] = number.floatValue();
}
this.qvMagnitude = (float) Math.sqrt(queryMagnitude);
field.getElementType().checkVectorBounds(vector);
field.getElementType().checkVectorBounds(validateValues);
}
}

1
server/src/main/java/org/elasticsearch/script/field/vectors/ByteKnnDenseVector.java
View File

@ -27,6 +27,7 @@ public class ByteKnnDenseVector implements DenseVector {
@Override
public float[] getVector() {
// TODO it would be really nice if we didn't transform the `byte[]` arrays to `float[]`
if (floatDocVector == null) {
floatDocVector = new float[docVector.length];

12
server/src/main/java/org/elasticsearch/script/field/vectors/ByteKnnDenseVectorDocValuesField.java
View File

@ -8,7 +8,7 @@
package org.elasticsearch.script.field.vectors;
import org.apache.lucene.index.VectorValues;
import org.apache.lucene.index.ByteVectorValues;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.core.Nullable;
import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType;
@ -19,12 +19,12 @@ import java.io.IOException;
import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
public class ByteKnnDenseVectorDocValuesField extends DenseVectorDocValuesField {
protected VectorValues input; // null if no vectors
protected ByteVectorValues input; // null if no vectors
protected BytesRef vector;
protected final int dims;
public ByteKnnDenseVectorDocValuesField(@Nullable VectorValues input, String name, ElementType elementType, int dims) {
super(name, elementType);
public ByteKnnDenseVectorDocValuesField(@Nullable ByteVectorValues input, String name, int dims) {
super(name, ElementType.BYTE);
this.dims = dims;
this.input = input;
}
@ -38,11 +38,11 @@ public class ByteKnnDenseVectorDocValuesField extends DenseVectorDocValuesField
if (currentDoc == NO_MORE_DOCS || docId < currentDoc) {
vector = null;
} else if (docId == currentDoc) {
vector = input.binaryValue();
vector = input.vectorValue();
} else {
currentDoc = input.advance(docId);
if (currentDoc == docId) {
vector = input.binaryValue();
vector = input.vectorValue();
} else {
vector = null;
}

4
server/src/main/java/org/elasticsearch/script/field/vectors/KnnDenseVectorDocValuesField.java
View File

@ -22,8 +22,8 @@ public class KnnDenseVectorDocValuesField extends DenseVectorDocValuesField {
protected float[] vector;
protected final int dims;
public KnnDenseVectorDocValuesField(@Nullable VectorValues input, String name, ElementType elementType, int dims) {
super(name, elementType);
public KnnDenseVectorDocValuesField(@Nullable VectorValues input, String name, int dims) {
super(name, ElementType.FLOAT);
this.dims = dims;
this.input = input;
}

2
server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/QueryToFilterAdapter.java
View File

@ -9,13 +9,13 @@
package org.elasticsearch.search.aggregations.bucket.filter;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.sandbox.search.IndexSortSortedNumericDocValuesRangeQuery;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.BulkScorer;
import org.apache.lucene.search.ConstantScoreQuery;
import org.apache.lucene.search.IndexOrDocValuesQuery;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery;
import org.apache.lucene.search.LeafCollector;
import org.apache.lucene.search.PointRangeQuery;
import org.apache.lucene.search.Query;

91
server/src/main/java/org/elasticsearch/search/internal/ExitableDirectoryReader.java
View File

@ -9,6 +9,7 @@
package org.elasticsearch.search.internal;
import org.apache.lucene.codecs.StoredFieldsReader;
import org.apache.lucene.index.ByteVectorValues;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.FilterDirectoryReader;
import org.apache.lucene.index.FilterLeafReader;
@ -126,6 +127,45 @@ class ExitableDirectoryReader extends FilterDirectoryReader {
return reader;
}
@Override
public ByteVectorValues getByteVectorValues(String field) throws IOException {
ByteVectorValues vectorValues = in.getByteVectorValues(field);
if (vectorValues == null) {
return null;
}
return queryCancellation.isEnabled() ? new ExitableByteVectorValues(queryCancellation, vectorValues) : vectorValues;
}
@Override
public TopDocs searchNearestVectors(String field, BytesRef target, int k, Bits acceptDocs, int visitedLimit) throws IOException {
if (queryCancellation.isEnabled() == false) {
return in.searchNearestVectors(field, target, k, acceptDocs, visitedLimit);
}
// when acceptDocs is null due to no doc deleted, we will instantiate a new one that would
// match all docs to allow timeout checking.
final Bits updatedAcceptDocs = acceptDocs == null ? new Bits.MatchAllBits(maxDoc()) : acceptDocs;
Bits timeoutCheckingAcceptDocs = new Bits() {
private static final int MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK = 10;
private int calls;
@Override
public boolean get(int index) {
if (calls++ % MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK == 0) {
queryCancellation.checkCancelled();
}
return updatedAcceptDocs.get(index);
}
@Override
public int length() {
return updatedAcceptDocs.length();
}
};
return in.searchNearestVectors(field, target, k, timeoutCheckingAcceptDocs, visitedLimit);
}
@Override
public VectorValues getVectorValues(String field) throws IOException {
VectorValues vectorValues = in.getVectorValues(field);
@ -433,6 +473,57 @@ class ExitableDirectoryReader extends FilterDirectoryReader {
}
}
private static class ExitableByteVectorValues extends ByteVectorValues {
private int calls;
private final QueryCancellation queryCancellation;
private final ByteVectorValues in;
private ExitableByteVectorValues(QueryCancellation queryCancellation, ByteVectorValues in) {
this.queryCancellation = queryCancellation;
this.in = in;
}
@Override
public int dimension() {
return in.dimension();
}
@Override
public int size() {
return in.size();
}
@Override
public BytesRef vectorValue() throws IOException {
return in.vectorValue();
}
@Override
public int docID() {
return in.docID();
}
@Override
public int nextDoc() throws IOException {
final int nextDoc = in.nextDoc();
checkAndThrowWithSampling();
return nextDoc;
}
@Override
public int advance(int target) throws IOException {
final int advance = in.advance(target);
checkAndThrowWithSampling();
return advance;
}
private void checkAndThrowWithSampling() {
if ((calls++ & ExitableIntersectVisitor.MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK) == 0) {
this.queryCancellation.checkCancelled();
}
}
}
private static class ExitableVectorValues extends FilterVectorValues {
private int calls;
private final QueryCancellation queryCancellation;

55
server/src/main/java/org/elasticsearch/search/internal/FieldUsageTrackingDirectoryReader.java
View File

@ -10,6 +10,7 @@ package org.elasticsearch.search.internal;
import org.apache.lucene.codecs.StoredFieldsReader;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.ByteVectorValues;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.Fields;
@ -23,6 +24,8 @@ import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.StoredFieldVisitor;
import org.apache.lucene.index.StoredFields;
import org.apache.lucene.index.TermVectors;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.VectorValues;
@ -108,6 +111,21 @@ public class FieldUsageTrackingDirectoryReader extends FilterDirectoryReader {
return f;
}
@Override
public TermVectors termVectors() throws IOException {
TermVectors termVectors = super.termVectors();
return new TermVectors() {
@Override
public Fields get(int doc) throws IOException {
Fields f = termVectors.get(doc);
if (f != null) {
f = new FieldUsageTrackingTermVectorFields(f);
}
return f;
}
};
}
@Override
public PointValues getPointValues(String field) throws IOException {
PointValues pointValues = super.getPointValues(field);
@ -126,6 +144,21 @@ public class FieldUsageTrackingDirectoryReader extends FilterDirectoryReader {
}
}
@Override
public StoredFields storedFields() throws IOException {
StoredFields storedFields = super.storedFields();
return new StoredFields() {
@Override
public void document(int docID, StoredFieldVisitor visitor) throws IOException {
if (visitor instanceof FieldNamesProvidingStoredFieldsVisitor) {
storedFields.document(docID, new FieldUsageFieldsVisitor((FieldNamesProvidingStoredFieldsVisitor) visitor));
} else {
storedFields.document(docID, new FieldUsageStoredFieldVisitor(visitor));
}
}
};
}
@Override
public Terms terms(String field) throws IOException {
Terms terms = super.terms(field);
@ -195,6 +228,24 @@ public class FieldUsageTrackingDirectoryReader extends FilterDirectoryReader {
return vectorValues;
}
@Override
public ByteVectorValues getByteVectorValues(String field) throws IOException {
ByteVectorValues vectorValues = super.getByteVectorValues(field);
if (vectorValues != null) {
notifier.onKnnVectorsUsed(field);
}
return vectorValues;
}
@Override
public TopDocs searchNearestVectors(String field, BytesRef target, int k, Bits acceptDocs, int visitedLimit) throws IOException {
TopDocs topDocs = super.searchNearestVectors(field, target, k, acceptDocs, visitedLimit);
if (topDocs != null) {
notifier.onKnnVectorsUsed(field);
}
return topDocs;
}
@Override
public TopDocs searchNearestVectors(String field, float[] target, int k, Bits acceptDocs, int visitedLimit) throws IOException {
TopDocs topDocs = super.searchNearestVectors(field, target, k, acceptDocs, visitedLimit);
@ -223,8 +274,8 @@ public class FieldUsageTrackingDirectoryReader extends FilterDirectoryReader {
}
@Override
public void visitDocument(int docID, StoredFieldVisitor visitor) throws IOException {
reader.visitDocument(docID, new FieldUsageStoredFieldVisitor(visitor));
public void document(int docID, StoredFieldVisitor visitor) throws IOException {
reader.document(docID, new FieldUsageStoredFieldVisitor(visitor));
}
@Override

1
server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java
View File

@ -44,6 +44,7 @@ public class KnnSearchBuilder implements Writeable, ToXContentFragment, Rewritea
private static final ConstructingObjectParser<KnnSearchBuilder, Void> PARSER = new ConstructingObjectParser<>("knn", args -> {
@SuppressWarnings("unchecked")
// TODO optimize parsing for when BYTE values are provided
List<Float> vector = (List<Float>) args[1];
float[] vectorArray = new float[vector.size()];
for (int i = 0; i < vector.size(); i++) {

68
server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java
View File

@ -12,9 +12,11 @@ import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.KnnVectorQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.Version;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.core.Nullable;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.DenseVectorFieldType;
@ -41,12 +43,22 @@ public class KnnVectorQueryBuilder extends AbstractQueryBuilder<KnnVectorQueryBu
private final String fieldName;
private final float[] queryVector;
private final byte[] byteQueryVector;
private final int numCands;
private final List<QueryBuilder> filterQueries;
public KnnVectorQueryBuilder(String fieldName, float[] queryVector, int numCands) {
this.fieldName = fieldName;
this.queryVector = queryVector;
this.queryVector = Objects.requireNonNull(queryVector);
this.byteQueryVector = null;
this.numCands = numCands;
this.filterQueries = new ArrayList<>();
}
public KnnVectorQueryBuilder(String fieldName, byte[] queryVector, int numCands) {
this.fieldName = fieldName;
this.queryVector = null;
this.byteQueryVector = Objects.requireNonNull(queryVector);
this.numCands = numCands;
this.filterQueries = new ArrayList<>();
}
@ -55,7 +67,13 @@ public class KnnVectorQueryBuilder extends AbstractQueryBuilder<KnnVectorQueryBu
super(in);
this.fieldName = in.readString();
this.numCands = in.readVInt();
this.queryVector = in.readFloatArray();
if (in.getVersion().before(Version.V_8_7_0)) {
this.queryVector = in.readFloatArray();
this.byteQueryVector = null;
} else {
this.queryVector = in.readBoolean() ? in.readFloatArray() : null;
this.byteQueryVector = in.readBoolean() ? in.readByteArray() : null;
}
if (in.getVersion().before(Version.V_8_2_0)) {
this.filterQueries = new ArrayList<>();
} else {
@ -67,10 +85,16 @@ public class KnnVectorQueryBuilder extends AbstractQueryBuilder<KnnVectorQueryBu
return fieldName;
}
@Nullable
public float[] queryVector() {
return queryVector;
}
@Nullable
public byte[] getByteQueryVector() {
return byteQueryVector;
}
public int numCands() {
return numCands;
}
@ -95,7 +119,29 @@ public class KnnVectorQueryBuilder extends AbstractQueryBuilder<KnnVectorQueryBu
protected void doWriteTo(StreamOutput out) throws IOException {
out.writeString(fieldName);
out.writeVInt(numCands);
out.writeFloatArray(queryVector);
if (out.getVersion().onOrAfter(Version.V_8_7_0)) {
boolean queryVectorNotNull = queryVector != null;
out.writeBoolean(queryVectorNotNull);
if (queryVectorNotNull) {
out.writeFloatArray(queryVector);
}
boolean byteVectorNotNull = byteQueryVector != null;
out.writeBoolean(byteVectorNotNull);
if (byteVectorNotNull) {
out.writeByteArray(byteQueryVector);
}
} else {
final float[] f;
if (queryVector != null) {
f = queryVector;
} else {
f = new float[byteQueryVector.length];
for (int i = 0; i < byteQueryVector.length; i++) {
f[i] = byteQueryVector[i];
}
}
out.writeFloatArray(f);
}
if (out.getVersion().onOrAfter(Version.V_8_2_0)) {
writeQueries(out, filterQueries);
}
@ -103,7 +149,10 @@ public class KnnVectorQueryBuilder extends AbstractQueryBuilder<KnnVectorQueryBu
@Override
protected void doXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(NAME).field("field", fieldName).field("vector", queryVector).field("num_candidates", numCands);
builder.startObject(NAME)
.field("field", fieldName)
.field("vector", queryVector != null ? queryVector : byteQueryVector)
.field("num_candidates", numCands);
if (filterQueries.isEmpty() == false) {
builder.startArray("filters");
for (QueryBuilder filterQuery : filterQueries) {
@ -135,7 +184,9 @@ public class KnnVectorQueryBuilder extends AbstractQueryBuilder<KnnVectorQueryBu
rewrittenQueries.add(rewrittenQuery);
}
if (changed) {
return new KnnVectorQueryBuilder(fieldName, queryVector, numCands).addFilterQueries(rewrittenQueries);
return byteQueryVector != null
? new KnnVectorQueryBuilder(fieldName, byteQueryVector, numCands).addFilterQueries(rewrittenQueries)
: new KnnVectorQueryBuilder(fieldName, queryVector, numCands).addFilterQueries(rewrittenQueries);
}
return this;
}
@ -161,18 +212,21 @@ public class KnnVectorQueryBuilder extends AbstractQueryBuilder<KnnVectorQueryBu
Query filterQuery = booleanQuery.clauses().isEmpty() ? null : booleanQuery;
DenseVectorFieldType vectorFieldType = (DenseVectorFieldType) fieldType;
return vectorFieldType.createKnnQuery(queryVector, numCands, filterQuery);
return queryVector != null
? vectorFieldType.createKnnQuery(queryVector, numCands, filterQuery)
: vectorFieldType.createKnnQuery(new BytesRef(byteQueryVector), numCands, filterQuery);
}
@Override
protected int doHashCode() {
return Objects.hash(fieldName, Arrays.hashCode(queryVector), numCands, filterQueries);
return Objects.hash(fieldName, Arrays.hashCode(queryVector), Arrays.hashCode(byteQueryVector), numCands, filterQueries);
}
@Override
protected boolean doEquals(KnnVectorQueryBuilder other) {
return Objects.equals(fieldName, other.fieldName)
&& Arrays.equals(queryVector, other.queryVector)
&& Arrays.equals(byteQueryVector, other.byteQueryVector)
&& numCands == other.numCands
&& Objects.equals(filterQueries, other.filterQueries);
}

31
server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java
View File

@ -13,8 +13,8 @@ import org.apache.lucene.codecs.KnnVectorsFormat;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat;
import org.apache.lucene.codecs.lucene94.Lucene94Codec;
import org.apache.lucene.codecs.lucene94.Lucene94HnswVectorsFormat;
import org.apache.lucene.codecs.lucene95.Lucene95Codec;
import org.apache.lucene.codecs.lucene95.Lucene95HnswVectorsFormat;
import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
@ -80,6 +80,7 @@ import java.util.function.Consumer;
import static org.hamcrest.Matchers.empty;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.hamcrest.Matchers.not;
import static org.hamcrest.Matchers.notNullValue;
@ -261,9 +262,11 @@ public class IndexDiskUsageAnalyzerTests extends ESTestCase {
final IndexDiskUsageStats stats = IndexDiskUsageAnalyzer.analyze(testShardId(), lastCommit(dir), () -> {});
logger.info("--> stats {}", stats);
int dataBytes = numDocs * dimension * Float.BYTES; // size of flat vector data
int indexBytesEstimate = numDocs * Integer.BYTES * Lucene94HnswVectorsFormat.DEFAULT_MAX_CONN * 2; // rough size of HNSW graph
assertTrue(stats.total().getKnnVectorsBytes() > dataBytes + indexBytesEstimate);
long dataBytes = (long) numDocs * dimension * Float.BYTES; // size of flat vector data
long indexBytesEstimate = (long) numDocs * (Lucene95HnswVectorsFormat.DEFAULT_MAX_CONN / 2); // rough size of HNSW graph
assertThat(stats.total().getKnnVectorsBytes(), greaterThan(dataBytes));
long connectionOverhead = stats.total().getKnnVectorsBytes() - dataBytes;
assertThat(connectionOverhead, greaterThan(indexBytesEstimate));
}
}
@ -323,7 +326,7 @@ public class IndexDiskUsageAnalyzerTests extends ESTestCase {
public void testCompletionField() throws Exception {
IndexWriterConfig config = new IndexWriterConfig().setCommitOnClose(true)
.setUseCompoundFile(false)
.setCodec(new Lucene94Codec(Lucene94Codec.Mode.BEST_SPEED) {
.setCodec(new Lucene95Codec(Lucene95Codec.Mode.BEST_SPEED) {
@Override
public PostingsFormat getPostingsFormatForField(String field) {
if (field.startsWith("suggest_")) {
@ -410,25 +413,25 @@ public class IndexDiskUsageAnalyzerTests extends ESTestCase {
enum CodecMode {
BEST_SPEED {
@Override
Lucene94Codec.Mode mode() {
return Lucene94Codec.Mode.BEST_SPEED;
Lucene95Codec.Mode mode() {
return Lucene95Codec.Mode.BEST_SPEED;
}
},
BEST_COMPRESSION {
@Override
Lucene94Codec.Mode mode() {
return Lucene94Codec.Mode.BEST_COMPRESSION;
Lucene95Codec.Mode mode() {
return Lucene95Codec.Mode.BEST_COMPRESSION;
}
};
abstract Lucene94Codec.Mode mode();
abstract Lucene95Codec.Mode mode();
}
static void indexRandomly(Directory directory, CodecMode codecMode, int numDocs, Consumer<Document> addFields) throws IOException {
IndexWriterConfig config = new IndexWriterConfig().setCommitOnClose(true)
.setUseCompoundFile(randomBoolean())
.setCodec(new Lucene94Codec(codecMode.mode()));
.setCodec(new Lucene95Codec(codecMode.mode()));
try (IndexWriter writer = new IndexWriter(directory, config)) {
for (int i = 0; i < numDocs; i++) {
final Document doc = new Document();
@ -636,7 +639,7 @@ public class IndexDiskUsageAnalyzerTests extends ESTestCase {
try (DirectoryReader reader = DirectoryReader.open(source)) {
IndexWriterConfig config = new IndexWriterConfig().setSoftDeletesField(Lucene.SOFT_DELETES_FIELD)
.setUseCompoundFile(randomBoolean())
.setCodec(new Lucene94Codec(mode.mode()) {
.setCodec(new Lucene95Codec(mode.mode()) {
@Override
public PostingsFormat getPostingsFormatForField(String field) {
return new Lucene90PostingsFormat();
@ -649,7 +652,7 @@ public class IndexDiskUsageAnalyzerTests extends ESTestCase {
@Override
public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
return new Lucene94HnswVectorsFormat();
return new Lucene95HnswVectorsFormat();
}
@Override

12
server/src/test/java/org/elasticsearch/index/codec/CodecTests.java
View File

@ -10,7 +10,7 @@ package org.elasticsearch.index.codec;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat;
import org.apache.lucene.codecs.lucene94.Lucene94Codec;
import org.apache.lucene.codecs.lucene95.Lucene95Codec;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
@ -42,21 +42,21 @@ public class CodecTests extends ESTestCase {
public void testResolveDefaultCodecs() throws Exception {
CodecService codecService = createCodecService();
assertThat(codecService.codec("default"), instanceOf(PerFieldMapperCodec.class));
assertThat(codecService.codec("default"), instanceOf(Lucene94Codec.class));
assertThat(codecService.codec("default"), instanceOf(Lucene95Codec.class));
}
public void testDefault() throws Exception {
Codec codec = createCodecService().codec("default");
assertStoredFieldsCompressionEquals(Lucene94Codec.Mode.BEST_SPEED, codec);
assertStoredFieldsCompressionEquals(Lucene95Codec.Mode.BEST_SPEED, codec);
}
public void testBestCompression() throws Exception {
Codec codec = createCodecService().codec("best_compression");
assertStoredFieldsCompressionEquals(Lucene94Codec.Mode.BEST_COMPRESSION, codec);
assertStoredFieldsCompressionEquals(Lucene95Codec.Mode.BEST_COMPRESSION, codec);
}
// write some docs with it, inspect .si to see this was the used compression
private void assertStoredFieldsCompressionEquals(Lucene94Codec.Mode expected, Codec actual) throws Exception {
private void assertStoredFieldsCompressionEquals(Lucene95Codec.Mode expected, Codec actual) throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = newIndexWriterConfig(null);
iwc.setCodec(actual);
@ -68,7 +68,7 @@ public class CodecTests extends ESTestCase {
SegmentReader sr = (SegmentReader) ir.leaves().get(0).reader();
String v = sr.getSegmentInfo().info.getAttribute(Lucene90StoredFieldsFormat.MODE_KEY);
assertNotNull(v);
assertEquals(expected, Lucene94Codec.Mode.valueOf(v));
assertEquals(expected, Lucene95Codec.Mode.valueOf(v));
ir.close();
dir.close();
}

4
server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java
View File

@ -8,7 +8,7 @@
package org.elasticsearch.index.codec;
import org.apache.lucene.codecs.lucene94.Lucene94Codec;
import org.apache.lucene.codecs.lucene95.Lucene95Codec;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.settings.Settings;
@ -74,7 +74,7 @@ public class PerFieldMapperCodecTests extends ESTestCase {
""";
mapperService.merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE);
}
return new PerFieldMapperCodec(Lucene94Codec.Mode.BEST_SPEED, mapperService, BigArrays.NON_RECYCLING_INSTANCE);
return new PerFieldMapperCodec(Lucene95Codec.Mode.BEST_SPEED, mapperService, BigArrays.NON_RECYCLING_INSTANCE);
}
}

4
server/src/test/java/org/elasticsearch/index/engine/CompletionStatsCacheTests.java
View File

@ -8,7 +8,7 @@
package org.elasticsearch.index.engine;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.lucene94.Lucene94Codec;
import org.apache.lucene.codecs.lucene95.Lucene95Codec;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
@ -45,7 +45,7 @@ public class CompletionStatsCacheTests extends ESTestCase {
public void testCompletionStatsCache() throws IOException, InterruptedException {
final IndexWriterConfig indexWriterConfig = newIndexWriterConfig();
final PostingsFormat postingsFormat = new Completion90PostingsFormat();
indexWriterConfig.setCodec(new Lucene94Codec() {
indexWriterConfig.setCodec(new Lucene95Codec() {
@Override
public PostingsFormat getPostingsFormatForField(String field) {
return postingsFormat; // all fields are suggest fields

2
server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java
View File

@ -16,9 +16,9 @@ import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.MultiReader;
import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.sandbox.search.IndexSortSortedNumericDocValuesRangeQuery;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.IndexOrDocValuesQuery;
import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.store.Directory;
import org.elasticsearch.Version;

2
server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java
View File

@ -20,9 +20,9 @@ import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.sandbox.document.HalfFloatPoint;
import org.apache.lucene.sandbox.search.IndexSortSortedNumericDocValuesRangeQuery;
import org.apache.lucene.search.IndexOrDocValuesQuery;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery;
import org.apache.lucene.search.MatchNoDocsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Sort;

45
server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java
View File

@ -12,8 +12,9 @@ import com.carrotsearch.randomizedtesting.generators.RandomPicks;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.KnnVectorsFormat;
import org.apache.lucene.codecs.lucene94.Lucene94HnswVectorsFormat;
import org.apache.lucene.codecs.lucene95.Lucene95HnswVectorsFormat;
import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.KnnByteVectorField;
import org.apache.lucene.document.KnnVectorField;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.search.FieldExistsQuery;
@ -42,8 +43,8 @@ import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.List;
import static org.apache.lucene.codecs.lucene94.Lucene94HnswVectorsFormat.DEFAULT_BEAM_WIDTH;
import static org.apache.lucene.codecs.lucene94.Lucene94HnswVectorsFormat.DEFAULT_MAX_CONN;
import static org.apache.lucene.codecs.lucene95.Lucene95HnswVectorsFormat.DEFAULT_BEAM_WIDTH;
import static org.apache.lucene.codecs.lucene95.Lucene95HnswVectorsFormat.DEFAULT_MAX_CONN;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.instanceOf;
@ -250,19 +251,19 @@ public class DenseVectorFieldMapperTests extends MapperTestCase {
)
);
float[] vector = { (byte) -1, (byte) 1, (byte) 127 };
byte[] vector = { (byte) -1, (byte) 1, (byte) 127 };
ParsedDocument doc1 = mapper.parse(source(b -> b.array("field", vector)));
IndexableField[] fields = doc1.rootDoc().getFields("field");
assertEquals(1, fields.length);
assertThat(fields[0], instanceOf(KnnVectorField.class));
assertThat(fields[0], instanceOf(KnnByteVectorField.class));
KnnVectorField vectorField = (KnnVectorField) fields[0];
vectorField.binaryValue();
KnnByteVectorField vectorField = (KnnByteVectorField) fields[0];
vectorField.vectorValue();
assertEquals(
"Parsed vector is not equal to original.",
new BytesRef(new byte[] { (byte) -1, (byte) 1, (byte) 127 }),
vectorField.binaryValue()
vectorField.vectorValue()
);
assertEquals(similarity.function, vectorField.fieldType().vectorSimilarityFunction());
}
@ -332,9 +333,7 @@ public class DenseVectorFieldMapperTests extends MapperTestCase {
assertNotNull(e.getCause());
assertThat(
e.getCause().getMessage(),
containsString(
"The [cosine] similarity does not support vectors with zero magnitude. Preview of invalid vector: [-0.0, 0.0, 0.0]"
)
containsString("The [cosine] similarity does not support vectors with zero magnitude. Preview of invalid vector: [0, 0, 0]")
);
}
@ -544,7 +543,13 @@ public class DenseVectorFieldMapperTests extends MapperTestCase {
);
assertThat(
e.getCause().getMessage(),
containsString("element_type [byte] vectors only support integers between [-128, 127] but found [128.0] at dim [0];")
containsString("element_type [byte] vectors only support integers between [-128, 127] but found [128] at dim [0];")
);
e = expectThrows(MapperParsingException.class, () -> mapper.parse(source(b -> b.array("field", new float[] { 18.2f, 0, 0 }))));
assertThat(
e.getCause().getMessage(),
containsString("element_type [byte] vectors only support non-decimal values but found decimal value [18.2] at dim [0];")
);
e = expectThrows(
@ -553,7 +558,7 @@ public class DenseVectorFieldMapperTests extends MapperTestCase {
);
assertThat(
e.getCause().getMessage(),
containsString("element_type [byte] vectors only support integers between [-128, 127] but found [-129.0] at dim [2];")
containsString("element_type [byte] vectors only support integers between [-128, 127] but found [-129] at dim [2];")
);
}
@ -693,8 +698,8 @@ public class DenseVectorFieldMapperTests extends MapperTestCase {
Codec codec = codecService.codec("default");
assertThat(codec, instanceOf(PerFieldMapperCodec.class));
KnnVectorsFormat knnVectorsFormat = ((PerFieldMapperCodec) codec).getKnnVectorsFormatForField("field");
assertThat(knnVectorsFormat, instanceOf(Lucene94HnswVectorsFormat.class));
String expectedString = "Lucene94HnswVectorsFormat(name=Lucene94HnswVectorsFormat, maxConn="
assertThat(knnVectorsFormat, instanceOf(Lucene95HnswVectorsFormat.class));
String expectedString = "Lucene95HnswVectorsFormat(name=Lucene95HnswVectorsFormat, maxConn="
+ m
+ ", beamWidth="
+ efConstruction
@ -725,14 +730,12 @@ public class DenseVectorFieldMapperTests extends MapperTestCase {
@Override
public SyntheticSourceExample example(int maxValues) throws IOException {
List<Float> value = randomList(dims, dims, this::randomValue);
Object value = elementType == ElementType.BYTE
? randomList(dims, dims, ESTestCase::randomByte)
: randomList(dims, dims, ESTestCase::randomFloat);
return new SyntheticSourceExample(value, value, this::mapping);
}
private float randomValue() {
return elementType == ElementType.BYTE ? ESTestCase.randomByte() : ESTestCase.randomFloat();
}
private void mapping(XContentBuilder b) throws IOException {
b.field("type", "dense_vector");
b.field("dims", dims);
@ -753,7 +756,7 @@ public class DenseVectorFieldMapperTests extends MapperTestCase {
}
@Override
public List<SyntheticSourceInvalidExample> invalidExample() throws IOException {
public List<SyntheticSourceInvalidExample> invalidExample() {
return List.of();
}
}

4
server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java
View File

@ -8,6 +8,7 @@
package org.elasticsearch.index.mapper.vectors;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.Version;
import org.elasticsearch.index.fielddata.FieldDataContext;
import org.elasticsearch.index.mapper.FieldTypeTestCase;
@ -174,5 +175,8 @@ public class DenseVectorFieldTypeTests extends FieldTypeTestCase {
);
e = expectThrows(IllegalArgumentException.class, () -> cosineField.createKnnQuery(new float[] { 0.0f, 0.0f, 0.0f }, 10, null));
assertThat(e.getMessage(), containsString("The [cosine] similarity does not support vectors with zero magnitude."));
e = expectThrows(IllegalArgumentException.class, () -> cosineField.createKnnQuery(new BytesRef(new byte[] { 0, 0, 0 }), 10, null));
assertThat(e.getMessage(), containsString("The [cosine] similarity does not support vectors with zero magnitude."));
}
}

126
server/src/test/java/org/elasticsearch/index/mapper/vectors/KnnDenseVectorScriptDocValuesTests.java
View File

@ -8,9 +8,9 @@
package org.elasticsearch.index.mapper.vectors;
import org.apache.lucene.index.ByteVectorValues;
import org.apache.lucene.index.VectorValues;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.VectorUtil;
import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType;
import org.elasticsearch.script.field.vectors.ByteKnnDenseVectorDocValuesField;
import org.elasticsearch.script.field.vectors.DenseVector;
@ -30,12 +30,7 @@ public class KnnDenseVectorScriptDocValuesTests extends ESTestCase {
float[][] vectors = { { 1, 1, 1 }, { 1, 1, 2 }, { 1, 1, 3 } };
float[] expectedMagnitudes = { 1.7320f, 2.4495f, 3.3166f };
DenseVectorDocValuesField field = new KnnDenseVectorDocValuesField(
wrap(vectors, ElementType.FLOAT),
"test",
ElementType.FLOAT,
dims
);
DenseVectorDocValuesField field = new KnnDenseVectorDocValuesField(wrap(vectors), "test", dims);
DenseVectorScriptDocValues scriptDocValues = field.toScriptDocValues();
for (int i = 0; i < vectors.length; i++) {
field.setNextDocId(i);
@ -51,12 +46,7 @@ public class KnnDenseVectorScriptDocValuesTests extends ESTestCase {
float[][] vectors = { { 1, 1, 1 }, { 1, 1, 2 }, { 1, 1, 3 } };
float[] expectedMagnitudes = { 1.7320f, 2.4495f, 3.3166f };
DenseVectorDocValuesField field = new ByteKnnDenseVectorDocValuesField(
wrap(vectors, ElementType.BYTE),
"test",
ElementType.BYTE,
dims
);
DenseVectorDocValuesField field = new ByteKnnDenseVectorDocValuesField(wrapBytes(vectors), "test", dims);
DenseVectorScriptDocValues scriptDocValues = field.toScriptDocValues();
for (int i = 0; i < vectors.length; i++) {
field.setNextDocId(i);
@ -70,12 +60,7 @@ public class KnnDenseVectorScriptDocValuesTests extends ESTestCase {
public void testFloatMetadataAndIterator() throws IOException {
int dims = 3;
float[][] vectors = fill(new float[randomIntBetween(1, 5)][dims], ElementType.FLOAT);
DenseVectorDocValuesField field = new KnnDenseVectorDocValuesField(
wrap(vectors, ElementType.FLOAT),
"test",
ElementType.FLOAT,
dims
);
DenseVectorDocValuesField field = new KnnDenseVectorDocValuesField(wrap(vectors), "test", dims);
for (int i = 0; i < vectors.length; i++) {
field.setNextDocId(i);
DenseVector dv = field.get();
@ -94,12 +79,7 @@ public class KnnDenseVectorScriptDocValuesTests extends ESTestCase {
public void testByteMetadataAndIterator() throws IOException {
int dims = 3;
float[][] vectors = fill(new float[randomIntBetween(1, 5)][dims], ElementType.BYTE);
DenseVectorDocValuesField field = new ByteKnnDenseVectorDocValuesField(
wrap(vectors, ElementType.BYTE),
"test",
ElementType.BYTE,
dims
);
DenseVectorDocValuesField field = new ByteKnnDenseVectorDocValuesField(wrapBytes(vectors), "test", dims);
for (int i = 0; i < vectors.length; i++) {
field.setNextDocId(i);
DenseVector dv = field.get();
@ -127,12 +107,7 @@ public class KnnDenseVectorScriptDocValuesTests extends ESTestCase {
public void testFloatMissingValues() throws IOException {
int dims = 3;
float[][] vectors = { { 1, 1, 1 }, { 1, 1, 2 }, { 1, 1, 3 } };
DenseVectorDocValuesField field = new KnnDenseVectorDocValuesField(
wrap(vectors, ElementType.FLOAT),
"test",
ElementType.FLOAT,
dims
);
DenseVectorDocValuesField field = new KnnDenseVectorDocValuesField(wrap(vectors), "test", dims);
DenseVectorScriptDocValues scriptDocValues = field.toScriptDocValues();
field.setNextDocId(3);
@ -146,12 +121,7 @@ public class KnnDenseVectorScriptDocValuesTests extends ESTestCase {
public void testByteMissingValues() throws IOException {
int dims = 3;
float[][] vectors = { { 1, 1, 1 }, { 1, 1, 2 }, { 1, 1, 3 } };
DenseVectorDocValuesField field = new ByteKnnDenseVectorDocValuesField(
wrap(vectors, ElementType.BYTE),
"test",
ElementType.BYTE,
dims
);
DenseVectorDocValuesField field = new ByteKnnDenseVectorDocValuesField(wrapBytes(vectors), "test", dims);
DenseVectorScriptDocValues scriptDocValues = field.toScriptDocValues();
field.setNextDocId(3);
@ -165,12 +135,7 @@ public class KnnDenseVectorScriptDocValuesTests extends ESTestCase {
public void testFloatGetFunctionIsNotAccessible() throws IOException {
int dims = 3;
float[][] vectors = { { 1, 1, 1 }, { 1, 1, 2 }, { 1, 1, 3 } };
DenseVectorDocValuesField field = new KnnDenseVectorDocValuesField(
wrap(vectors, ElementType.FLOAT),
"test",
ElementType.FLOAT,
dims
);
DenseVectorDocValuesField field = new KnnDenseVectorDocValuesField(wrap(vectors), "test", dims);
DenseVectorScriptDocValues scriptDocValues = field.toScriptDocValues();
field.setNextDocId(0);
@ -186,12 +151,7 @@ public class KnnDenseVectorScriptDocValuesTests extends ESTestCase {
public void testByteGetFunctionIsNotAccessible() throws IOException {
int dims = 3;
float[][] vectors = { { 1, 1, 1 }, { 1, 1, 2 }, { 1, 1, 3 } };
DenseVectorDocValuesField field = new ByteKnnDenseVectorDocValuesField(
wrap(vectors, ElementType.BYTE),
"test",
ElementType.BYTE,
dims
);
DenseVectorDocValuesField field = new ByteKnnDenseVectorDocValuesField(wrapBytes(vectors), "test", dims);
DenseVectorScriptDocValues scriptDocValues = field.toScriptDocValues();
field.setNextDocId(0);
@ -206,7 +166,7 @@ public class KnnDenseVectorScriptDocValuesTests extends ESTestCase {
public void testFloatMissingVectorValues() throws IOException {
int dims = 7;
DenseVectorDocValuesField emptyKnn = new KnnDenseVectorDocValuesField(null, "test", ElementType.FLOAT, dims);
DenseVectorDocValuesField emptyKnn = new KnnDenseVectorDocValuesField(null, "test", dims);
emptyKnn.setNextDocId(0);
assertEquals(0, emptyKnn.toScriptDocValues().size());
@ -220,7 +180,7 @@ public class KnnDenseVectorScriptDocValuesTests extends ESTestCase {
public void testByteMissingVectorValues() throws IOException {
int dims = 7;
DenseVectorDocValuesField emptyKnn = new ByteKnnDenseVectorDocValuesField(null, "test", ElementType.BYTE, dims);
DenseVectorDocValuesField emptyKnn = new ByteKnnDenseVectorDocValuesField(null, "test", dims);
emptyKnn.setNextDocId(0);
assertEquals(0, emptyKnn.toScriptDocValues().size());
@ -232,7 +192,50 @@ public class KnnDenseVectorScriptDocValuesTests extends ESTestCase {
assertEquals("Cannot iterate over single valued dense_vector field, use get() instead", e.getMessage());
}
public static VectorValues wrap(float[][] vectors, ElementType elementType) {
public static ByteVectorValues wrapBytes(float[][] vectors) {
return new ByteVectorValues() {
int index = 0;
byte[] byteVector = new byte[vectors[0].length];
@Override
public int dimension() {
return 0;
}
@Override
public int size() {
return vectors.length;
}
@Override
public BytesRef vectorValue() {
for (int i = 0; i < byteVector.length; i++) {
byteVector[i] = (byte) vectors[index][i];
}
return new BytesRef(byteVector);
}
@Override
public int docID() {
return index;
}
@Override
public int nextDoc() {
throw new UnsupportedOperationException();
}
@Override
public int advance(int target) {
if (target >= size()) {
return NO_MORE_DOCS;
}
return index = target;
}
};
}
public static VectorValues wrap(float[][] vectors) {
return new VectorValues() {
int index = 0;
@ -253,17 +256,11 @@ public class KnnDenseVectorScriptDocValuesTests extends ESTestCase {
@Override
public BytesRef binaryValue() {
if (elementType == ElementType.FLOAT) {
ByteBuffer byteBuffer = ByteBuffer.allocate(elementType.elementBytes * vectors[index].length);
for (float value : vectors[index]) {
elementType.writeValue(byteBuffer, value);
}
return new BytesRef(byteBuffer.array());
} else if (elementType == ElementType.BYTE) {
return VectorUtil.toBytesRef(vectors[index]);
} else {
throw new IllegalStateException("unknown element_type [" + elementType + "]");
ByteBuffer byteBuffer = ByteBuffer.allocate(ElementType.FLOAT.elementBytes * vectors[index].length);
for (float value : vectors[index]) {
ElementType.FLOAT.writeValue(byteBuffer, value);
}
return new BytesRef(byteBuffer.array());
}
@Override
@ -283,11 +280,6 @@ public class KnnDenseVectorScriptDocValuesTests extends ESTestCase {
}
return index = target;
}
@Override
public long cost() {
return size();
}
};
}
}

35
server/src/test/java/org/elasticsearch/script/VectorScoreScriptUtilsTests.java
View File

@ -55,12 +55,7 @@ public class VectorScoreScriptUtilsTests extends ESTestCase {
dims,
Version.CURRENT
),
new KnnDenseVectorDocValuesField(
KnnDenseVectorScriptDocValuesTests.wrap(new float[][] { docVector }, ElementType.FLOAT),
"test",
ElementType.FLOAT,
dims
)
new KnnDenseVectorDocValuesField(KnnDenseVectorScriptDocValuesTests.wrap(new float[][] { docVector }), "test", dims)
);
for (DenseVectorDocValuesField field : fields) {
field.setNextDocId(0);
@ -140,12 +135,7 @@ public class VectorScoreScriptUtilsTests extends ESTestCase {
ElementType.BYTE,
dims
),
new ByteKnnDenseVectorDocValuesField(
KnnDenseVectorScriptDocValuesTests.wrap(new float[][] { docVector }, ElementType.BYTE),
"test",
ElementType.BYTE,
dims
)
new ByteKnnDenseVectorDocValuesField(KnnDenseVectorScriptDocValuesTests.wrapBytes(new float[][] { docVector }), "test", dims)
);
for (DenseVectorDocValuesField field : fields) {
field.setNextDocId(0);
@ -233,24 +223,14 @@ public class VectorScoreScriptUtilsTests extends ESTestCase {
dims,
Version.CURRENT
),
new KnnDenseVectorDocValuesField(
KnnDenseVectorScriptDocValuesTests.wrap(new float[][] { docVector }, ElementType.FLOAT),
"field2",
ElementType.FLOAT,
dims
),
new KnnDenseVectorDocValuesField(KnnDenseVectorScriptDocValuesTests.wrap(new float[][] { docVector }), "field2", dims),
new ByteBinaryDenseVectorDocValuesField(
BinaryDenseVectorScriptDocValuesTests.wrap(new float[][] { docVector }, ElementType.BYTE, Version.CURRENT),
"field3",
ElementType.BYTE,
dims
),
new ByteKnnDenseVectorDocValuesField(
KnnDenseVectorScriptDocValuesTests.wrap(new float[][] { docVector }, ElementType.BYTE),
"field4",
ElementType.BYTE,
dims
)
new ByteKnnDenseVectorDocValuesField(KnnDenseVectorScriptDocValuesTests.wrapBytes(new float[][] { docVector }), "field4", dims)
);
for (DenseVectorDocValuesField field : fields) {
field.setNextDocId(0);
@ -388,12 +368,7 @@ public class VectorScoreScriptUtilsTests extends ESTestCase {
ElementType.BYTE,
dims
),
new ByteKnnDenseVectorDocValuesField(
KnnDenseVectorScriptDocValuesTests.wrap(new float[][] { docVector }, ElementType.BYTE),
"test",
ElementType.BYTE,
dims
)
new ByteKnnDenseVectorDocValuesField(KnnDenseVectorScriptDocValuesTests.wrapBytes(new float[][] { docVector }), "test", dims)
);
for (DenseVectorDocValuesField field : fields) {

245
server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java Normal file
View File

@ -0,0 +1,245 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
package org.elasticsearch.search.vectors;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.KnnByteVectorQuery;
import org.apache.lucene.search.KnnVectorQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.Version;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.core.Tuple;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
import org.elasticsearch.index.query.MatchNoneQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.SearchExecutionContext;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.test.AbstractBuilderTestCase;
import org.elasticsearch.test.AbstractQueryTestCase;
import org.elasticsearch.test.VersionUtils;
import org.elasticsearch.xcontent.XContentBuilder;
import org.elasticsearch.xcontent.XContentFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.instanceOf;
abstract class AbstractKnnVectorQueryBuilderTestCase extends AbstractQueryTestCase<KnnVectorQueryBuilder> {
private static final String VECTOR_FIELD = "vector";
private static final String VECTOR_ALIAS_FIELD = "vector_alias";
private static final int VECTOR_DIMENSION = 3;
abstract DenseVectorFieldMapper.ElementType elementType();
@Override
protected void initializeAdditionalMappings(MapperService mapperService) throws IOException {
XContentBuilder builder = XContentFactory.jsonBuilder()
.startObject()
.startObject("properties")
.startObject(VECTOR_FIELD)
.field("type", "dense_vector")
.field("dims", VECTOR_DIMENSION)
.field("index", true)
.field("similarity", "l2_norm")
.field("element_type", elementType())
.endObject()
.startObject(VECTOR_ALIAS_FIELD)
.field("type", "alias")
.field("path", VECTOR_FIELD)
.endObject()
.endObject()
.endObject();
mapperService.merge(
MapperService.SINGLE_MAPPING_NAME,
new CompressedXContent(Strings.toString(builder)),
MapperService.MergeReason.MAPPING_UPDATE
);
}
@Override
protected KnnVectorQueryBuilder doCreateTestQueryBuilder() {
String fieldName = randomBoolean() ? VECTOR_FIELD : VECTOR_ALIAS_FIELD;
byte[] byteVector = new byte[VECTOR_DIMENSION];
float[] vector = new float[VECTOR_DIMENSION];
for (int i = 0; i < vector.length; i++) {
vector[i] = randomFloat();
byteVector[i] = randomByte();
}
int numCands = randomIntBetween(1, 1000);
KnnVectorQueryBuilder queryBuilder = switch (elementType()) {
case BYTE -> new KnnVectorQueryBuilder(fieldName, byteVector, numCands);
case FLOAT -> new KnnVectorQueryBuilder(fieldName, vector, numCands);
};
if (randomBoolean()) {
List<QueryBuilder> filters = new ArrayList<>();
int numFilters = randomIntBetween(1, 5);
for (int i = 0; i < numFilters; i++) {
String filterFieldName = randomBoolean() ? KEYWORD_FIELD_NAME : TEXT_FIELD_NAME;
filters.add(QueryBuilders.termQuery(filterFieldName, randomAlphaOfLength(10)));
}
queryBuilder.addFilterQueries(filters);
}
return queryBuilder;
}
@Override
protected void doAssertLuceneQuery(KnnVectorQueryBuilder queryBuilder, Query query, SearchExecutionContext context) throws IOException {
switch (elementType()) {
case FLOAT -> assertTrue(query instanceof KnnVectorQuery);
case BYTE -> assertTrue(query instanceof KnnByteVectorQuery);
}
BooleanQuery.Builder builder = new BooleanQuery.Builder();
for (QueryBuilder qb : queryBuilder.filterQueries()) {
builder.add(qb.toQuery(context), BooleanClause.Occur.FILTER);
}
BooleanQuery booleanQuery = builder.build();
Query filterQuery = booleanQuery.clauses().isEmpty() ? null : booleanQuery;
// The field should always be resolved to the concrete field
Query knnVectorQueryBuilt = switch (elementType()) {
case BYTE -> new KnnByteVectorQuery(
VECTOR_FIELD,
new BytesRef(queryBuilder.getByteQueryVector()),
queryBuilder.numCands(),
filterQuery
);
case FLOAT -> new KnnVectorQuery(VECTOR_FIELD, queryBuilder.queryVector(), queryBuilder.numCands(), filterQuery);
};
assertEquals(query, knnVectorQueryBuilt);
}
public void testWrongDimension() {
SearchExecutionContext context = createSearchExecutionContext();
KnnVectorQueryBuilder query = new KnnVectorQueryBuilder(VECTOR_FIELD, new float[] { 1.0f, 2.0f }, 10);
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> query.doToQuery(context));
assertThat(e.getMessage(), containsString("the query vector has a different dimension [2] than the index vectors [3]"));
}
public void testNonexistentField() {
SearchExecutionContext context = createSearchExecutionContext();
KnnVectorQueryBuilder query = new KnnVectorQueryBuilder("nonexistent", new float[] { 1.0f, 1.0f, 1.0f }, 10);
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> query.doToQuery(context));
assertThat(e.getMessage(), containsString("field [nonexistent] does not exist in the mapping"));
}
public void testWrongFieldType() {
SearchExecutionContext context = createSearchExecutionContext();
KnnVectorQueryBuilder query = new KnnVectorQueryBuilder(
AbstractBuilderTestCase.KEYWORD_FIELD_NAME,
new float[] { 1.0f, 1.0f, 1.0f },
10
);
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> query.doToQuery(context));
assertThat(e.getMessage(), containsString("[knn] queries are only supported on [dense_vector] fields"));
}
@Override
public void testValidOutput() {
KnnVectorQueryBuilder query = new KnnVectorQueryBuilder(VECTOR_FIELD, new float[] { 1.0f, 2.0f, 3.0f }, 10);
String expected = """
{
"knn" : {
"field" : "vector",
"vector" : [
1.0,
2.0,
3.0
],
"num_candidates" : 10
}
}""";
assertEquals(expected, query.toString());
}
@Override
public void testMustRewrite() throws IOException {
SearchExecutionContext context = createSearchExecutionContext();
context.setAllowUnmappedFields(true);
TermQueryBuilder termQuery = new TermQueryBuilder("unmapped_field", 42);
KnnVectorQueryBuilder query = new KnnVectorQueryBuilder(VECTOR_FIELD, new float[] { 1.0f, 2.0f, 3.0f }, VECTOR_DIMENSION);
query.addFilterQuery(termQuery);
IllegalStateException e = expectThrows(IllegalStateException.class, () -> query.toQuery(context));
assertEquals("Rewrite first", e.getMessage());
QueryBuilder rewrittenQuery = query.rewrite(context);
assertThat(rewrittenQuery, instanceOf(MatchNoneQueryBuilder.class));
}
public void testBWCVersionSerialization() throws IOException {
float[] bwcFloat = new float[VECTOR_DIMENSION];
KnnVectorQueryBuilder query = createTestQueryBuilder();
if (query.queryVector() != null) {
bwcFloat = query.queryVector();
} else {
for (int i = 0; i < query.getByteQueryVector().length; i++) {
bwcFloat[i] = query.getByteQueryVector()[i];
}
}
KnnVectorQueryBuilder queryWithNoFilters = new KnnVectorQueryBuilder(query.getFieldName(), bwcFloat, query.numCands()).queryName(
query.queryName()
).boost(query.boost());
KnnVectorQueryBuilder queryNoByteQuery = new KnnVectorQueryBuilder(query.getFieldName(), bwcFloat, query.numCands()).queryName(
query.queryName()
).boost(query.boost()).addFilterQueries(query.filterQueries());
Version newVersion = VersionUtils.randomVersionBetween(random(), Version.V_8_7_0, Version.CURRENT);
Version beforeByteQueryVersion = VersionUtils.randomVersionBetween(random(), Version.V_8_2_0, Version.V_8_6_0);
Version beforeFilterVersion = VersionUtils.randomVersionBetween(random(), Version.V_8_0_0, Version.V_8_1_0);
assertSerialization(query, newVersion);
assertSerialization(queryNoByteQuery, beforeByteQueryVersion);
assertSerialization(queryWithNoFilters, beforeFilterVersion);
for (var tuple : List.of(
Tuple.tuple(beforeByteQueryVersion, queryNoByteQuery),
Tuple.tuple(beforeFilterVersion, queryWithNoFilters)
)) {
try (BytesStreamOutput output = new BytesStreamOutput()) {
output.setVersion(tuple.v1());
output.writeNamedWriteable(query);
try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry())) {
in.setVersion(tuple.v1());
KnnVectorQueryBuilder deserializedQuery = (KnnVectorQueryBuilder) in.readNamedWriteable(QueryBuilder.class);
assertEquals(tuple.v2(), deserializedQuery);
assertEquals(tuple.v2().hashCode(), deserializedQuery.hashCode());
}
}
}
}
@Override
public void testUnknownObjectException() {
assumeTrue("Test isn't relevant, since query is never parsed from xContent", false);
}
@Override
public void testFromXContent() {
assumeTrue("Test isn't relevant, since query is never parsed from xContent", false);
}
@Override
public void testUnknownField() {
assumeTrue("Test isn't relevant, since query is never parsed from xContent", false);
}
}

18
server/src/test/java/org/elasticsearch/search/vectors/KnnByteVectorQueryBuilderTests.java Normal file
View File

@ -0,0 +1,18 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
package org.elasticsearch.search.vectors;
import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
public class KnnByteVectorQueryBuilderTests extends AbstractKnnVectorQueryBuilderTestCase {
@Override
DenseVectorFieldMapper.ElementType elementType() {
return DenseVectorFieldMapper.ElementType.BYTE;
}
}

193
server/src/test/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilderTests.java
View File

@ -8,196 +8,11 @@
package org.elasticsearch.search.vectors;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.KnnVectorQuery;
import org.apache.lucene.search.Query;
import org.elasticsearch.Version;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.query.MatchNoneQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.SearchExecutionContext;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.test.AbstractBuilderTestCase;
import org.elasticsearch.test.AbstractQueryTestCase;
import org.elasticsearch.test.VersionUtils;
import org.elasticsearch.xcontent.XContentBuilder;
import org.elasticsearch.xcontent.XContentFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.instanceOf;
public class KnnVectorQueryBuilderTests extends AbstractQueryTestCase<KnnVectorQueryBuilder> {
private static final String VECTOR_FIELD = "vector";
private static final String VECTOR_ALIAS_FIELD = "vector_alias";
private static final int VECTOR_DIMENSION = 3;
import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
public class KnnVectorQueryBuilderTests extends AbstractKnnVectorQueryBuilderTestCase {
@Override
protected void initializeAdditionalMappings(MapperService mapperService) throws IOException {
XContentBuilder builder = XContentFactory.jsonBuilder()
.startObject()
.startObject("properties")
.startObject(VECTOR_FIELD)
.field("type", "dense_vector")
.field("dims", VECTOR_DIMENSION)
.field("index", true)
.field("similarity", "l2_norm")
.endObject()
.startObject(VECTOR_ALIAS_FIELD)
.field("type", "alias")
.field("path", VECTOR_FIELD)
.endObject()
.endObject()
.endObject();
mapperService.merge(
MapperService.SINGLE_MAPPING_NAME,
new CompressedXContent(Strings.toString(builder)),
MapperService.MergeReason.MAPPING_UPDATE
);
}
@Override
protected KnnVectorQueryBuilder doCreateTestQueryBuilder() {
String fieldName = randomBoolean() ? VECTOR_FIELD : VECTOR_ALIAS_FIELD;
float[] vector = new float[VECTOR_DIMENSION];
for (int i = 0; i < vector.length; i++) {
vector[i] = randomFloat();
}
int numCands = randomIntBetween(1, 1000);
KnnVectorQueryBuilder queryBuilder = new KnnVectorQueryBuilder(fieldName, vector, numCands);
if (randomBoolean()) {
List<QueryBuilder> filters = new ArrayList<>();
int numFilters = randomIntBetween(1, 5);
for (int i = 0; i < numFilters; i++) {
String filterFieldName = randomBoolean() ? KEYWORD_FIELD_NAME : TEXT_FIELD_NAME;
filters.add(QueryBuilders.termQuery(filterFieldName, randomAlphaOfLength(10)));
}
queryBuilder.addFilterQueries(filters);
}
return queryBuilder;
}
@Override
protected void doAssertLuceneQuery(KnnVectorQueryBuilder queryBuilder, Query query, SearchExecutionContext context) throws IOException {
assertTrue(query instanceof KnnVectorQuery);
KnnVectorQuery knnVectorQuery = (KnnVectorQuery) query;
BooleanQuery.Builder builder = new BooleanQuery.Builder();
for (QueryBuilder qb : queryBuilder.filterQueries()) {
builder.add(qb.toQuery(context), BooleanClause.Occur.FILTER);
}
BooleanQuery booleanQuery = builder.build();
Query filterQuery = booleanQuery.clauses().isEmpty() ? null : booleanQuery;
// The field should always be resolved to the concrete field
Query knnVectorQueryBuilt = new KnnVectorQuery(VECTOR_FIELD, queryBuilder.queryVector(), queryBuilder.numCands(), filterQuery);
assertEquals(knnVectorQuery, knnVectorQueryBuilt);
}
public void testWrongDimension() {
SearchExecutionContext context = createSearchExecutionContext();
KnnVectorQueryBuilder query = new KnnVectorQueryBuilder(VECTOR_FIELD, new float[] { 1.0f, 2.0f }, 10);
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> query.doToQuery(context));
assertThat(e.getMessage(), containsString("the query vector has a different dimension [2] than the index vectors [3]"));
}
public void testNonexistentField() {
SearchExecutionContext context = createSearchExecutionContext();
KnnVectorQueryBuilder query = new KnnVectorQueryBuilder("nonexistent", new float[] { 1.0f, 1.0f, 1.0f }, 10);
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> query.doToQuery(context));
assertThat(e.getMessage(), containsString("field [nonexistent] does not exist in the mapping"));
}
public void testWrongFieldType() {
SearchExecutionContext context = createSearchExecutionContext();
KnnVectorQueryBuilder query = new KnnVectorQueryBuilder(
AbstractBuilderTestCase.KEYWORD_FIELD_NAME,
new float[] { 1.0f, 1.0f, 1.0f },
10
);
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> query.doToQuery(context));
assertThat(e.getMessage(), containsString("[knn] queries are only supported on [dense_vector] fields"));
}
@Override
public void testValidOutput() {
KnnVectorQueryBuilder query = new KnnVectorQueryBuilder(VECTOR_FIELD, new float[] { 1.0f, 2.0f, 3.0f }, 10);
String expected = """
{
"knn" : {
"field" : "vector",
"vector" : [
1.0,
2.0,
3.0
],
"num_candidates" : 10
}
}""";
assertEquals(expected, query.toString());
}
@Override
public void testMustRewrite() throws IOException {
SearchExecutionContext context = createSearchExecutionContext();
context.setAllowUnmappedFields(true);
TermQueryBuilder termQuery = new TermQueryBuilder("unmapped_field", 42);
KnnVectorQueryBuilder query = new KnnVectorQueryBuilder(VECTOR_FIELD, new float[] { 1.0f, 2.0f, 3.0f }, VECTOR_DIMENSION);
query.addFilterQuery(termQuery);
IllegalStateException e = expectThrows(IllegalStateException.class, () -> query.toQuery(context));
assertEquals("Rewrite first", e.getMessage());
QueryBuilder rewrittenQuery = query.rewrite(context);
assertThat(rewrittenQuery, instanceOf(MatchNoneQueryBuilder.class));
}
public void testOldVersionSerialization() throws IOException {
KnnVectorQueryBuilder query = createTestQueryBuilder();
KnnVectorQueryBuilder queryWithNoFilters = new KnnVectorQueryBuilder(query.getFieldName(), query.queryVector(), query.numCands());
queryWithNoFilters.queryName(query.queryName()).boost(query.boost());
Version newVersion = VersionUtils.randomVersionBetween(random(), Version.V_8_2_0, Version.CURRENT);
Version oldVersion = VersionUtils.randomVersionBetween(random(), Version.V_8_0_0, Version.V_8_1_0);
assertSerialization(query, newVersion);
assertSerialization(queryWithNoFilters, oldVersion);
try (BytesStreamOutput output = new BytesStreamOutput()) {
output.setVersion(newVersion);
output.writeNamedWriteable(query);
try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry())) {
in.setVersion(oldVersion);
KnnVectorQueryBuilder deserializedQuery = (KnnVectorQueryBuilder) in.readNamedWriteable(QueryBuilder.class);
assertEquals(queryWithNoFilters, deserializedQuery);
assertEquals(queryWithNoFilters.hashCode(), deserializedQuery.hashCode());
}
}
}
@Override
public void testUnknownObjectException() throws IOException {
// Test isn't relevant, since query is never parsed from xContent
}
@Override
public void testFromXContent() throws IOException {
// Test isn't relevant, since query is never parsed from xContent
}
@Override
public void testUnknownField() throws IOException {
// Test isn't relevant, since query is never parsed from xContent
DenseVectorFieldMapper.ElementType elementType() {
return DenseVectorFieldMapper.ElementType.FLOAT;
}
}

24
x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/frozen/RewriteCachingDirectoryReader.java
View File

@ -7,6 +7,7 @@
package org.elasticsearch.index.engine.frozen;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.ByteVectorValues;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
@ -22,11 +23,14 @@ import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.StoredFieldVisitor;
import org.apache.lucene.index.StoredFields;
import org.apache.lucene.index.TermVectors;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.VectorValues;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import java.io.IOException;
import java.util.Comparator;
@ -216,11 +220,21 @@ final class RewriteCachingDirectoryReader extends DirectoryReader {
throw new UnsupportedOperationException();
}
@Override
public ByteVectorValues getByteVectorValues(String field) throws IOException {
throw new UnsupportedOperationException();
}
@Override
public TopDocs searchNearestVectors(String field, float[] target, int k, Bits acceptDocs, int visitedLimit) throws IOException {
throw new UnsupportedOperationException();
}
@Override
public TopDocs searchNearestVectors(String field, BytesRef target, int k, Bits acceptDocs, int visitedLimit) throws IOException {
throw new UnsupportedOperationException();
}
@Override
public FieldInfos getFieldInfos() {
return fieldInfos;
@ -249,6 +263,16 @@ final class RewriteCachingDirectoryReader extends DirectoryReader {
throw new UnsupportedOperationException();
}
@Override
public TermVectors termVectors() throws IOException {
throw new UnsupportedOperationException();
}
@Override
public StoredFields storedFields() throws IOException {
throw new UnsupportedOperationException();
}
@Override
public int numDocs() {
return numDocs;

45
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java
View File

@ -8,6 +8,7 @@ package org.elasticsearch.xpack.core.security.authz.accesscontrol;
import org.apache.lucene.codecs.StoredFieldsReader;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.ByteVectorValues;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
@ -21,7 +22,9 @@ import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.StoredFieldVisitor;
import org.apache.lucene.index.StoredFields;
import org.apache.lucene.index.TermState;
import org.apache.lucene.index.TermVectors;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.VectorValues;
@ -162,6 +165,34 @@ public final class FieldSubsetReader extends SequentialStoredFieldsLeafReader {
return f.iterator().hasNext() ? f : null;
}
@Override
public TermVectors termVectors() throws IOException {
TermVectors termVectors = super.termVectors();
return new TermVectors() {
@Override
public Fields get(int doc) throws IOException {
Fields f = termVectors.get(doc);
if (f == null) {
return null;
}
f = new FieldFilterFields(f);
// we need to check for emptyness, so we can return null:
return f.iterator().hasNext() ? f : null;
}
};
}
@Override
public StoredFields storedFields() throws IOException {
StoredFields storedFields = super.storedFields();
return new StoredFields() {
@Override
public void document(int docID, StoredFieldVisitor visitor) throws IOException {
storedFields.document(docID, new FieldSubsetStoredFieldVisitor(visitor));
}
};
}
/** Filter a map by a {@link CharacterRunAutomaton} that defines the fields to retain. */
@SuppressWarnings("unchecked")
static Map<String, Object> filter(Map<String, ?> map, CharacterRunAutomaton includeAutomaton, int initialState) {
@ -287,6 +318,16 @@ public final class FieldSubsetReader extends SequentialStoredFieldsLeafReader {
return hasField(field) ? super.searchNearestVectors(field, target, k, acceptDocs, visitedLimit) : null;
}
@Override
public ByteVectorValues getByteVectorValues(String field) throws IOException {
return hasField(field) ? super.getByteVectorValues(field) : null;
}
@Override
public TopDocs searchNearestVectors(String field, BytesRef target, int k, Bits acceptDocs, int visitedLimit) throws IOException {
return hasField(field) ? super.searchNearestVectors(field, target, k, acceptDocs, visitedLimit) : null;
}
// we share core cache keys (for e.g. fielddata)
@Override
@ -310,8 +351,8 @@ public final class FieldSubsetReader extends SequentialStoredFieldsLeafReader {
}
@Override
public void visitDocument(int docID, StoredFieldVisitor visitor) throws IOException {
reader.visitDocument(docID, new FieldSubsetStoredFieldVisitor(visitor));
public void document(int docID, StoredFieldVisitor visitor) throws IOException {
reader.document(docID, new FieldSubsetStoredFieldVisitor(visitor));
}
@Override

145
x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java
View File

@ -11,6 +11,7 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.IntPoint;
import org.apache.lucene.document.KnnByteVectorField;
import org.apache.lucene.document.KnnVectorField;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
@ -20,6 +21,7 @@ import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.ByteVectorValues;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
@ -214,6 +216,39 @@ public class FieldSubsetReaderTests extends ESTestCase {
IOUtils.close(ir, iw, dir);
}
public void testKnnByteVectors() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(null);
IndexWriter iw = new IndexWriter(dir, iwc);
Document doc = new Document();
doc.add(new KnnByteVectorField("fieldA", new BytesRef(new byte[] { 1, 2, 3 })));
doc.add(new KnnByteVectorField("fieldB", new BytesRef(new byte[] { 3, 2, 1 })));
iw.addDocument(doc);
DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")));
LeafReader leafReader = ir.leaves().get(0).reader();
// Check that fieldA behaves as normal
ByteVectorValues vectorValues = leafReader.getByteVectorValues("fieldA");
assertEquals(3, vectorValues.dimension());
assertEquals(1, vectorValues.size());
assertEquals(0, vectorValues.nextDoc());
assertNotNull(vectorValues.binaryValue());
assertNotNull(vectorValues.vectorValue());
TopDocs topDocs = leafReader.searchNearestVectors("fieldA", new BytesRef(new byte[] { 1, 1, 1 }), 5, null, Integer.MAX_VALUE);
assertNotNull(topDocs);
assertEquals(1, topDocs.scoreDocs.length);
// Check that we can't see fieldB
assertNull(leafReader.getByteVectorValues("fieldB"));
assertNull(leafReader.searchNearestVectors("fieldB", new BytesRef(new byte[] { 1, 1, 1 }), 5, null, Integer.MAX_VALUE));
TestUtil.checkReader(ir);
IOUtils.close(ir, iw, dir);
}
/**
* test filtering two stored fields (string)
*/
@ -232,10 +267,16 @@ public class FieldSubsetReaderTests extends ESTestCase {
DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")));
// see only one field
Document d2 = ir.document(0);
assertEquals(1, d2.getFields().size());
assertEquals("testA", d2.get("fieldA"));
{
Document d2 = ir.document(0);
assertEquals(1, d2.getFields().size());
assertEquals("testA", d2.get("fieldA"));
}
{
Document d2 = ir.storedFields().document(0);
assertEquals(1, d2.getFields().size());
assertEquals("testA", d2.get("fieldA"));
}
TestUtil.checkReader(ir);
IOUtils.close(ir, iw, dir);
}
@ -258,10 +299,16 @@ public class FieldSubsetReaderTests extends ESTestCase {
DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")));
// see only one field
Document d2 = ir.document(0);
assertEquals(1, d2.getFields().size());
assertEquals(new BytesRef("testA"), d2.getBinaryValue("fieldA"));
{
Document d2 = ir.document(0);
assertEquals(1, d2.getFields().size());
assertEquals(new BytesRef("testA"), d2.getBinaryValue("fieldA"));
}
{
Document d2 = ir.storedFields().document(0);
assertEquals(1, d2.getFields().size());
assertEquals(new BytesRef("testA"), d2.getBinaryValue("fieldA"));
}
TestUtil.checkReader(ir);
IOUtils.close(ir, iw, dir);
}
@ -284,10 +331,16 @@ public class FieldSubsetReaderTests extends ESTestCase {
DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")));
// see only one field
Document d2 = ir.document(0);
assertEquals(1, d2.getFields().size());
assertEquals(1, d2.getField("fieldA").numericValue());
{
Document d2 = ir.document(0);
assertEquals(1, d2.getFields().size());
assertEquals(1, d2.getField("fieldA").numericValue());
}
{
Document d2 = ir.storedFields().document(0);
assertEquals(1, d2.getFields().size());
assertEquals(1, d2.getField("fieldA").numericValue());
}
TestUtil.checkReader(ir);
IOUtils.close(ir, iw, dir);
}
@ -310,10 +363,16 @@ public class FieldSubsetReaderTests extends ESTestCase {
DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")));
// see only one field
Document d2 = ir.document(0);
assertEquals(1, d2.getFields().size());
assertEquals(1L, d2.getField("fieldA").numericValue());
{
Document d2 = ir.document(0);
assertEquals(1, d2.getFields().size());
assertEquals(1L, d2.getField("fieldA").numericValue());
}
{
Document d2 = ir.storedFields().document(0);
assertEquals(1, d2.getFields().size());
assertEquals(1L, d2.getField("fieldA").numericValue());
}
TestUtil.checkReader(ir);
IOUtils.close(ir, iw, dir);
}
@ -336,10 +395,16 @@ public class FieldSubsetReaderTests extends ESTestCase {
DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")));
// see only one field
Document d2 = ir.document(0);
assertEquals(1, d2.getFields().size());
assertEquals(1F, d2.getField("fieldA").numericValue());
{
Document d2 = ir.document(0);
assertEquals(1, d2.getFields().size());
assertEquals(1F, d2.getField("fieldA").numericValue());
}
{
Document d2 = ir.storedFields().document(0);
assertEquals(1, d2.getFields().size());
assertEquals(1F, d2.getField("fieldA").numericValue());
}
TestUtil.checkReader(ir);
IOUtils.close(ir, iw, dir);
}
@ -362,10 +427,16 @@ public class FieldSubsetReaderTests extends ESTestCase {
DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")));
// see only one field
Document d2 = ir.document(0);
assertEquals(1, d2.getFields().size());
assertEquals(1D, d2.getField("fieldA").numericValue());
{
Document d2 = ir.document(0);
assertEquals(1, d2.getFields().size());
assertEquals(1D, d2.getField("fieldA").numericValue());
}
{
Document d2 = ir.storedFields().document(0);
assertEquals(1, d2.getFields().size());
assertEquals(1D, d2.getField("fieldA").numericValue());
}
TestUtil.checkReader(ir);
IOUtils.close(ir, iw, dir);
}
@ -624,10 +695,16 @@ public class FieldSubsetReaderTests extends ESTestCase {
DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(automaton));
// see only one field
Document d2 = ir.document(0);
assertEquals(1, d2.getFields().size());
assertEquals("{\"fieldA\":\"testA\"}", d2.getBinaryValue(SourceFieldMapper.NAME).utf8ToString());
{
Document d2 = ir.document(0);
assertEquals(1, d2.getFields().size());
assertEquals("{\"fieldA\":\"testA\"}", d2.getBinaryValue(SourceFieldMapper.NAME).utf8ToString());
}
{
Document d2 = ir.storedFields().document(0);
assertEquals(1, d2.getFields().size());
assertEquals("{\"fieldA\":\"testA\"}", d2.getBinaryValue(SourceFieldMapper.NAME).utf8ToString());
}
TestUtil.checkReader(ir);
IOUtils.close(ir, iw, dir);
}
@ -1025,11 +1102,17 @@ public class FieldSubsetReaderTests extends ESTestCase {
// see no vectors
assertNull(segmentReader.getTermVectors(0));
assertNull(segmentReader.termVectors().get(0));
// see no stored fields
Document document = segmentReader.document(0);
assertEquals(0, document.getFields().size());
{
Document document = segmentReader.document(0);
assertEquals(0, document.getFields().size());
}
{
Document document = segmentReader.storedFields().document(0);
assertEquals(0, document.getFields().size());
}
TestUtil.checkReader(ir);
IOUtils.close(ir, iw, dir);
}

2
x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java
View File

@ -11,8 +11,8 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.LongPoint;
import org.apache.lucene.document.SortedNumericDocValuesField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.sandbox.search.IndexSortSortedNumericDocValuesRangeQuery;
import org.apache.lucene.search.IndexOrDocValuesQuery;
import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery;
import org.apache.lucene.search.MatchNoDocsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.util.BytesRef;