Merge branch 'main' into native_int7_overflow

docs/changelog/108088.yaml

@@ -0,0 +1,5 @@
+pr: 108088
+summary: Add a SIMD (AVX2) optimised vector distance function for int7 on x64
+area: "Search"
+type: enhancement
+issues: []

docs/changelog/108459.yaml

@@ -0,0 +1,6 @@
+pr: 108459
+summary: Do not use global ordinals strategy if the leaf reader context cannot be
+  obtained
+area: Machine Learning
+type: bug
+issues: []

docs/reference/esql/functions/date-time-functions.asciidoc

@@ -21,4 +21,4 @@ include::layout/date_extract.asciidoc[]
 include::layout/date_format.asciidoc[]
 include::layout/date_parse.asciidoc[]
 include::layout/date_trunc.asciidoc[]
-include::now.asciidoc[]
+include::layout/now.asciidoc[]

docs/reference/esql/functions/description/now.asciidoc

@@ -0,0 +1,5 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Description*
+
+Returns current date and time.

docs/reference/esql/functions/examples/now.asciidoc

@@ -0,0 +1,22 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Examples*
+
+[source.merge.styled,esql]
+----
+include::{esql-specs}/date.csv-spec[tag=docsNow]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/date.csv-spec[tag=docsNow-result]
+|===
+To retrieve logs from the last hour:
+[source.merge.styled,esql]
+----
+include::{esql-specs}/date.csv-spec[tag=docsNowWhere]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/date.csv-spec[tag=docsNowWhere-result]
+|===
+

docs/reference/esql/functions/kibana/definition/now.json

@@ -0,0 +1,16 @@
+{
+  "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.",
+  "type" : "eval",
+  "name" : "now",
+  "description" : "Returns current date and time.",
+  "signatures" : [
+    {
+      "params" : [ ],
+      "returnType" : "datetime"
+    }
+  ],
+  "examples" : [
+    "ROW current_date = NOW()",
+    "FROM sample_data\n| WHERE @timestamp > NOW() - 1 hour"
+  ]
+}

docs/reference/esql/functions/kibana/docs/now.md

@@ -0,0 +1,10 @@
+<!--
+This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+-->
+
+### NOW
+Returns current date and time.
+
+```
+ROW current_date = NOW()
+```

docs/reference/esql/functions/layout/now.asciidoc

@@ -0,0 +1,15 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+[discrete]
+[[esql-now]]
+=== `NOW`
+
+*Syntax*
+
+[.text-center]
+image::esql/functions/signature/now.svg[Embedded,opts=inline]
+
+include::../parameters/now.asciidoc[]
+include::../description/now.asciidoc[]
+include::../types/now.asciidoc[]
+include::../examples/now.asciidoc[]

docs/reference/esql/functions/parameters/now.asciidoc

@@ -0,0 +1,3 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Parameters*

docs/reference/esql/functions/types/now.asciidoc

@@ -0,0 +1,9 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Supported types*
+
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+result
+datetime
+|===

libs/native/libraries/build.gradle

@@ -18,7 +18,7 @@ configurations {
 }
 
 var zstdVersion = "1.5.5"
-var vecVersion = "1.0.6"
+var vecVersion = "1.0.8"
 
 repositories {
   exclusiveContent {

libs/native/src/main/java/org/elasticsearch/nativeaccess/PosixNativeAccess.java

@@ -45,7 +45,15 @@ public Optional<VectorSimilarityFunctions> getVectorSimilarityFunctions() {
     }
 
     static boolean isNativeVectorLibSupported() {
-        return Runtime.version().feature() >= 21 && isMacOrLinuxAarch64() && checkEnableSystemProperty();
+        return Runtime.version().feature() >= 21 && (isMacOrLinuxAarch64() || isLinuxAmd64()) && checkEnableSystemProperty();
+    }
+
+    /**
+     * Returns true iff the architecture is x64 (amd64) and the OS Linux (the OS we currently support for the native lib).
+     */
+    static boolean isLinuxAmd64() {
+        String name = System.getProperty("os.name");
+        return (name.startsWith("Linux")) && System.getProperty("os.arch").equals("amd64");
     }
 
     /** Returns true iff the OS is Mac or Linux, and the architecture is aarch64. */

libs/native/src/test/java/org/elasticsearch/nativeaccess/VectorSimilarityFunctionsTests.java

@@ -37,7 +37,9 @@ public boolean supported() {
         var arch = System.getProperty("os.arch");
         var osName = System.getProperty("os.name");
 
-        if (jdkVersion >= 21 && arch.equals("aarch64") && (osName.startsWith("Mac") || osName.equals("Linux"))) {
+        if (jdkVersion >= 21
+            && ((arch.equals("aarch64") && (osName.startsWith("Mac") || osName.equals("Linux")))
+                || (arch.equals("amd64") && osName.equals("Linux")))) {
             assertThat(vectorSimilarityFunctions, isPresent());
             return true;
         } else {

libs/vec/native/Dockerfile

@@ -4,6 +4,7 @@ RUN apt update
 RUN apt install -y gcc g++ openjdk-17-jdk
 COPY . /workspace
 WORKDIR /workspace
-RUN ./gradlew --quiet --console=plain clean vecSharedLibrary
+RUN ./gradlew --quiet --console=plain clean buildSharedLibrary
+RUN strip --strip-unneeded build/output/libvec.so
 
-CMD cat build/libs/vec/shared/libvec.so
+CMD cat build/output/libvec.so

libs/vec/native/build.gradle

@@ -12,9 +12,10 @@ var os = org.gradle.internal.os.OperatingSystem.current()
 // To update this library run publish_vec_binaries.sh  ( or ./gradlew vecSharedLibrary )
 // Or
 // For local development, build the docker image with:
-//   docker build --platform linux/arm64 --progress=plain .
+//   docker build --platform linux/arm64 --progress=plain . (for aarch64)
+//   docker build --platform linux/amd64 --progress=plain . (for x64)
 // Grab the image id from the console output, then, e.g.
-//   docker run 9c9f36564c148b275aeecc42749e7b4580ded79dcf51ff6ccc008c8861e7a979 > build/libs/vec/shared/libvec.so
+//   docker run 9c9f36564c148b275aeecc42749e7b4580ded79dcf51ff6ccc008c8861e7a979 > build/libs/vec/shared/$arch/libvec.so
 //
 // To run tests and benchmarks on a locally built libvec,
 //  1. Temporarily comment out the download in libs/native/library/build.gradle
@@ -30,26 +31,83 @@ var os = org.gradle.internal.os.OperatingSystem.current()
 
 group = 'org.elasticsearch'
 
+def platformName = System.getProperty("os.arch");
+
 model {
+  platforms {
+    aarch64 {
+      architecture "aarch64"
+    }
+    amd64 {
+      architecture "x86-64"
+    }
+  }
   toolChains {
     gcc(Gcc) {
       target("aarch64") {
         cCompiler.executable = "/usr/bin/gcc"
+        cCompiler.withArguments { args -> args.addAll(["-O3", "-std=c99", "-march=armv8-a"]) }
+      }
+      target("amd64") {
+        cCompiler.executable = "/usr/bin/gcc"
+        cCompiler.withArguments { args -> args.addAll(["-O3", "-std=c99", "-march=core-avx2", "-Wno-incompatible-pointer-types"]) }
       }
     }
-    clang(Clang)
-  }
-  platforms {
-    aarch64 {
-      architecture "aarch64"
+    cl(VisualCpp) {
+      eachPlatform { toolchain ->
+        def platform = toolchain.getPlatform()
+        if (platform.name == "x64") {
+          cCompiler.withArguments { args -> args.addAll(["/O2", "/LD", "-march=core-avx2"]) }
+        }
+      }
+    }
+    clang(Clang) {
+      target("amd64") {
+        cCompiler.withArguments { args -> args.addAll(["-O3", "-std=c99", "-march=core-avx2"]) }
+      }
     }
   }
   components {
     vec(NativeLibrarySpec) {
       targetPlatform "aarch64"
-      binaries.withType(SharedLibraryBinarySpec) {
-        cCompiler.args "-O3", "-std=c99", "-march=armv8-a"
+      targetPlatform "amd64"
+
+      sources {
+        c {
+          source {
+            srcDir "src/vec/c/${platformName}/"
+            include "*.c"
+          }
+          exportedHeaders {
+            srcDir "src/vec/headers/"
+          }
+        }
+      }
+    }
+  }
+}
+
+tasks.register('buildSharedLibrary') {
+  description = 'Assembles native shared library for the host architecture'
+  if (platformName.equals("aarch64")) {
+    dependsOn tasks.vecAarch64SharedLibrary
+    doLast {
+      copy {
+        from tasks.linkVecAarch64SharedLibrary.outputs.files.files
+        into layout.buildDirectory.dir('output');
+        duplicatesStrategy = 'INCLUDE'
+      }
+    }
+  } else if (platformName.equals("amd64")) {
+    dependsOn tasks.vecAmd64SharedLibrary
+    doLast {
+      copy {
+        from tasks.linkVecAmd64SharedLibrary.outputs.files.files
+        into layout.buildDirectory.dir('output');
+        duplicatesStrategy = 'INCLUDE'
       }
     }
+  } else {
+    throw new GradleException("Unsupported platform: " + platformName)
   }
 }

libs/vec/native/publish_vec_binaries.sh

@@ -19,7 +19,7 @@ if [ -z "$ARTIFACTORY_API_KEY" ]; then
   exit 1;
 fi
 
-VERSION="1.0.6"
+VERSION="1.0.8"
 ARTIFACTORY_REPOSITORY="${ARTIFACTORY_REPOSITORY:-https://artifactory.elastic.dev/artifactory/elasticsearch-native/}"
 TEMP=$(mktemp -d)
 
@@ -29,16 +29,22 @@ if curl -sS -I --fail --location "${ARTIFACTORY_REPOSITORY}/org/elasticsearch/ve
 fi
 
 echo 'Building Darwin binary...'
-./gradlew --quiet --console=plain vecSharedLibrary
+./gradlew --quiet --console=plain vecAarch64SharedLibrary
 
 echo 'Building Linux binary...'
 DOCKER_IMAGE=$(docker build --platform linux/arm64 --quiet .)
-docker run $DOCKER_IMAGE > build/libs/vec/shared/libvec.so
+docker run $DOCKER_IMAGE > build/libs/vec/shared/aarch64/libvec.so
+
+echo 'Building Linux x64 binary...'
+DOCKER_IMAGE=$(docker build --platform linux/amd64 --quiet .)
+docker run --platform linux/amd64 $DOCKER_IMAGE > build/libs/vec/shared/amd64/libvec.so
 
 mkdir -p $TEMP/darwin-aarch64
 mkdir -p $TEMP/linux-aarch64
-cp build/libs/vec/shared/libvec.dylib $TEMP/darwin-aarch64/
-cp build/libs/vec/shared/libvec.so $TEMP/linux-aarch64/
+mkdir -p $TEMP/linux-x64
+cp build/libs/vec/shared/aarch64/libvec.dylib $TEMP/darwin-aarch64/
+cp build/libs/vec/shared/aarch64/libvec.so $TEMP/linux-aarch64/
+cp build/libs/vec/shared/amd64/libvec.so $TEMP/linux-x64/
 
 echo 'Uploading to Artifactory...'
 (cd $TEMP && zip -rq - .) | curl -sS -X PUT -H "X-JFrog-Art-Api: ${ARTIFACTORY_API_KEY}" --data-binary @- --location "${ARTIFACTORY_REPOSITORY}/org/elasticsearch/vec/${VERSION}/vec-${VERSION}.zip"

libs/vec/native/src/vec/c/vec.c → libs/vec/native/src/vec/c/aarch64/vec.c

@@ -121,7 +121,7 @@ static inline int32_t sqr7u_inner(int8_t *a, int8_t *b, size_t dims) {
 EXPORT int32_t sqr7u(int8_t* a, int8_t* b, size_t dims) {
     int32_t res = 0;
     int i = 0;
-    if (i > SQR7U_STRIDE_BYTES_LEN) {
+    if (dims > SQR7U_STRIDE_BYTES_LEN) {
         i += dims & ~(SQR7U_STRIDE_BYTES_LEN - 1);
         res = sqr7u_inner(a, b, i);
     }