[8.11] ESQL: Introduce language versioning to REST API (#106824)

docs/changelog/106824.yaml

@@ -0,0 +1,5 @@
+pr: 106824
+summary: "ESQL: Introduce language versioning to REST API"
+area: ES|QL
+type: enhancement
+issues: []

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequest.java

@@ -28,6 +28,7 @@
 import org.elasticsearch.xpack.esql.parser.ContentLocation;
 import org.elasticsearch.xpack.esql.parser.TypedParamValue;
 import org.elasticsearch.xpack.esql.plugin.QueryPragmas;
+import org.elasticsearch.xpack.esql.version.EsqlVersion;
 
 import java.io.IOException;
 import java.time.ZoneId;
@@ -57,6 +58,7 @@ public class EsqlQueryRequest extends ActionRequest implements CompositeIndicesR
         PARAM_PARSER.declareString(constructorArg(), TYPE);
     }
 
+    private static final ParseField ESQL_VERSION_FIELD = new ParseField("version");
     private static final ParseField QUERY_FIELD = new ParseField("query");
     private static final ParseField COLUMNAR_FIELD = new ParseField("columnar");
     private static final ParseField TIME_ZONE_FIELD = new ParseField("time_zone");
@@ -67,13 +69,15 @@ public class EsqlQueryRequest extends ActionRequest implements CompositeIndicesR
 
     private static final ObjectParser<EsqlQueryRequest, Void> PARSER = objectParser(EsqlQueryRequest::new);
 
+    private String esqlVersion;
     private String query;
     private boolean columnar;
     private ZoneId zoneId;
     private Locale locale;
     private QueryBuilder filter;
     private QueryPragmas pragmas = new QueryPragmas(Settings.EMPTY);
     private List<TypedParamValue> params = List.of();
+    private boolean onSnapshotBuild = Build.current().isSnapshot();
 
     public EsqlQueryRequest(StreamInput in) throws IOException {
         super(in);
@@ -82,17 +86,50 @@ public EsqlQueryRequest(StreamInput in) throws IOException {
     @Override
     public ActionRequestValidationException validate() {
         ActionRequestValidationException validationException = null;
+        if (Strings.hasText(esqlVersion) == false) {
+            // TODO: make this required
+            // "https://github.com/elastic/elasticsearch/issues/104890"
+            // validationException = addValidationError(invalidVersion("is required"), validationException);
+        } else {
+            EsqlVersion version = EsqlVersion.parse(esqlVersion);
+            if (version == null) {
+                validationException = addValidationError(invalidVersion("has invalid value [" + esqlVersion + "]"), validationException);
+            } else if (version == EsqlVersion.SNAPSHOT && onSnapshotBuild == false) {
+                validationException = addValidationError(
+                    invalidVersion("with value [" + esqlVersion + "] only allowed in snapshot builds"),
+                    validationException
+                );
+            }
+        }
         if (Strings.hasText(query) == false) {
-            validationException = addValidationError("[query] is required", validationException);
+            validationException = addValidationError("[" + QUERY_FIELD + "] is required", validationException);
         }
-        if (Build.current().isSnapshot() == false && pragmas.isEmpty() == false) {
-            validationException = addValidationError("[pragma] only allowed in snapshot builds", validationException);
+        if (onSnapshotBuild == false && pragmas.isEmpty() == false) {
+            validationException = addValidationError("[" + PRAGMA_FIELD + "] only allowed in snapshot builds", validationException);
         }
         return validationException;
     }
 
+    private static String invalidVersion(String reason) {
+        return "["
+            + ESQL_VERSION_FIELD
+            + "] "
+            + reason
+            + ", latest available version is ["
+            + EsqlVersion.latestReleased().versionStringWithoutEmoji()
+            + "]";
+    }
+
     public EsqlQueryRequest() {}
 
+    public void esqlVersion(String esqlVersion) {
+        this.esqlVersion = esqlVersion;
+    }
+
+    public String esqlVersion() {
+        return esqlVersion;
+    }
+
     public void query(String query) {
         this.query = query;
     }
@@ -155,6 +192,7 @@ public static EsqlQueryRequest fromXContent(XContentParser parser) {
 
     private static ObjectParser<EsqlQueryRequest, Void> objectParser(Supplier<EsqlQueryRequest> supplier) {
         ObjectParser<EsqlQueryRequest, Void> parser = new ObjectParser<>("esql/query", false, supplier);
+        parser.declareString(EsqlQueryRequest::esqlVersion, ESQL_VERSION_FIELD);
         parser.declareString(EsqlQueryRequest::query, QUERY_FIELD);
         parser.declareBoolean(EsqlQueryRequest::columnar, COLUMNAR_FIELD);
         parser.declareString((request, zoneId) -> request.zoneId(ZoneId.of(zoneId)), TIME_ZONE_FIELD);
@@ -255,4 +293,8 @@ static org.elasticsearch.xcontent.XContentLocation fromProto(ContentLocation fro
         return new org.elasticsearch.xcontent.XContentLocation(fromProto.lineNumber, fromProto.columnNumber);
     }
 
+    // Setter for tests
+    void onSnapshotBuild(boolean onSnapshotBuild) {
+        this.onSnapshotBuild = onSnapshotBuild;
+    }
 }

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestBuilder.java

@@ -24,6 +24,11 @@ public EsqlQueryRequestBuilder(ElasticsearchClient client, EsqlQueryAction actio
         this(client, action, new EsqlQueryRequest());
     }
 
+    public EsqlQueryRequestBuilder esqlVersion(String esqlVersion) {
+        request.esqlVersion(esqlVersion);
+        return this;
+    }
+
     public EsqlQueryRequestBuilder query(String query) {
         request.query(query);
         return this;

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/version/EsqlVersion.java

@@ -0,0 +1,111 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.version;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.VersionId;
+
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+public enum EsqlVersion implements VersionId<EsqlVersion> {
+    /**
+     * Breaking changes go here until the next version is released.
+     */
+    SNAPSHOT(Integer.MAX_VALUE, 12, 99, "📷"),
+    ROCKET(2024, 4, "🚀");
+
+    static final Map<String, EsqlVersion> VERSION_MAP_WITH_AND_WITHOUT_EMOJI = versionMapWithAndWithoutEmoji();
+
+    private static Map<String, EsqlVersion> versionMapWithAndWithoutEmoji() {
+        Map<String, EsqlVersion> stringToVersion = new LinkedHashMap<>(EsqlVersion.values().length * 2);
+
+        for (EsqlVersion version : EsqlVersion.values()) {
+            putVersionCheckNoDups(stringToVersion, version.versionStringWithoutEmoji(), version);
+            putVersionCheckNoDups(stringToVersion, version.toString(), version);
+        }
+
+        return stringToVersion;
+    }
+
+    private static void putVersionCheckNoDups(Map<String, EsqlVersion> stringToVersion, String versionString, EsqlVersion version) {
+        EsqlVersion existingVersionForKey = stringToVersion.put(versionString, version);
+        if (existingVersionForKey != null) {
+            throw new IllegalArgumentException("Duplicate esql version with version string [" + versionString + "]");
+        }
+    }
+
+    /**
+     * Accepts a version string with the emoji suffix or without it.
+     * E.g. both "2024.04.01.🚀" and "2024.04.01" will be interpreted as {@link EsqlVersion#ROCKET}.
+     */
+    public static EsqlVersion parse(String versionString) {
+        return VERSION_MAP_WITH_AND_WITHOUT_EMOJI.get(versionString);
+    }
+
+    public static EsqlVersion latestReleased() {
+        return Arrays.stream(EsqlVersion.values()).filter(v -> v != SNAPSHOT).max(Comparator.comparingInt(EsqlVersion::id)).get();
+    }
+
+    private int year;
+    private byte month;
+    private byte revision;
+    private String emoji;
+
+    EsqlVersion(int year, int month, String emoji) {
+        this(year, month, 1, emoji);
+    }
+
+    EsqlVersion(int year, int month, int revision, String emoji) {
+        if ((1 <= revision && revision <= 99) == false) {
+            throw new IllegalArgumentException("Version revision number must be between 1 and 99 but was [" + revision + "]");
+        }
+        if ((1 <= month && month <= 12) == false) {
+            throw new IllegalArgumentException("Version month must be between 1 and 12 but was [" + month + "]");
+        }
+        if ((emoji.codePointCount(0, emoji.length()) == 1) == false) {
+            throw new IllegalArgumentException("Version emoji must be a single unicode character but was [" + emoji + "]");
+        }
+        this.year = year;
+        this.month = (byte) month;
+        this.revision = (byte) revision;
+        this.emoji = emoji;
+    }
+
+    public int year() {
+        return year;
+    }
+
+    public byte month() {
+        return month;
+    }
+
+    public byte revision() {
+        return revision;
+    }
+
+    public String emoji() {
+        return emoji;
+    }
+
+    public String versionStringWithoutEmoji() {
+        return this == SNAPSHOT ? "snapshot" : Strings.format("%d.%02d.%02d", year, month, revision);
+    }
+
+    @Override
+    public String toString() {
+        return versionStringWithoutEmoji() + "." + emoji;
+    }
+
+    @Override
+    public int id() {
+        return this == SNAPSHOT ? Integer.MAX_VALUE : (10000 * year + 100 * month + revision);
+    }
+}

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestTests.java

@@ -24,6 +24,8 @@
 import org.elasticsearch.xcontent.XContentParserConfiguration;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.esql.parser.TypedParamValue;
+import org.elasticsearch.xpack.esql.version.EsqlVersion;
+import org.elasticsearch.xpack.esql.version.EsqlVersionTests;
 
 import java.io.IOException;
 import java.time.ZoneId;
@@ -45,21 +47,24 @@ public void testParseFields() throws IOException {
         ZoneId zoneId = randomZone();
         Locale locale = randomLocale(random());
         QueryBuilder filter = randomQueryBuilder();
+        EsqlVersion esqlVersion = randomFrom(EsqlVersion.values());
 
         List<TypedParamValue> params = randomParameters();
         boolean hasParams = params.isEmpty() == false;
         StringBuilder paramsString = paramsString(params, hasParams);
         String json = String.format(Locale.ROOT, """
             {
+                "version": "%s",
                 "query": "%s",
                 "columnar": %s,
                 "time_zone": "%s",
                 "locale": "%s",
                 "filter": %s
-                %s""", query, columnar, zoneId, locale.toLanguageTag(), filter, paramsString);
+                %s""", esqlVersion, query, columnar, zoneId, locale.toLanguageTag(), filter, paramsString);
 
         EsqlQueryRequest request = parseEsqlQueryRequest(json);
 
+        assertEquals(esqlVersion.toString(), request.esqlVersion());
         assertEquals(query, request.query());
         assertEquals(columnar, request.columnar());
         assertEquals(zoneId, request.zoneId());
@@ -87,15 +92,122 @@ public void testRejectUnknownFields() {
             }""", "unknown field [asdf]");
     }
 
-    public void testMissingQueryIsNotValidation() throws IOException {
-        EsqlQueryRequest request = parseEsqlQueryRequest("""
+    public void testKnownVersionIsValid() throws IOException {
+        for (EsqlVersion version : EsqlVersion.values()) {
+            String validVersionString = randomBoolean() ? version.versionStringWithoutEmoji() : version.toString();
+
+            String json = String.format(Locale.ROOT, """
+                {
+                    "version": "%s",
+                    "query": "ROW x = 1"
+                }
+                """, validVersionString);
+
+            EsqlQueryRequest request = parseEsqlQueryRequest(json);
+            assertNull(request.validate());
+        }
+    }
+
+    public void testUnknownVersionIsNotValid() throws IOException {
+        String invalidVersionString = EsqlVersionTests.randomInvalidVersionString();
+
+        String json = String.format(Locale.ROOT, """
+            {
+                "version": "%s",
+                "query": "ROW x = 1"
+            }
+            """, invalidVersionString);
+
+        EsqlQueryRequest request = parseEsqlQueryRequest(json);
+        assertNotNull(request.validate());
+        assertThat(
+            request.validate().getMessage(),
+            containsString(
+                "[version] has invalid value ["
+                    + invalidVersionString
+                    + "], latest available version is ["
+                    + EsqlVersion.latestReleased().versionStringWithoutEmoji()
+                    + "]"
+            )
+        );
+    }
+
+    public void testSnapshotVersionIsOnlyValidOnSnapshot() throws IOException {
+        String esqlVersion = randomBoolean() ? "snapshot" : "snapshot.📷";
+        String json = String.format(Locale.ROOT, """
+            {
+                "version": "%s",
+                "query": "ROW x = 1"
+            }
+            """, esqlVersion);
+
+        EsqlQueryRequest request = parseEsqlQueryRequest(json);
+        request.onSnapshotBuild(true);
+        assertNull(request.validate());
+
+        request.onSnapshotBuild(false);
+        assertNotNull(request.validate());
+        assertThat(
+            request.validate().getMessage(),
+            containsString(
+                "[version] with value ["
+                    + esqlVersion
+                    + "] only allowed in snapshot builds, latest available version is ["
+                    + EsqlVersion.latestReleased().versionStringWithoutEmoji()
+                    + "]"
+            )
+        );
+    }
+
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/104890")
+    public void testMissingVersionIsNotValid() throws IOException {
+        String missingVersion = randomBoolean() ? "" : ", \"version\": \"\"";
+        String json = String.format(Locale.ROOT, """
+            {
+                "columnar": true,
+                "query": "row x = 1"
+                %s
+            }""", missingVersion);
+
+        EsqlQueryRequest request = parseEsqlQueryRequest(json);
+        assertNotNull(request.validate());
+        assertThat(
+            request.validate().getMessage(),
+            containsString(
+                "[version] is required, latest available version is [" + EsqlVersion.latestReleased().versionStringWithoutEmoji() + "]"
+            )
+        );
+    }
+
+    public void testMissingQueryIsNotValid() throws IOException {
+        String json = """
             {
-                "time_zone": "Z"
-            }""");
+                "columnar": true,
+                "version": "snapshot"
+            }""";
+        EsqlQueryRequest request = parseEsqlQueryRequest(json);
         assertNotNull(request.validate());
         assertThat(request.validate().getMessage(), containsString("[query] is required"));
     }
 
+    public void testPragmasOnlyValidOnSnapshot() throws IOException {
+        String json = """
+            {
+                "version": "2024.04.01",
+                "query": "ROW x = 1",
+                "pragma": {"foo": "bar"}
+            }
+            """;
+
+        EsqlQueryRequest request = parseEsqlQueryRequest(json);
+        request.onSnapshotBuild(true);
+        assertNull(request.validate());
+
+        request.onSnapshotBuild(false);
+        assertNotNull(request.validate());
+        assertThat(request.validate().getMessage(), containsString("[pragma] only allowed in snapshot builds"));
+    }
+
     public void testTask() throws IOException {
         String query = randomAlphaOfLength(10);
         int id = randomInt();