From 03669a757454ea337c32e3f7baee3672f139c968 Mon Sep 17 00:00:00 2001
From: Mohammad Qureshi <47198598+qreshi@users.noreply.github.com>
Date: Tue, 2 Apr 2024 11:47:47 -0700
Subject: [PATCH] Add DerivedFieldMapper and support parsing it in mappings
 (#12569)

Adds a DerivedFieldMapper to support the Derived Fields feature enhancement as well as updating the mapper parsing logic to recognize and currently parse derived fields in the mappings.

---------

Signed-off-by: Mohammad Qureshi <qreshi@amazon.com>
Signed-off-by: Rishabh Maurya <rishabhmaurya05@gmail.com>
Co-authored-by: Rishabh Maurya <rishabhmaurya05@gmail.com>
(cherry picked from commit 8ca3e6ad51d73b6b89cb88eca8f4c759f708e59b)
---
 CHANGELOG.md                                  |   1 +
 .../index/mapper/DerivedFieldMapper.java      | 129 ++++++++
 .../opensearch/index/mapper/ObjectMapper.java |  85 +++++-
 .../index/mapper/ParametrizedFieldMapper.java |   6 +-
 .../org/opensearch/indices/IndicesModule.java |   2 +
 .../mapper/DerivedFieldMapperQueryTests.java  | 279 ++++++++++++++++++
 .../index/mapper/DerivedFieldMapperTests.java | 117 ++++++++
 .../index/mapper/ObjectMapperTests.java       |  47 +++
 .../index/mapper/MapperServiceTestCase.java   |  21 +-
 .../aggregations/AggregatorTestCase.java      |   2 +
 10 files changed, 686 insertions(+), 3 deletions(-)
 create mode 100644 server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java
 create mode 100644 server/src/test/java/org/opensearch/index/mapper/DerivedFieldMapperQueryTests.java
 create mode 100644 server/src/test/java/org/opensearch/index/mapper/DerivedFieldMapperTests.java

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0151648cc16b9..b39edb7f5c937 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -37,6 +37,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add additional handling in SearchTemplateRequest when simulate is set to true ([#11591](https://github.com/opensearch-project/OpenSearch/pull/11591))
 - Introduce cluster level setting `cluster.index.restrict.replication.type` to prevent replication type setting override during index creations([#11583](https://github.com/opensearch-project/OpenSearch/pull/11583))
 - Add match_only_text field that is optimized for storage by trading off positional queries performance ([#6836](https://github.com/opensearch-project/OpenSearch/pull/11039))
+- Derived fields support to derive field values at query time without indexing ([#12569](https://github.com/opensearch-project/OpenSearch/pull/12569))
 
 ### Dependencies
 - Bumps jetty version to 9.4.52.v20230823 to fix GMS-2023-1857 ([#9822](https://github.com/opensearch-project/OpenSearch/pull/9822))
diff --git a/server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java
new file mode 100644
index 0000000000000..b448487a4f810
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java
@@ -0,0 +1,129 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.mapper;
+
+import org.apache.lucene.index.IndexableField;
+import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.script.Script;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.function.Function;
+
+/**
+ * A field mapper for derived fields
+ *
+ * @opensearch.internal
+ */
+public class DerivedFieldMapper extends ParametrizedFieldMapper {
+
+    public static final String CONTENT_TYPE = "derived";
+
+    private static DerivedFieldMapper toType(FieldMapper in) {
+        return (DerivedFieldMapper) in;
+    }
+
+    /**
+     * Builder for this field mapper
+     *
+     * @opensearch.internal
+     */
+    public static class Builder extends ParametrizedFieldMapper.Builder {
+        // TODO: The type of parameter may change here if the actual underlying FieldType object is needed
+        private final Parameter<String> type = Parameter.stringParam("type", false, m -> toType(m).type, "text");
+
+        private final Parameter<Script> script = new Parameter<>(
+            "script",
+            false,
+            () -> null,
+            (n, c, o) -> o == null ? null : Script.parse(o),
+            m -> toType(m).script
+        ).setSerializerCheck((id, ic, value) -> value != null);
+
+        public Builder(String name) {
+            super(name);
+        }
+
+        @Override
+        protected List<Parameter<?>> getParameters() {
+            return Arrays.asList(type, script);
+        }
+
+        @Override
+        public DerivedFieldMapper build(BuilderContext context) {
+            FieldMapper fieldMapper = DerivedFieldSupportedTypes.getFieldMapperFromType(type.getValue(), name, context);
+            Function<Object, IndexableField> fieldFunction = DerivedFieldSupportedTypes.getIndexableFieldGeneratorType(
+                type.getValue(),
+                name
+            );
+            DerivedFieldType ft = new DerivedFieldType(
+                buildFullName(context),
+                type.getValue(),
+                script.getValue(),
+                fieldMapper,
+                fieldFunction
+            );
+            return new DerivedFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo.build(), this);
+        }
+    }
+
+    public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n));
+    private final String type;
+    private final Script script;
+
+    protected DerivedFieldMapper(
+        String simpleName,
+        MappedFieldType mappedFieldType,
+        MultiFields multiFields,
+        CopyTo copyTo,
+        Builder builder
+    ) {
+        super(simpleName, mappedFieldType, multiFields, copyTo);
+        this.type = builder.type.getValue();
+        this.script = builder.script.getValue();
+    }
+
+    @Override
+    public DerivedFieldType fieldType() {
+        return (DerivedFieldType) super.fieldType();
+    }
+
+    @Override
+    protected void parseCreateField(ParseContext context) throws IOException {
+        // Leaving this empty as the parsing should be handled via the Builder when root object is parsed.
+        // The context would not contain anything in this case since the DerivedFieldMapper is not indexed or stored.
+        throw new UnsupportedOperationException("should not be invoked");
+    }
+
+    @Override
+    public ParametrizedFieldMapper.Builder getMergeBuilder() {
+        return new Builder(simpleName()).init(this);
+    }
+
+    @Override
+    protected String contentType() {
+        return CONTENT_TYPE;
+    }
+
+    @Override
+    protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException {
+        getMergeBuilder().toXContent(builder, includeDefaults);
+        multiFields.toXContent(builder, params);
+        copyTo.toXContent(builder, params);
+    }
+
+    public String getType() {
+        return type;
+    }
+
+    public Script getScript() {
+        return script;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java
index 39b0ec224b57c..92ffdb60e6cde 100644
--- a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java
@@ -291,6 +291,15 @@ protected static boolean parseObjectOrDocumentTypeProperties(
             } else if (fieldName.equals("enabled")) {
                 builder.enabled(XContentMapValues.nodeBooleanValue(fieldNode, fieldName + ".enabled"));
                 return true;
+            } else if (fieldName.equals("derived")) {
+                if (fieldNode instanceof Collection && ((Collection) fieldNode).isEmpty()) {
+                    // nothing to do here, empty (to support "derived: []" case)
+                } else if (fieldNode instanceof Map) {
+                    parseDerived(builder, (Map<String, Object>) fieldNode, parserContext);
+                } else {
+                    throw new OpenSearchParseException("derived must be a map type");
+                }
+                return true;
             } else if (fieldName.equals("properties")) {
                 if (fieldNode instanceof Collection && ((Collection) fieldNode).isEmpty()) {
                     // nothing to do here, empty (to support "properties: []" case)
@@ -349,6 +358,55 @@ protected static void parseNested(
             }
         }
 
+        protected static void parseDerived(ObjectMapper.Builder objBuilder, Map<String, Object> derivedNode, ParserContext parserContext) {
+            Iterator<Map.Entry<String, Object>> iterator = derivedNode.entrySet().iterator();
+            while (iterator.hasNext()) {
+                Map.Entry<String, Object> entry = iterator.next();
+                String fieldName = entry.getKey();
+                // Should accept empty arrays, as a work around for when the
+                // user can't provide an empty Map. (PHP for example)
+                boolean isEmptyList = entry.getValue() instanceof List && ((List<?>) entry.getValue()).isEmpty();
+
+                if (entry.getValue() instanceof Map) {
+                    @SuppressWarnings("unchecked")
+                    Map<String, Object> node = (Map<String, Object>) entry.getValue();
+
+                    // Derived fields are a bit unique in that the 'type' attribute does not map to the TypeParser
+                    // like it would for traditional fields in properties.
+                    // So in this case, the DerivedFieldMapper's TypeParser will explicitly be used
+                    Mapper.TypeParser typeParser = parserContext.typeParser(DerivedFieldMapper.CONTENT_TYPE);
+                    String[] fieldNameParts = fieldName.split("\\.");
+                    // field name is just ".", which is invalid
+                    if (fieldNameParts.length < 1) {
+                        throw new MapperParsingException("Invalid field name " + fieldName);
+                    }
+                    String realFieldName = fieldNameParts[fieldNameParts.length - 1];
+                    Mapper.Builder<?> fieldBuilder = typeParser.parse(realFieldName, node, parserContext);
+                    for (int i = fieldNameParts.length - 2; i >= 0; --i) {
+                        ObjectMapper.Builder<?> intermediate = new ObjectMapper.Builder<>(fieldNameParts[i]);
+                        intermediate.add(fieldBuilder);
+                        fieldBuilder = intermediate;
+                    }
+                    objBuilder.add(fieldBuilder);
+                    node.remove("type");
+                    DocumentMapperParser.checkNoRemainingFields(fieldName, node, parserContext.indexVersionCreated());
+                    iterator.remove();
+                } else if (isEmptyList) {
+                    iterator.remove();
+                } else {
+                    throw new MapperParsingException(
+                        "Expected map for property [derived_fields] on field [" + fieldName + "] but got a " + fieldName.getClass()
+                    );
+                }
+            }
+
+            DocumentMapperParser.checkNoRemainingFields(
+                derivedNode,
+                parserContext.indexVersionCreated(),
+                "DocType mapping definition has unsupported parameters: "
+            );
+        }
+
         protected static void parseProperties(ObjectMapper.Builder objBuilder, Map<String, Object> propsNode, ParserContext parserContext) {
             Iterator<Map.Entry<String, Object>> iterator = propsNode.entrySet().iterator();
             while (iterator.hasNext()) {
@@ -663,7 +721,21 @@ public void toXContent(XContentBuilder builder, Params params, ToXContent custom
         doXContent(builder, params);
 
         // sort the mappers so we get consistent serialization format
-        Mapper[] sortedMappers = mappers.values().stream().toArray(size -> new Mapper[size]);
+        Mapper[] derivedSortedMappers = mappers.values()
+            .stream()
+            .filter(m -> m instanceof DerivedFieldMapper)
+            .toArray(size -> new Mapper[size]);
+        Arrays.sort(derivedSortedMappers, new Comparator<Mapper>() {
+            @Override
+            public int compare(Mapper o1, Mapper o2) {
+                return o1.name().compareTo(o2.name());
+            }
+        });
+
+        Mapper[] sortedMappers = mappers.values()
+            .stream()
+            .filter(m -> !(m instanceof DerivedFieldMapper))
+            .toArray(size -> new Mapper[size]);
         Arrays.sort(sortedMappers, new Comparator<Mapper>() {
             @Override
             public int compare(Mapper o1, Mapper o2) {
@@ -672,6 +744,17 @@ public int compare(Mapper o1, Mapper o2) {
         });
 
         int count = 0;
+        for (Mapper mapper : derivedSortedMappers) {
+            if (count++ == 0) {
+                builder.startObject("derived");
+            }
+            mapper.toXContent(builder, params);
+        }
+        if (count > 0) {
+            builder.endObject();
+        }
+
+        count = 0;
         for (Mapper mapper : sortedMappers) {
             if (!(mapper instanceof MetadataFieldMapper)) {
                 if (count++ == 0) {
diff --git a/server/src/main/java/org/opensearch/index/mapper/ParametrizedFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/ParametrizedFieldMapper.java
index f4723b6178137..4f4ec45001a54 100644
--- a/server/src/main/java/org/opensearch/index/mapper/ParametrizedFieldMapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/ParametrizedFieldMapper.java
@@ -670,7 +670,11 @@ public final void parse(String name, ParserContext parserContext, Map<String, Ob
                     deprecatedParamsMap.put(deprecatedName, param);
                 }
             }
-            String type = (String) fieldNode.remove("type");
+            String type = (String) fieldNode.get("type");
+            if (paramsMap.get("type") == null) {
+                fieldNode.remove("type");
+            }
+
             for (Iterator<Map.Entry<String, Object>> iterator = fieldNode.entrySet().iterator(); iterator.hasNext();) {
                 Map.Entry<String, Object> entry = iterator.next();
                 final String propName = entry.getKey();
diff --git a/server/src/main/java/org/opensearch/indices/IndicesModule.java b/server/src/main/java/org/opensearch/indices/IndicesModule.java
index eea5dbbf57f6c..e989ef2708988 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesModule.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesModule.java
@@ -48,6 +48,7 @@
 import org.opensearch.index.mapper.CompletionFieldMapper;
 import org.opensearch.index.mapper.DataStreamFieldMapper;
 import org.opensearch.index.mapper.DateFieldMapper;
+import org.opensearch.index.mapper.DerivedFieldMapper;
 import org.opensearch.index.mapper.DocCountFieldMapper;
 import org.opensearch.index.mapper.FieldAliasMapper;
 import org.opensearch.index.mapper.FieldNamesFieldMapper;
@@ -167,6 +168,7 @@ public static Map<String, Mapper.TypeParser> getMappers(List<MapperPlugin> mappe
         mappers.put(FieldAliasMapper.CONTENT_TYPE, new FieldAliasMapper.TypeParser());
         mappers.put(GeoPointFieldMapper.CONTENT_TYPE, new GeoPointFieldMapper.TypeParser());
         mappers.put(FlatObjectFieldMapper.CONTENT_TYPE, FlatObjectFieldMapper.PARSER);
+        mappers.put(DerivedFieldMapper.CONTENT_TYPE, DerivedFieldMapper.PARSER);
 
         for (MapperPlugin mapperPlugin : mapperPlugins) {
             for (Map.Entry<String, Mapper.TypeParser> entry : mapperPlugin.getMappers().entrySet()) {
diff --git a/server/src/test/java/org/opensearch/index/mapper/DerivedFieldMapperQueryTests.java b/server/src/test/java/org/opensearch/index/mapper/DerivedFieldMapperQueryTests.java
new file mode 100644
index 0000000000000..64f4f1f3e083e
--- /dev/null
+++ b/server/src/test/java/org/opensearch/index/mapper/DerivedFieldMapperQueryTests.java
@@ -0,0 +1,279 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.mapper;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.store.Directory;
+import org.opensearch.common.lucene.Lucene;
+import org.opensearch.core.index.Index;
+import org.opensearch.index.query.QueryBuilders;
+import org.opensearch.index.query.QueryShardContext;
+import org.opensearch.script.DerivedFieldScript;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+import org.mockito.Mockito;
+
+import static org.mockito.Mockito.when;
+
+public class DerivedFieldMapperQueryTests extends MapperServiceTestCase {
+
+    // First element is the document ingested, other elements corresponds to value returned against a given derived field script
+    // Raw Message, Request Succeeded (boolean), Timestamp (long), Client IP, Method, Request Size (double), Duration (long)
+    private static final Object[][] raw_requests = new Object[][] {
+        {
+            "40.135.0.0 GET /images/hm_bg.jpg?size=1.5KB HTTP/1.0 200 2024-03-20T08:30:45 1500",
+            true,
+            1710923445000L,
+            "40.135.0.0",
+            "GET",
+            1.5,
+            1500L },
+        {
+            "232.0.0.0 GET /images/hm_bg.jpg?size=2.3KB HTTP/1.0 400 2024-03-20T09:15:20 2300",
+            false,
+            1710926120000L,
+            "232.0.0.0",
+            "GET",
+            2.3,
+            2300L },
+        {
+            "26.1.0.0 DELETE /images/hm_bg.jpg?size=3.7KB HTTP/1.0 200 2024-03-20T10:05:55 3700",
+            true,
+            1710929155000L,
+            "26.1.0.0",
+            "DELETE",
+            3.7,
+            3700L },
+        {
+            "247.37.0.0 GET /french/splash_inet.html?size=4.1KB HTTP/1.0 400 2024-03-20T11:20:10 4100",
+            false,
+            1710933610000L,
+            "247.37.0.0",
+            "GET",
+            4.1,
+            4100L },
+        {
+            "247.37.0.0 DELETE /french/splash_inet.html?size=5.8KB HTTP/1.0 400 2024-03-20T12:45:30 5800",
+            false,
+            1710938730000L,
+            "247.37.0.0",
+            "DELETE",
+            5.8,
+            5800L },
+        {
+            "10.20.30.40 GET /path/to/resource?size=6.3KB HTTP/1.0 200 2024-03-20T13:10:15 6300",
+            true,
+            1710940215000L,
+            "10.20.30.40",
+            "GET",
+            6.3,
+            6300L },
+        {
+            "50.60.70.80 GET /path/to/resource?size=7.2KB HTTP/1.0 404 2024-03-20T14:20:50 7200",
+            false,
+            1710944450000L,
+            "50.60.70.80",
+            "GET",
+            7.2,
+            7200L },
+        {
+            "127.0.0.1 PUT /path/to/resource?size=8.9KB HTTP/1.0 500 2024-03-20T15:30:25 8900",
+            false,
+            1710948625000L,
+            "127.0.0.1",
+            "PUT",
+            8.9,
+            8900L },
+        {
+            "127.0.0.1 GET /path/to/resource?size=9.4KB HTTP/1.0 200 2024-03-20T16:40:15 9400",
+            true,
+            1710952815000L,
+            "127.0.0.1",
+            "GET",
+            9.4,
+            9400L },
+        {
+            "192.168.1.1 GET /path/to/resource?size=10.7KB HTTP/1.0 400 2024-03-20T17:50:40 10700",
+            false,
+            1710957040000L,
+            "192.168.1.1",
+            "GET",
+            10.7,
+            10700L } };
+
+    public void testAllPossibleQueriesOnDerivedFields() throws IOException {
+        MapperService mapperService = createMapperService(topMapping(b -> {
+            b.startObject("properties");
+            {
+                b.startObject("raw_message");
+                {
+                    b.field("type", "text");
+                }
+                b.endObject();
+            }
+            b.endObject();
+            b.startObject("derived");
+            {
+                b.startObject("request_succeeded");
+                {
+                    b.field("type", "boolean");
+                    b.field("script", "");
+                }
+                b.endObject();
+                b.startObject("@timestamp");
+                {
+                    b.field("type", "date");
+                    b.field("script", "");
+                }
+                b.endObject();
+                b.startObject("client_ip");
+                {
+                    b.field("type", "ip");
+                    b.field("script", "");
+                }
+                b.endObject();
+                b.startObject("method");
+                {
+                    b.field("type", "keyword");
+                    b.field("script", "");
+                }
+                b.endObject();
+                b.startObject("request_size");
+                {
+                    b.field("type", "double");
+                    b.field("script", "");
+                }
+                b.endObject();
+                b.startObject("duration");
+                {
+                    b.field("type", "long");
+                    b.field("script", "");
+                }
+                b.endObject();
+            }
+            b.endObject();
+        }));
+
+        List<Document> docs = new ArrayList<>();
+        for (Object[] request : raw_requests) {
+            Document document = new Document();
+            document.add(new TextField("raw_message", (String) request[0], Field.Store.YES));
+            docs.add(document);
+        }
+
+        int[] scriptIndex = { 1 };
+
+        // Mock DerivedFieldScript.Factory
+        DerivedFieldScript.Factory factory = (params, lookup) -> (DerivedFieldScript.LeafFactory) ctx -> new DerivedFieldScript(
+            params,
+            lookup,
+            ctx
+        ) {
+            int docId = 0;
+
+            @Override
+            public void setDocument(int docId) {
+                super.setDocument(docId);
+                this.docId = docId;
+            }
+
+            @Override
+            public Object execute() {
+                return raw_requests[docId][scriptIndex[0]];
+            }
+        };
+
+        QueryShardContext queryShardContext = createQueryShardContext(mapperService);
+        when(queryShardContext.compile(Mockito.any(), Mockito.any())).thenReturn(factory);
+        when(queryShardContext.sourcePath("raw_message")).thenReturn(Set.of("raw_message"));
+        when(queryShardContext.index()).thenReturn(new Index("test_index", "uuid"));
+
+        // Index and Search
+        try (Directory dir = newDirectory()) {
+            IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER));
+            for (Document d : docs) {
+                iw.addDocument(d);
+            }
+            try (IndexReader reader = DirectoryReader.open(iw)) {
+                iw.close();
+
+                IndexSearcher searcher = new IndexSearcher(reader);
+                Query query = QueryBuilders.termQuery("request_succeeded", "true").toQuery(queryShardContext);
+                TopDocs topDocs = searcher.search(query, 10);
+                assertEquals(4, topDocs.totalHits.value);
+
+                // IP Field Term Query
+                scriptIndex[0] = 3;
+                query = QueryBuilders.termQuery("client_ip", "192.168.0.0/16").toQuery(queryShardContext);
+                topDocs = searcher.search(query, 10);
+                assertEquals(1, topDocs.totalHits.value);
+
+                scriptIndex[0] = 4;
+                query = QueryBuilders.termsQuery("method", "DELETE", "PUT").toQuery(queryShardContext);
+                topDocs = searcher.search(query, 10);
+                assertEquals(3, topDocs.totalHits.value);
+
+                query = QueryBuilders.termsQuery("method", "delete").toQuery(queryShardContext);
+                topDocs = searcher.search(query, 10);
+                assertEquals(0, topDocs.totalHits.value);
+
+                query = QueryBuilders.termQuery("method", "delete").caseInsensitive(true).toQuery(queryShardContext);
+                topDocs = searcher.search(query, 10);
+                assertEquals(2, topDocs.totalHits.value);
+
+                // Range queries of types - date, long and double
+                scriptIndex[0] = 2;
+                query = QueryBuilders.rangeQuery("@timestamp").from("2024-03-20T14:20:50").toQuery(queryShardContext);
+                topDocs = searcher.search(query, 10);
+                assertEquals(4, topDocs.totalHits.value);
+
+                scriptIndex[0] = 5;
+                query = QueryBuilders.rangeQuery("request_size").from("4.1").toQuery(queryShardContext);
+                topDocs = searcher.search(query, 10);
+                assertEquals(7, topDocs.totalHits.value);
+
+                scriptIndex[0] = 6;
+                query = QueryBuilders.rangeQuery("duration").from("5800").toQuery(queryShardContext);
+                topDocs = searcher.search(query, 10);
+                assertEquals(6, topDocs.totalHits.value);
+
+                scriptIndex[0] = 4;
+
+                // Prefix Query
+                query = QueryBuilders.prefixQuery("method", "DE").toQuery(queryShardContext);
+                topDocs = searcher.search(query, 10);
+                assertEquals(2, topDocs.totalHits.value);
+
+                scriptIndex[0] = 4;
+                query = QueryBuilders.wildcardQuery("method", "G*").toQuery(queryShardContext);
+                topDocs = searcher.search(query, 10);
+                assertEquals(7, topDocs.totalHits.value);
+
+                // Regexp Query
+                scriptIndex[0] = 4;
+                query = QueryBuilders.regexpQuery("method", ".*LET.*").toQuery(queryShardContext);
+                topDocs = searcher.search(query, 10);
+                assertEquals(2, topDocs.totalHits.value);
+            }
+        }
+    }
+}
diff --git a/server/src/test/java/org/opensearch/index/mapper/DerivedFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/DerivedFieldMapperTests.java
new file mode 100644
index 0000000000000..e3e2b978369ff
--- /dev/null
+++ b/server/src/test/java/org/opensearch/index/mapper/DerivedFieldMapperTests.java
@@ -0,0 +1,117 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.mapper;
+
+import org.opensearch.common.CheckedConsumer;
+import org.opensearch.common.xcontent.XContentFactory;
+import org.opensearch.core.xcontent.ToXContent;
+import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.script.Script;
+
+import java.io.IOException;
+
+public class DerivedFieldMapperTests extends MapperTestCase {
+
+    protected boolean supportsOrIgnoresBoost() {
+        return false;
+    }
+
+    protected boolean supportsMeta() {
+        return false;
+    }
+
+    // Overriding fieldMapping to make it create derived mappings by default.
+    // This way, the parent tests are checking the right behavior for this Mapper.
+    @Override
+    protected final XContentBuilder fieldMapping(CheckedConsumer<XContentBuilder, IOException> buildField) throws IOException {
+        return fieldMapping(buildField, true);
+    }
+
+    // Overriding this to do nothing since derived fields are not written in the documents
+    @Override
+    protected void writeFieldValue(XContentBuilder builder) throws IOException {}
+
+    @Override
+    protected void minimalMapping(XContentBuilder b) throws IOException {
+        b.field("type", "keyword");
+    }
+
+    protected void registerParameters(ParameterChecker checker) throws IOException {
+        // TODO Any conflicts or updates to check for here? Parameters index, store, doc_values and boost are not
+        // supported for DerivedFieldMapper (we explicitly set these values on initialization)
+    }
+
+    // TODO: Can update this once the query implementation is completed
+    // This is also being left blank because the super assertExistsQuery is trying to parse
+    // an empty source and fails.
+    @Override
+    protected void assertExistsQuery(MapperService mapperService) {}
+
+    public void testSerialization() throws IOException {
+
+        DocumentMapper defaultMapper = createDocumentMapper(fieldMapping(this::minimalMapping));
+        Mapper mapper = defaultMapper.mappers().getMapper("field");
+        assertTrue(mapper instanceof DerivedFieldMapper);
+
+        XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
+        mapper.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        builder.endObject();
+        assertEquals("{\"field\":{\"type\":\"keyword\"}}", builder.toString());
+    }
+
+    public void testParsesScript() throws IOException {
+
+        String scriptString = "doc['test'].value";
+        DocumentMapper defaultMapper = createDocumentMapper(fieldMapping(b -> {
+            b.field("type", "keyword");
+            b.startObject("script");
+            b.field("source", scriptString);
+            b.endObject();
+        }));
+        Mapper mapper = defaultMapper.mappers().getMapper("field");
+        assertTrue(mapper instanceof DerivedFieldMapper);
+        DerivedFieldMapper derivedFieldMapper = (DerivedFieldMapper) mapper;
+        assertEquals(Script.parse(scriptString), derivedFieldMapper.getScript());
+
+        XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
+        mapper.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        builder.endObject();
+        assertEquals(
+            "{\"field\":{\"type\":\"keyword\",\"script\":{\"source\":\"doc['test'].value\",\"lang\":\"painless\"}}}",
+            builder.toString()
+        );
+    }
+
+    // TODO: Is there a case where we want to allow the field to be defined in both 'derived' and 'properties'?
+    // If the user wants to move a field they were testing as derived to be indexed, they should be able to update
+    // the mappings in index template to move the field from 'derived' to 'properties' and it should take affect
+    // during the next index rollover (so even in this case, the field is only defined in one or the other).
+    public void testFieldInDerivedAndProperties() throws IOException {
+        MapperParsingException ex = expectThrows(MapperParsingException.class, () -> createDocumentMapper(topMapping(b -> {
+            b.startObject("derived");
+            b.startObject("field");
+            b.field("type", "keyword");
+            b.endObject();
+            b.endObject();
+            b.startObject("properties");
+            b.startObject("field");
+            b.field("type", "keyword");
+            b.endObject();
+            b.endObject();
+        })));
+        // TODO: Do we want to handle this as a different error? As it stands, it fails as a merge conflict which makes sense.
+        // If it didn't fail here, it would hit the MapperParsingException for the field being defined more than once
+        // when MappingLookup is initialized
+        assertEquals("Failed to parse mapping [_doc]: mapper [field] cannot be changed from type [derived] to [keyword]", ex.getMessage());
+    }
+
+    // TODO TESTCASE: testWithFieldInSource() (derived field with that field present in source)
+    // This is more checking search behavior so may need to revisit this after query implementation
+
+}
diff --git a/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java
index be947935dc4ea..b10a7d8155056 100644
--- a/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java
+++ b/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java
@@ -39,6 +39,7 @@
 import org.opensearch.index.mapper.MapperService.MergeReason;
 import org.opensearch.index.mapper.ObjectMapper.Dynamic;
 import org.opensearch.plugins.Plugin;
+import org.opensearch.script.Script;
 import org.opensearch.test.InternalSettingsPlugin;
 import org.opensearch.test.OpenSearchSingleNodeTestCase;
 
@@ -440,6 +441,52 @@ public void testEmptyName() throws Exception {
         assertThat(e.getMessage(), containsString("name cannot be empty string"));
     }
 
+    public void testDerivedFields() throws Exception {
+        String mapping = XContentFactory.jsonBuilder()
+            .startObject()
+            .startObject("tweet")
+            .startObject("derived")
+            .startObject("derived_field_name1")
+            .field("type", "boolean")
+            .endObject()
+            .startObject("derived_field_name2")
+            .field("type", "keyword")
+            .startObject("script")
+            .field("source", "doc['test'].value")
+            .endObject()
+            .endObject()
+            .endObject()
+            .startObject("properties")
+            .startObject("field_name")
+            .field("type", "date")
+            .endObject()
+            .endObject()
+            .endObject()
+            .endObject()
+            .toString();
+
+        DocumentMapper documentMapper = createIndex("test").mapperService()
+            .documentMapperParser()
+            .parse("tweet", new CompressedXContent(mapping));
+
+        Mapper mapper = documentMapper.root().getMapper("derived_field_name1");
+        assertTrue(mapper instanceof DerivedFieldMapper);
+        DerivedFieldMapper derivedFieldMapper = (DerivedFieldMapper) mapper;
+        assertEquals("boolean", derivedFieldMapper.getType());
+        assertNull(derivedFieldMapper.getScript());
+
+        mapper = documentMapper.root().getMapper("derived_field_name2");
+        assertTrue(mapper instanceof DerivedFieldMapper);
+        derivedFieldMapper = (DerivedFieldMapper) mapper;
+        assertEquals("keyword", derivedFieldMapper.getType());
+        assertEquals(Script.parse("doc['test'].value"), derivedFieldMapper.getScript());
+
+        // Check that field in properties was parsed correctly as well
+        mapper = documentMapper.root().getMapper("field_name");
+        assertNotNull(mapper);
+        assertEquals("date", mapper.typeName());
+    }
+
     @Override
     protected Collection<Class<? extends Plugin>> getPlugins() {
         return pluginList(InternalSettingsPlugin.class);
diff --git a/test/framework/src/main/java/org/opensearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/opensearch/index/mapper/MapperServiceTestCase.java
index ac78a0d1936ea..ddf8a23da6e46 100644
--- a/test/framework/src/main/java/org/opensearch/index/mapper/MapperServiceTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/index/mapper/MapperServiceTestCase.java
@@ -225,13 +225,32 @@ protected final XContentBuilder mapping(CheckedConsumer<XContentBuilder, IOExcep
         return builder.endObject().endObject().endObject();
     }
 
+    protected final XContentBuilder derivedMapping(CheckedConsumer<XContentBuilder, IOException> buildFields) throws IOException {
+        XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject("_doc").startObject("derived");
+        buildFields.accept(builder);
+        return builder.endObject().endObject().endObject();
+    }
+
     protected final XContentBuilder dynamicMapping(Mapping dynamicMapping) throws IOException {
         XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
         dynamicMapping.toXContent(builder, ToXContent.EMPTY_PARAMS);
         return builder.endObject();
     }
 
-    protected final XContentBuilder fieldMapping(CheckedConsumer<XContentBuilder, IOException> buildField) throws IOException {
+    protected XContentBuilder fieldMapping(CheckedConsumer<XContentBuilder, IOException> buildField) throws IOException {
+        return fieldMapping(buildField, false);
+    }
+
+    protected final XContentBuilder fieldMapping(CheckedConsumer<XContentBuilder, IOException> buildField, Boolean isDerived)
+        throws IOException {
+        if (isDerived) {
+            return derivedMapping(b -> {
+                b.startObject("field");
+                buildField.accept(b);
+                b.endObject();
+            });
+        }
+
         return mapping(b -> {
             b.startObject("field");
             buildField.accept(b);
diff --git a/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java
index ac0447dbebf7e..7dbd080519c27 100644
--- a/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java
@@ -94,6 +94,7 @@
 import org.opensearch.index.mapper.CompletionFieldMapper;
 import org.opensearch.index.mapper.ContentPath;
 import org.opensearch.index.mapper.DateFieldMapper;
+import org.opensearch.index.mapper.DerivedFieldMapper;
 import org.opensearch.index.mapper.FieldAliasMapper;
 import org.opensearch.index.mapper.FieldMapper;
 import org.opensearch.index.mapper.GeoPointFieldMapper;
@@ -185,6 +186,7 @@ public abstract class AggregatorTestCase extends OpenSearchTestCase {
         denylist.add(ObjectMapper.NESTED_CONTENT_TYPE); // TODO support for nested
         denylist.add(CompletionFieldMapper.CONTENT_TYPE); // TODO support completion
         denylist.add(FieldAliasMapper.CONTENT_TYPE); // TODO support alias
+        denylist.add(DerivedFieldMapper.CONTENT_TYPE); // TODO support derived fields
         TYPE_TEST_DENYLIST = denylist;
     }