From c0b0394281b70323833fbac4bd641177446cb8c3 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]" <github-actions[bot]@users.noreply.github.com>
Date: Mon, 9 Dec 2024 18:24:09 +0000
Subject: [PATCH] add lowercase tokenizer docs (#8497)

* add lowercase tokenizer docs

Signed-off-by: Anton Rubin <anton.rubin@eliatra.com>

* Doc review

Signed-off-by: Fanit Kolchina <kolchfa@amazon.com>

* Apply suggestions from code review

Co-authored-by: Nathan Bower <nbower@amazon.com>
Signed-off-by: kolchfa-aws <105444904+kolchfa-aws@users.noreply.github.com>

---------

Signed-off-by: Anton Rubin <anton.rubin@eliatra.com>
Signed-off-by: Fanit Kolchina <kolchfa@amazon.com>
Signed-off-by: kolchfa-aws <105444904+kolchfa-aws@users.noreply.github.com>
Co-authored-by: Fanit Kolchina <kolchfa@amazon.com>
Co-authored-by: kolchfa-aws <105444904+kolchfa-aws@users.noreply.github.com>
Co-authored-by: Nathan Bower <nbower@amazon.com>
(cherry picked from commit 68a8af5fcf24d846b72a3024a88bce5c58c829e6)
Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
---
 _analyzers/tokenizers/lowercase.md | 93 ++++++++++++++++++++++++++++++
 1 file changed, 93 insertions(+)
 create mode 100644 _analyzers/tokenizers/lowercase.md

diff --git a/_analyzers/tokenizers/lowercase.md b/_analyzers/tokenizers/lowercase.md
new file mode 100644
index 0000000000..5542ecbf50
--- /dev/null
+++ b/_analyzers/tokenizers/lowercase.md
@@ -0,0 +1,93 @@
+---
+layout: default
+title: Lowercase
+parent: Tokenizers
+nav_order: 70
+---
+
+# Lowercase tokenizer
+
+The `lowercase` tokenizer breaks text into terms at white space and then lowercases all the terms. Functionally, this is identical to configuring a `letter` tokenizer with a `lowercase` token filter. However, using a `lowercase` tokenizer is more efficient because the tokenizer actions are performed in a single step.
+
+## Example usage
+
+The following example request creates a new index named `my-lowercase-index` and configures an analyzer with a `lowercase` tokenizer:
+
+```json
+PUT /my-lowercase-index
+{
+  "settings": {
+    "analysis": {
+      "tokenizer": {
+        "my_lowercase_tokenizer": {
+          "type": "lowercase"
+        }
+      },
+      "analyzer": {
+        "my_lowercase_analyzer": {
+          "type": "custom",
+          "tokenizer": "my_lowercase_tokenizer"
+        }
+      }
+    }
+  }
+}
+```
+{% include copy-curl.html %}
+
+## Generated tokens
+
+Use the following request to examine the tokens generated using the analyzer:
+
+```json
+POST /my-lowercase-index/_analyze
+{
+  "analyzer": "my_lowercase_analyzer",
+  "text": "This is a Test. OpenSearch 123!"
+}
+```
+{% include copy-curl.html %}
+
+The response contains the generated tokens:
+
+```json
+{
+  "tokens": [
+    {
+      "token": "this",
+      "start_offset": 0,
+      "end_offset": 4,
+      "type": "word",
+      "position": 0
+    },
+    {
+      "token": "is",
+      "start_offset": 5,
+      "end_offset": 7,
+      "type": "word",
+      "position": 1
+    },
+    {
+      "token": "a",
+      "start_offset": 8,
+      "end_offset": 9,
+      "type": "word",
+      "position": 2
+    },
+    {
+      "token": "test",
+      "start_offset": 10,
+      "end_offset": 14,
+      "type": "word",
+      "position": 3
+    },
+    {
+      "token": "opensearch",
+      "start_offset": 16,
+      "end_offset": 26,
+      "type": "word",
+      "position": 4
+    }
+  ]
+}
+```