Merge branch 'main' into adding-whitespace-tokenizer-docs

.github/CODEOWNERS

@@ -1 +1 @@
-*  @kolchfa-aws @Naarcha-AWS @vagimeli @AMoo-Miki @natebower @dlvenable @stephen-crawford @epugh
+*  @kolchfa-aws @Naarcha-AWS @AMoo-Miki @natebower @dlvenable @epugh

.github/ISSUE_TEMPLATE/issue_template.md

@@ -15,7 +15,6 @@ assignees: ''
 
 **Tell us about your request.** Provide a summary of the request.
 
-***Version:** List the OpenSearch version to which this issue applies, e.g. 2.14, 2.12--2.14, or all.
+**Version:** List the OpenSearch version to which this issue applies, e.g. 2.14, 2.12--2.14, or all.
 
 **What other resources are available?** Provide links to related issues, POCs, steps for testing, etc.
-

.github/vale/styles/Vocab/OpenSearch/Products/accept.txt

@@ -85,6 +85,7 @@ Python
 PyTorch
 Querqy
 Query Workbench
+RankLib
 RCF Summarize
 RPM Package Manager
 Ruby
@@ -97,4 +98,5 @@ TorchScript
 Tribuo
 VisBuilder
 Winlogbeat
-Zstandard
+XGBoost
+Zstandard

.github/vale/styles/Vocab/OpenSearch/Words/accept.txt

@@ -77,13 +77,16 @@ Levenshtein
 [Mm]ultivalued
 [Mm]ultiword
 [Nn]amespace
-[Oo]versamples?
+[Oo]ffline
 [Oo]nboarding
+[Oo]versamples?
 pebibyte
 p\d{2}
 [Pp]erformant
 [Pp]laintext
 [Pp]luggable
+[Pp]reaggregate(s|d)?
+[Pp]recompute(s|d)?
 [Pp]reconfigure
 [Pp]refetch
 [Pp]refilter
@@ -105,6 +108,7 @@ p\d{2}
 [Rr]eprovision(ed|ing)?
 [Rr]erank(er|ed|ing)?
 [Rr]epo
+[Rr]escor(e|ed|ing)?
 [Rr]ewriter
 [Rr]ollout
 [Rr]ollup

.github/workflows/jekyll-spec-insert.yml

@@ -0,0 +1,20 @@
+name: Lint and Test Jekyll Spec Insert
+on:
+  push:
+    paths:
+      - 'spec-insert/**'
+  pull_request:
+    paths:
+      - 'spec-insert/**'
+jobs:
+  lint-and-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ruby/setup-ruby@v1
+        with: { ruby-version: 3.3.0 }
+      - run: bundle install
+      - working-directory: spec-insert
+        run: |
+          bundle exec rubocop
+          bundle exec rspec

.github/workflows/pr_checklist.yml

@@ -29,7 +29,7 @@ jobs:
         with:
           script: |
             let assignee = context.payload.pull_request.user.login;
-            const prOwners = ['Naarcha-AWS', 'kolchfa-aws', 'vagimeli', 'natebower'];
+            const prOwners = ['Naarcha-AWS', 'kolchfa-aws', 'natebower'];
             
             if (!prOwners.includes(assignee)) {
               assignee = 'kolchfa-aws'
@@ -40,4 +40,4 @@ jobs:
                 owner: context.repo.owner,
                 repo: context.repo.repo,
                 assignees: [assignee]
-              });
+              });

.github/workflows/update-api-components.yml

@@ -0,0 +1,52 @@
+name: Update API Components
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 0 * * 0" # Every Sunday at midnight GMT
+jobs:
+  update-api-components:
+    if: ${{ github.repository == 'opensearch-project/documentation-website' }}
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+          fetch-depth: 0
+
+      - run: git config --global pull.rebase true
+
+      - uses: ruby/setup-ruby@v1
+        with: { ruby-version: 3.3.0 }
+
+      - run: bundle install
+
+      - name: Download spec and insert into documentation
+        run: bundle exec jekyll spec-insert
+
+      - name: Get current date
+        id: date
+        run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
+
+      - name: GitHub App token
+        id: github_app_token
+        uses: tibdex/[email protected]
+        with:
+          app_id: ${{ secrets.APP_ID }}
+          private_key: ${{ secrets.APP_PRIVATE_KEY }}
+
+      - name: Create pull request
+        uses: peter-evans/create-pull-request@v6
+        with:
+          token: ${{ steps.github_app_token.outputs.token }}
+          commit-message: "Updated API components to reflect the latest OpenSearch API spec (${{ env.date }})"
+          title: "[AUTOCUT] Update API components to reflect the latest OpenSearch API spec (${{ env.date }})"
+          body: |
+            Update API components to reflect the latest [OpenSearch API spec](https://github.com/opensearch-project/opensearch-api-specification/releases/download/main-latest/opensearch-openapi.yaml).
+            Date: ${{ env.date }}
+          branch: update-api-components-${{ env.date }}
+          base: main
+          signoff: true
+          labels: autocut

.gitignore

@@ -7,3 +7,4 @@ Gemfile.lock
 *.iml
 .jekyll-cache
 .project
+vendor/bundle

API_STYLE_GUIDE.md

@@ -103,7 +103,7 @@ For GET and DELETE APIs: Introduce what you can do with the optional parameters.
 Parameter | Data type | Description
 :--- | :--- | :---
 
-### Request fields
+### Request body fields
 
 For PUT and POST APIs: Introduce what the request fields are allowed to provide in the body of the request.
 
@@ -189,7 +189,7 @@ The `POST _reindex` request returns the following response fields:
 }
 ```
 
-### Response fields
+### Response body fields
 
 For PUT and POST APIs: Define all allowable response fields that can be returned in the body of the response.
 

CONTRIBUTING.md

@@ -78,11 +78,13 @@ Follow these steps to set up your local copy of the repository:
 
 1. Navigate to your cloned repository.
 
+##### Building by using locally installed packages 
+
 1. Install [Ruby](https://www.ruby-lang.org/en/) if you don't already have it. We recommend [RVM](https://rvm.io/), but you can use any method you prefer:
 
    ```
    curl -sSL https://get.rvm.io | bash -s stable
-   rvm install 3.2.4
+   rvm install 3.3.2
    ruby -v
    ```
 
@@ -98,6 +100,14 @@ Follow these steps to set up your local copy of the repository:
    bundle install
    ```
 
+##### Building by using containerization
+
+Assuming you have `docker-compose` installed, run the following command:
+
+   ```
+   docker compose -f docker-compose.dev.yml up
+   ```
+
 #### Troubleshooting
 
 Try the following troubleshooting steps if you encounter an error when trying to build the documentation website:  
@@ -148,6 +158,23 @@ To ensure that our documentation adheres to the [OpenSearch Project Style Guidel
 
 Optionally, you can install the [Vale VSCode](https://github.com/chrischinchilla/vale-vscode) extension, which integrates Vale with Visual Studio Code. By default, only _errors_ and _warnings_ are underlined. To change the minimum alert level to include _suggestions_, go to **Vale VSCode** > **Extension Settings** and select **suggestion** in the **Vale > Vale CLI: Min Alert Level** dropdown list. 
 
+## Troubleshooting
+
+This section provides information about potential solutions for known issues.
+
+### Installing Ruby on an Apple silicon machine
+
+If you're having trouble installing Ruby with `rvm` on an Apple silicon machine, it could be because of an OpenSSL version misalignment. To fix this issue, use the following command, replacing `<openssl-version>` with your [desired version](https://github.com/ruby/openssl/blob/master/README.md):
+
+```
+# Assumes Brew is installed
+curl -sSL https://get.rvm.io | bash -s stable
+rvm install 3.2.4 --with-openssl-dir=$(brew --prefix openssl@<openssl-version>)
+ruby -v
+```
+
 ## Getting help
 
 For help with the contribution process, reach out to one of the [points of contact](README.md#points-of-contact).
+
+

DEVELOPER_GUIDE.md

@@ -0,0 +1,135 @@
+# Developer guide
+- [Introduction](#introduction)
+- [Starting the Jekyll server locally](#starting-the-jekyll-server-locally)
+- [Using the spec-insert Jekyll plugin](#using-the-spec-insert-jekyll-plugin)
+  - [Ignoring files and folders](#ignoring-files-and-folders)
+- [CI/CD](#cicd)
+- [Spec insert components](#spec-insert-components)
+  - [Query parameters](#query-parameters)
+  - [Path parameters](#path-parameters)
+  - [Paths and HTTP methods](#paths-and-http-methods)
+
+## Introduction
+
+The `.md` documents in this repository are rendered into HTML pages using [Jekyll](https://jekyllrb.com/). These HTML pages are hosted on [opensearch.org](https://opensearch.org/docs/latest/).
+
+## Starting the Jekyll server locally
+You can run the Jekyll server locally to view the rendered HTML pages using the following steps:
+
+1. Install [Ruby](https://www.ruby-lang.org/en/documentation/installation/) 3.1.0 or later for your operating system.
+2. Install the required gems by running `bundle install`.
+3. Run `bundle exec jekyll serve` to start the Jekyll server locally (this can take several minutes to complete).
+4. Open your browser and navigate to `http://localhost:4000` to view the rendered HTML pages.
+
+## Using the `spec-insert` Jekyll plugin
+The `spec-insert` Jekyll plugin is used to insert API components into Markdown files. The plugin downloads the [latest OpenSearch specification](https://github.com/opensearch-project/opensearch-api-specification) and renders the API components from the spec. This aims to reduce the manual effort required to keep the documentation up to date.
+
+To use this plugin, make sure that you have installed Ruby 3.1.0 or later and the required gems by running `bundle install`.
+
+Edit your Markdown file and insert the following snippet where you want render an API component:
+
+```markdown
+<!-- spec_insert_start 
+api: <API_NAME>
+component: <COMPONENT_NAME>
+other_argument: <OTHER_ARGUMENT>
+-->
+
+This is where the API component will be inserted.
+Everything between the `spec_insert_start` and `spec_insert_end` tags will be overwritten.
+
+<!-- spec_insert_end -->
+```
+
+Then run the following Jekyll command to render the API components:
+```shell
+bundle exec jekyll spec-insert
+```
+
+If you are working on multiple Markdown files and do not want to keep running the `jekyll spec-insert` command, you can add the `--watch` (or `-W`) flag to the command to watch for changes in the Markdown files and automatically render the API components:
+
+```shell
+bundle exec jekyll spec-insert --watch
+```
+
+Depending on the text editor you are using, you may need to manually reload the file from disk to see the changes applied by the plugin if the editor does not automatically reload the file periodically.
+
+The plugin will pull the newest OpenSearch API spec from its [repository](https://github.com/opensearch-project/opensearch-api-specification) if the spec file does not exist locally or if it is older than 24 hours. To tell the plugin to always pull the newest spec, you can add the `--refresh-spec` (or `-R`) flag to the command:
+
+```shell
+bundle exec jekyll spec-insert --refresh-spec
+```
+
+### Ignoring files and folders
+The `spec-insert` plugin ignores all files and folders listed in the [./_config.yml#exclude](./_config.yml) list, which is also the list of files and folders that Jekyll ignores.
+
+## CI/CD
+The `spec-insert` plugin is run as part of the CI/CD pipeline to ensure that the API components are up to date in the documentation. This is performed through the [update-api-components.yml](.github/workflows/update-api-components.yml) GitHub Actions workflow, which creates a pull request containing the updated API components every Sunday.
+
+## Spec insert components
+All spec insert components accept the following arguments:
+- `api` (String; required): The name of the API to render the component from. This is equivalent to the `x-operation-group` field in the OpenSearch OpenAPI Spec.
+- `component` (String; required): The name of the component to render,  such as `query_parameters`, `path_parameters`, or `paths_and_http_methods`.
+- `omit_header` (Boolean; Default is `false`): If set to `true`, the markdown header of the component will not be rendered.
+
+### Paths and HTTP methods
+To insert paths and HTTP methods for the `search` API, use the following snippet:
+```markdown
+<!-- spec_insert_start
+api: search
+component: paths_and_http_methods
+-->
+<!-- spec_insert_end -->
+```
+
+### Path parameters
+
+To insert a path parameters table of the `indices.create` API, use the following snippet. Use the `x-operation-group` field from OpenSearch OpenAPI Spec for the `api` value:
+
+```markdown
+<!-- spec_insert_start
+api: indices.create
+component: path_parameters
+-->
+<!-- spec_insert_end -->
+```
+This table accepts the same arguments as the query parameters table except the `include_global` argument.
+
+### Query parameters
+To insert the API query parameters table of the `cat.indices` API, use the following snippet:
+```markdown
+<!-- spec_insert_start
+api: cat.indices
+component: query_parameters
+-->
+<!-- spec_insert_end -->
+```
+
+This will insert the query parameters of the `cat.indices` API into the `.md` file with three default columns: `Parameter`, `Type`, and `Description`. You can customize the query parameters table by adding the `columns` argument which accepts a comma-separated list of column names. The available column names are:
+
+- `Parameter`
+- `Type`
+- `Description`
+- `Required`
+- `Default`
+
+_When `Required`/`Default` is not chosen, the information will be written in the `Description` column._
+
+You can also customize this component with the following settings:
+
+- `include_global` (Boolean; default is `false`): Includes global query parameters in the table.
+- `include_deprecated` (Boolean; default is `true`): Includes deprecated parameters in the table.
+- `pretty` (Boolean; default is `false`): Renders the table in the pretty format instead of the compact format.
+
+The following snippet inserts the specified columns into the query parameters table:
+
+```markdown
+<!-- spec_insert_start
+api: cat.indices
+component: query_parameters
+include_global: true
+include_deprecated: false
+pretty: true
+-->
+<!-- spec_insert_end -->
+```