Merge branch 'main' into enable-cleanup-closed

Signed-off-by: Daniel (dB.) Doubrovkine <[email protected]>
opensearch-project · Nov 9, 2023 · 2470d48 · 2470d48
2 parents 024f791 + 56c96d7
commit 2470d48
Show file tree

Hide file tree

Showing 336 changed files with 16,311 additions and 21,280 deletions.
diff --git a/.ci/test-matrix.yml b/.ci/test-matrix.yml
@@ -2,9 +2,6 @@ TEST_SUITE:
   - oss
 
 PYTHON_VERSION:
-  - "2.7"
-  - "3.4"
-  - "3.5"
   - "3.6"
   - "3.7"
   - "3.8"

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -1,2 +1,2 @@
 # This should match the team set up in https://github.com/orgs/opensearch-project/teams and include any additional contributors
-*   @VachaShah @dblock @harshavamsi @axeoman @deztructor @Shephalimittal @saimedhi
+*   @VachaShah @dblock @harshavamsi @axeoman @deztructor @Shephalimittal @saimedhi @florianvazelle
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
@@ -9,7 +9,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        opensearch_version: [ '1.0.1', '1.1.0', '1.2.4', '1.3.7', '2.0.1', '2.1.0', '2.2.1', '2.3.0', '2.4.0', '2.5.0', '2.6.0', '2.7.0', '2.8.0' ]
+        opensearch_version: [ '1.0.1', '1.1.0', '1.2.4', '1.3.7', '2.0.1', '2.1.0', '2.2.1', '2.3.0', '2.4.0', '2.5.0', '2.6.0', '2.7.0', '2.8.0', '2.9.0', '2.10.0' ]
         secured: [ "true", "false" ]
 
     steps:

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -7,7 +7,6 @@ jobs:
     strategy:
       matrix:
         entry:
-          - { os: 'ubuntu-20.04', python-version: "3.5" }
           - { os: 'ubuntu-20.04', python-version: "3.6" }
           - { os: 'ubuntu-latest', python-version: "3.7" }
           - { os: 'ubuntu-latest', python-version: "3.8" }
@@ -16,7 +15,7 @@ jobs:
           - { os: 'ubuntu-latest', python-version: "3.11" }
           - { os: 'macos-latest', python-version: "3.11" }
 
-    name: test (ruby=${{ matrix.entry.os }}, python=${{ matrix.entry.python-version }})
+    name: test (os=${{ matrix.entry.os }}, python=${{ matrix.entry.python-version }})
     continue-on-error: ${{ matrix.entry.experimental || false }}
     runs-on: ${{ matrix.entry.os }}
     steps:

diff --git a/.github/workflows/unified-release.yml b/.github/workflows/unified-release.yml
@@ -9,7 +9,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        stack_version: ['2.3.0']
+        stack_version: ['2.3.2']
 
     steps:
       - name: Checkout

diff --git a/.gitignore b/.gitignore
@@ -148,6 +148,9 @@ test_opensearch/cover
 test_opensearch/local.py
 .ci/output
 
-#Vi text editor
+# vi text editor
 .*.swp
 *~
+
+# Visual Studio Code
+.vscode
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,15 +4,57 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 ## [Unreleased]
 ### Added
 - Added generating imports and headers to API generator ([#467](https://github.com/opensearch-project/opensearch-py/pull/467))
+- Added point-in-time APIs (create_pit, delete_pit, delete_all_pits, get_all_pits) and Security Client APIs (health and update_audit_configuration) ([#502](https://github.com/opensearch-project/opensearch-py/pull/502))
+- Added guide on using index templates ([#531](https://github.com/opensearch-project/opensearch-py/pull/531))
+- Added `pool_maxsize` for `Urllib3HttpConnection` ([#535](https://github.com/opensearch-project/opensearch-py/pull/535))
+- Added benchmarks ([#537](https://github.com/opensearch-project/opensearch-py/pull/537))
+- Added guide on making raw JSON REST requests ([#542](https://github.com/opensearch-project/opensearch-py/pull/542))
+- Added support for AWS SigV4 for urllib3 ([#547](https://github.com/opensearch-project/opensearch-py/pull/547))
+- Added `remote store` client APIs ([#552](https://github.com/opensearch-project/opensearch-py/pull/552))
+- Added `nox -rs generate` ([#554](https://github.com/opensearch-project/opensearch-py/pull/554))
+- Added a utf-8 header to all .py files ([#557](https://github.com/opensearch-project/opensearch-py/pull/557))
+- Added `samples`, `benchmarks` and `docs` to `nox -rs format` ([#556](https://github.com/opensearch-project/opensearch-py/pull/556))
+- Added guide on the document lifecycle API(s) ([#559](https://github.com/opensearch-project/opensearch-py/pull/559))
 ### Changed
+- Generate `tasks` client from API specs ([#508](https://github.com/opensearch-project/opensearch-py/pull/508))
+- Generate `ingest` client from API specs ([#513](https://github.com/opensearch-project/opensearch-py/pull/513))
+- Generate `dangling_indices` client from API specs ([#511](https://github.com/opensearch-project/opensearch-py/pull/511))
+- Generate `cluster` client from API specs ([#530](https://github.com/opensearch-project/opensearch-py/pull/530))
+- Generate `nodes` client from API specs ([#514](https://github.com/opensearch-project/opensearch-py/pull/514))
+- Generate `cat` client from API specs ([#529](https://github.com/opensearch-project/opensearch-py/pull/529))
+- Use API generator for all APIs ([#551](https://github.com/opensearch-project/opensearch-py/pull/551))
+- Merge `.pyi` type stubs inline ([#563](https://github.com/opensearch-project/opensearch-py/pull/563))
+- Expanded type coverage to benchmarks, samples and tests ([#566](https://github.com/opensearch-project/opensearch-py/pull/566))
+- Defaulted `enable_cleanup_closed=True` in `aiohttp.TCPConnector` to prevent TLS connection leaks ([#468](https://github.com/opensearch-project/opensearch-py/pull/468))
 ### Deprecated
+- Deprecated point-in-time APIs (list_all_point_in_time, create_point_in_time, delete_point_in_time) and Security Client APIs (health_check and update_audit_config) ([#502](https://github.com/opensearch-project/opensearch-py/pull/502))
 ### Removed
+- Removed leftover support for Python 2.7 ([#548](https://github.com/opensearch-project/opensearch-py/pull/548))
 ### Fixed
-- Added `enable_cleanup_closed=True` argument to `aiohttp.TCPConnector` to fix TLS connection leaks ([#468](https://github.com/opensearch-project/opensearch-py/pull/468))
 ### Security
 ### Dependencies
-- Bumps `sphinx` from <7.1 to <7.2
-- Bumps `urllib3` from >=1.21.1, <2 to >=1.21.1 ([#466](https://github.com/opensearch-project/opensearch-py/pull/466))
+- Bumps `sphinx` from <7.1 to <7.3
+
+## [2.3.2]
+### Added
+### Changed
+### Deprecated
+### Removed
+### Fixed
+### Security
+### Dependencies
+- Bumps `urllib3` from >=1.21.1, <2 to >=1.26.9 ([#518](https://github.com/opensearch-project/opensearch-py/pull/518))
+
+## [2.3.1]
+### Added
+### Changed
+### Deprecated
+### Removed
+### Fixed
+- Fixed race condition in AWSV4SignerAuth & AWSV4SignerAsyncAuth when using refreshable credentials ([#470](https://github.com/opensearch-project/opensearch-py/pull/470))
+### Security
+### Dependencies
+- Bumps `urllib3` from >= 1.26.9 to >= 1.26.17 [#533](https://github.com/opensearch-project/opensearch-py/pull/533)
 
 ## [2.3.0]
 ### Added
@@ -24,7 +66,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Added support for the security plugin ([#399](https://github.com/opensearch-project/opensearch-py/pull/399))
 - Supports OpenSearch 2.1.0 - 2.6.0 ([#381](https://github.com/opensearch-project/opensearch-py/pull/381))
 - Added `allow_redirects` to `RequestsHttpConnection#perform_request` ([#401](https://github.com/opensearch-project/opensearch-py/pull/401))
-- Enhanced YAML test runner to use OpenSearch `rest-api-spec` YAML tests ([#414](https://github.com/opensearch-project/opensearch-py/pull/414)
+- Enhanced YAML test runner to use OpenSearch `rest-api-spec` YAML tests ([#414](https://github.com/opensearch-project/opensearch-py/pull/414))
 - Added `Search#collapse` ([#409](https://github.com/opensearch-project/opensearch-py/issues/409))
 - Added support for the ISM API ([#398](https://github.com/opensearch-project/opensearch-py/pull/398))
 - Added `trust_env` to `AIOHttpConnection` ([#398](https://github.com/opensearch-project/opensearch-py/pull/438))
@@ -39,6 +81,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 ### Deprecated
 ### Removed
 - Removed support for Python 2.7 ([#421](https://github.com/opensearch-project/opensearch-py/pull/421))
+- Removed support for Python 3.5 ([#533](https://github.com/opensearch-project/opensearch-py/pull/533))
 ### Fixed
 - Fixed flaky CI tests by replacing httpbin with a simple http_server ([#395](https://github.com/opensearch-project/opensearch-py/pull/395))
 - Fixed import cycle when importing async helpers ([#311](https://github.com/opensearch-project/opensearch-py/pull/311))
@@ -47,6 +90,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Include parsed error info in `TransportError` in async connections ([#226](https://github.com/opensearch-project/opensearch-py/pull/226))
 - Enhanced existing API generator to use OpenSearch OpenAPI spec ([#412](https://github.com/opensearch-project/opensearch-py/pull/412))
 - Fix crash when attempting to authenticate with an async connection ([#424](https://github.com/opensearch-project/opensearch-py/pull/424))
+- Fixed poetry run command issue on Windows/Mac machines ([#494](https://github.com/opensearch-project/opensearch-py/pull/494))
 ### Security
 - Fixed CVE-2022-23491 reported in opensearch-dsl-py ([#295](https://github.com/opensearch-project/opensearch-py/pull/295))
 ### Dependencies
@@ -109,11 +153,14 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Removed patch versions in integration tests for OpenSearch 1.0.0 - 2.3.0 to reduce Github Action jobs ([#262](https://github.com/opensearch-project/opensearch-py/pull/262))
 ### Fixed
 - Fixed DeprecationWarning emitted from urllib3 1.26.13+ ([#246](https://github.com/opensearch-project/opensearch-py/pull/246))
+- Fixed Wrong return type hint in `async_scan` ([520](https://github.com/opensearch-project/opensearch-py/pull/520))
 ### Security
 
-[Unreleased]: https://github.com/opensearch-project/opensearch-py/compare/v2.2.0...HEAD
+[Unreleased]: https://github.com/opensearch-project/opensearch-py/compare/v2.3.2...HEAD
 [2.0.1]: https://github.com/opensearch-project/opensearch-py/compare/v2.0.0...v2.0.1
 [2.1.0]: https://github.com/opensearch-project/opensearch-py/compare/v2.0.1...v2.1.0
 [2.1.1]: https://github.com/opensearch-project/opensearch-py/compare/v2.1.0...v2.1.1
 [2.2.0]: https://github.com/opensearch-project/opensearch-py/compare/v2.1.1...v2.2.0
 [2.3.0]: https://github.com/opensearch-project/opensearch-py/compare/v2.2.0...v2.3.0
+[2.3.1]: https://github.com/opensearch-project/opensearch-py/compare/v2.3.0...v2.3.1
+[2.3.2]: https://github.com/opensearch-project/opensearch-py/compare/v2.3.1...v2.3.2
diff --git a/COMPATIBILITY.md b/COMPATIBILITY.md
@@ -9,10 +9,12 @@ The below matrix shows the compatibility of the [`opensearch-py`](https://pypi.o
 | --- | --- | --- |
 | 1.0.0 | 1.0.0-1.2.4 | |
 | 1.1.0 | 1.3.0-1.3.7 | |
-| 2.0.x | 1.0.0-2.8.0 | client works against Opensearch Version 1.x as long as features removed in 2.0 are not used |
-| 2.1.x | 1.0.0-2.8.0 | client works against Opensearch Version 1.x as long as features removed in 2.0 are not used |
-| 2.2.0 | 1.0.0-2.8.0 | client works against Opensearch Version 1.x as long as features removed in 2.0 are not used |
-| 2.3.0 | 1.0.0-2.8.0 | client works against Opensearch Version 1.x as long as features removed in 2.0 are not used |
+| 2.0.x | 1.0.0-2.10.0 | client works against Opensearch Version 1.x as long as features removed in 2.0 are not used |
+| 2.1.x | 1.0.0-2.10.0 | client works against Opensearch Version 1.x as long as features removed in 2.0 are not used |
+| 2.2.0 | 1.0.0-2.10.0 | client works against Opensearch Version 1.x as long as features removed in 2.0 are not used |
+| 2.3.0 | 1.0.0-2.10.0 | client works against Opensearch Version 1.x as long as features removed in 2.0 are not used |
+| 2.3.1 | 1.0.0-2.10.0 | client works against Opensearch Version 1.x as long as features removed in 2.0 are not used |
+| 2.3.2 | 1.0.0-2.10.0 | client works against Opensearch Version 1.x as long as features removed in 2.0 are not used |
 
 ## Upgrading
 

diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md
@@ -1,10 +1,10 @@
 - [Developer Guide](#developer-guide)
   - [Prerequisites](#prerequisites)
-  - [Docker Image Installation](#docker-setup)
+  - [Install Docker Image](#install-docker-image)
   - [Running Tests](#running-tests)
-  - [Integration Tests](#integration-tests)
+  - [Linter](#linter)
   - [Documentation](#documentation)
-  - [Running Python Client Generator](#running-python-client-generator)
+  - [Client Code Generator](#client-code-generator)
 
 # Developer Guide
 
@@ -45,7 +45,19 @@ docker run -d -p 9200:9200 -p 9600:9600 -e "discovery.type=single-node" opensear
 
 Tests require a live instance of OpenSearch running in docker.
 
-This will start a new instance and run tests against the latest version of OpenSearch.
+If you have one running.
+
+```
+python setup.py test
+```
+
+To run tests in a specific test file.
+
+```
+python setup.py test -s test_opensearchpy/test_connection.py
+```
+
+If you want to auto-start one, the following will start a new instance and run tests against the latest version of OpenSearch.
 
 ```
 ./.ci/run-tests
@@ -54,9 +66,11 @@ This will start a new instance and run tests against the latest version of OpenS
 If your OpenSearch docker instance is running, you can execute the test suite directly.
 
 ```
-$ nox -rs test
+$ nox -rs test-3.9
 ```
 
+Substitute `3.9` with your Python version above, or use `nox -rs test` to run with multiple.
+
 To run tests against different versions of OpenSearch, use `run-tests [with/without security] [version]`:
 
 ```
@@ -76,7 +90,7 @@ You can also run individual tests matching a pattern (`pytest -k [pattern]`).
 ```
 ./.ci/run-tests true 1.3.0 test_no_http_compression
 
-test_opensearchpy/test_connection.py::TestUrllib3Connection::test_no_http_compression PASSED [ 33%]
+test_opensearchpy/test_connection.py::TestUrllib3HttpConnection::test_no_http_compression PASSED [ 33%]
 test_opensearchpy/test_connection.py::TestRequestsConnection::test_no_http_compression PASSED [ 66%]
 test_opensearchpy/test_async/test_connection.py::TestAIOHttpConnection::test_no_http_compression PASSED [100%]
 ```
@@ -103,12 +117,10 @@ make html
 
 Open `opensearch-py/docs/build/html/index.html` to see results.
 
-## Running Python Client Generator
+## Client Code Generator
 
-The following code executes a python client generator that updates the client by utilizing the [openapi specifications](https://github.com/opensearch-project/opensearch-api-specification/blob/main/OpenSearch.openapi.json) found in the "opensearch-api-specification" repository. This process allows for the automatic generation and synchronization of the client code with the latest API specifications.
+OpenSearch publishes an [OpenAPI specification](https://github.com/opensearch-project/opensearch-api-specification/blob/main/OpenSearch.openapi.json) in the [opensearch-api-specification](https://github.com/opensearch-project/opensearch-api-specification) repository, which is used to auto-generate the less interesting parts of the client.
 
 ```
-cd opensearch-py
-python utils/generate-api.py
-nox -rs format
+nox -rs generate
 ```
diff --git a/MAINTAINERS.md b/MAINTAINERS.md
@@ -13,6 +13,7 @@ This document contains a list of maintainers in this repo. See [opensearch-proje
 | Denis Zalevskiy           | [deztructor](https://github.com/deztructor)         | Aiven       |
 | Shephali Mittal           | [Shephalimittal](https://github.com/Shephalimittal) | Amazon      |
 | Sai Medhini Reddy Maryada | [saimedhi](https://github.com/saimedhi)             | Amazon      |
+| Florian Vazelle           | [florianvazelle](https://github.com/florianvazelle) | harfanglab  |
 
 ## Emeritus
 

diff --git a/MANIFEST.in b/MANIFEST.in
@@ -5,7 +5,7 @@ include LICENSE
 include MANIFEST.in
 include README.md
 include setup.py
-recursive-include opensearch* py.typed *.pyi
+recursive-include opensearch* py.typed
 
 prune test_opensearch
 recursive-exclude * __pycache__

diff --git a/README.md b/README.md
@@ -26,7 +26,7 @@ For more information, see [opensearch.org](https://opensearch.org/) and the [API
 
 ## User Guide
 
-To get started with the OpenSearch Python Client, see [User Guide](https://github.com/opensearch-project/opensearch-py/blob/main/USER_GUIDE.md).
+To get started with the OpenSearch Python Client, see [User Guide](https://github.com/opensearch-project/opensearch-py/blob/main/USER_GUIDE.md). This repository also contains [working samples](https://github.com/opensearch-project/opensearch-py/tree/main/samples) and [benchmarks](https://github.com/opensearch-project/opensearch-py/tree/main/benchmarks).
 
 ## Compatibility with OpenSearch
 

diff --git a/USER_GUIDE.md b/USER_GUIDE.md
@@ -153,6 +153,11 @@ print(response)
 - [Search](guides/search.md)
 - [Point in Time](guides/point_in_time.md)
 - [Using a Proxy](guides/proxy.md)
+- [Index Templates](guides/index_template.md)
+- [Advanced Index Actions](guides/advanced_index_actions.md)
+- [Making Raw JSON REST Requests](guides/json.md)
+- [Connection Classes](guides/connection_classes.md)
+- [Document Lifecycle](guides/document_lifecycle.md)
 
 ## Plugins
-Original file line number
+Diff line change
@@ Expand Up / @@ -2,9 +2,6 @@ TEST_SUITE: @@
       - oss
     PYTHON_VERSION:
-      - "2.7"
-      - "3.4"
-      - "3.5"
       - "3.6"
       - "3.7"
       - "3.8"
@@ Expand Down @@