From 3a0b22322c0c4263acaf029a4c57ad2ec8fc99cb Mon Sep 17 00:00:00 2001 From: Harsha Vamsi Kalluri Date: Thu, 18 Apr 2024 15:33:01 -0400 Subject: [PATCH 1/4] Adds newer OpenSearch builds to integration tests and updates unreleased wait mechanism Signed-off-by: dblock --- ...ait_for_es.php => wait_for_opensearch.php} | 0 .github/workflows/test.yml | 46 ++++++++++------ .github/workflows/test_unreleased.yml | 53 ++++++++++++------- .github/workflows/update_docs.yml | 4 +- CHANGELOG.md | 23 ++++---- 5 files changed, 78 insertions(+), 48 deletions(-) rename .github/{wait_for_es.php => wait_for_opensearch.php} (100%) diff --git a/.github/wait_for_es.php b/.github/wait_for_opensearch.php similarity index 100% rename from .github/wait_for_es.php rename to .github/wait_for_opensearch.php diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 49d1fd26b..9f9f394b6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,12 +12,12 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - - name: Use PHP 8.1 + - name: Use PHP 8.2 uses: shivammathur/setup-php@v2 with: - php-version: 8.1 + php-version: 8.2 extensions: yaml, zip, curl env: COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -35,12 +35,12 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - - name: Use PHP 8.1 + - name: Use PHP 8.2 uses: shivammathur/setup-php@v2 with: - php-version: 8.1 + php-version: 8.2 extensions: yaml, zip, curl env: COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -58,7 +58,7 @@ jobs: runs-on: ubuntu-latest services: search-server: - image: opensearchproject/opensearch:2.2.1 + image: opensearchproject/opensearch:2.6.0 ports: - 9200:9200 env: @@ -67,12 +67,12 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - - name: Use PHP 8.1 + - name: Use PHP 8.2 uses: shivammathur/setup-php@v2 with: - php-version: 8.1 + php-version: 8.2 extensions: yaml, zip, curl, pcov env: COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -82,7 +82,7 @@ jobs: composer install --prefer-dist - name: Wait for Search server - run: php ./.github/wait_for_es.php + run: php ./.github/wait_for_opensearch.php - name: PHPUnit run: | @@ -112,7 +112,7 @@ jobs: - windows-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Use PHP ${{ matrix.php-version }} uses: shivammathur/setup-php@v2 @@ -142,10 +142,22 @@ jobs: - docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.0 - opensearchproject/opensearch:1.1.0 - opensearchproject/opensearch:1.2.4 - - opensearchproject/opensearch:1.3.6 + - opensearchproject/opensearch:1.3.15 + - opensearchproject/opensearch:2.0.1 - opensearchproject/opensearch:2.1.0 - opensearchproject/opensearch:2.2.1 - opensearchproject/opensearch:2.5.0 + - opensearchproject/opensearch:2.3.0 + - opensearchproject/opensearch:2.4.1 + - opensearchproject/opensearch:2.5.0 + - opensearchproject/opensearch:2.6.0 + - opensearchproject/opensearch:2.7.0 + - opensearchproject/opensearch:2.8.0 + - opensearchproject/opensearch:2.9.0 + - opensearchproject/opensearch:2.10.0 + - opensearchproject/opensearch:2.11.1 + - opensearchproject/opensearch:2.12.0 + - opensearchproject/opensearch:2.13.0 services: search-server: image: ${{ matrix.search-server-image }} @@ -157,12 +169,12 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - - name: Use PHP 8.1 + - name: Use PHP 8.2 uses: shivammathur/setup-php@v2 with: - php-version: 8.1 + php-version: 8.2 extensions: yaml, zip, curl env: COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -172,7 +184,7 @@ jobs: composer install --prefer-dist - name: Wait for Search server - run: php ./.github/wait_for_es.php + run: php ./.github/wait_for_opensearch.php - name: Integration tests run: | diff --git a/.github/workflows/test_unreleased.yml b/.github/workflows/test_unreleased.yml index 43dfb6433..a1f5697af 100644 --- a/.github/workflows/test_unreleased.yml +++ b/.github/workflows/test_unreleased.yml @@ -14,37 +14,53 @@ jobs: strategy: fail-fast: false matrix: - entry: - - { opensearch_ref: '1.x' } - - { opensearch_ref: '2.0' } - - { opensearch_ref: '2.x' } - - { opensearch_ref: 'main' } + opensearch_ref: [ '1.x', '2.x', '2.0', 'main' ] steps: - name: Checkout OpenSearch - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: repository: opensearch-project/OpenSearch - ref: ${{ matrix.entry.opensearch_ref }} + ref: ${{ matrix.opensearch_ref }} path: opensearch + - name: Get OpenSearch branch top + id: get-key + working-directory: opensearch + run: echo key=`git log -1 --format='%H'` >> $GITHUB_OUTPUT + + - name: Restore cached build + id: cache-restore + uses: actions/cache/restore@v3 + with: + path: opensearch/distribution/archives/linux-tar/build/distributions + key: ${{ steps.get-key.outputs.key }} + - name: Assemble OpenSearch + if: steps.cache-restore.outputs.cache-hit != 'true' + working-directory: opensearch + run: ./gradlew :distribution:archives:linux-tar:assemble + + - name: Save cached build + if: steps.cache-restore.outputs.cache-hit != 'true' + uses: actions/cache/save@v3 + with: + path: opensearch/distribution/archives/linux-tar/build/distributions + key: ${{ steps.get-key.outputs.key }} + + - name: Run OpenSearch + working-directory: opensearch/distribution/archives/linux-tar/build/distributions run: | - cd opensearch - ./gradlew assemble - # This step runs the docker image generated during gradle assemble in OpenSearch. It is tagged as opensearch:test. - # Reference: https://github.com/opensearch-project/OpenSearch/blob/2.0/distribution/docker/build.gradle#L190 - - name: Run Docker Image - run: | - docker run -p 9200:9200 -p 9600:9600 -d -e "discovery.type=single-node" -e "bootstrap.memory_lock=true" opensearch:test - sleep 90 + tar xf opensearch-min-* + ./opensearch-*/bin/opensearch & + for attempt in {1..20}; do sleep 5; if curl -s localhost:9200; then echo '=====> ready'; break; fi; echo '=====> waiting...'; done - name: Checkout PHP Client uses: actions/checkout@v2 - - name: Use PHP 8.1 + - name: Use PHP 8.2 uses: shivammathur/setup-php@v2 with: - php-version: 8.1 + php-version: 8.2 extensions: yaml, zip, curl env: COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -54,7 +70,8 @@ jobs: composer install --prefer-dist - name: Wait for Search server - run: php ./.github/wait_for_es.php + run: php ./.github/wait_for_opensearch.php + - name: Integration tests run: | diff --git a/.github/workflows/update_docs.yml b/.github/workflows/update_docs.yml index 623465e20..be4aaca6c 100644 --- a/.github/workflows/update_docs.yml +++ b/.github/workflows/update_docs.yml @@ -25,10 +25,10 @@ jobs: with: fetch-depth: 0 - - name: Use PHP 8.1 + - name: Use PHP 8.2 uses: shivammathur/setup-php@v2 with: - php-version: 8.1 + php-version: 8.2 env: COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/CHANGELOG.md b/CHANGELOG.md index ec1eceebf..4c1160153 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,36 +1,37 @@ # CHANGELOG + Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ## [Unreleased] + ### Added -- Added PHP 8.2 to the CI -- Github workflow for changelog verification ([#92](https://github.com/opensearch-project/opensearch-php/pull/92)) +- Added a GitHub workflow for verifying CHANGELOG ([#92](https://github.com/opensearch-project/opensearch-php/pull/92)) - Added class docs generator ([#96](https://github.com/opensearch-project/opensearch-php/pull/96)) - Added support for Amazon OpenSearch Serverless SigV4 signing ([#119](https://github.com/opensearch-project/opensearch-php/pull/119)) - Added `includePortInHostHeader` option to `ClientBuilder::fromConfig` ([#118](https://github.com/opensearch-project/opensearch-php/pull/118)) - Added the `RefreshSearchAnalyzers` endpoint ([[#152](https://github.com/opensearch-project/opensearch-php/issues/152)) - Added support for `format` parameter to specify the sql response format ([#161](https://github.com/opensearch-project/opensearch-php/pull/161)) -- Added ml commons model, model group and connector APIs ([#170](https://github.com/opensearch-project/opensearch-php/pull/170)) -- [Feature] Send raw json request ([#171](https://github.com/opensearch-project/opensearch-php/pull/177)) +- Added ml-commons model, model group and connector APIs ([#170](https://github.com/opensearch-project/opensearch-php/pull/170)) +- Added support for sending raw JSON requests ([#171](https://github.com/opensearch-project/opensearch-php/pull/177)) +- Added PHP 8.2 support ([#87](https://github.com/opensearch-project/opensearch-php/issues/87)) +- Added Windows and MacOS support ([#100](https://github.com/opensearch-project/opensearch-php/pull/100)) +- Added code coverage reporting ([#100](https://github.com/opensearch-project/opensearch-php/pull/100)) +- Added support for a custom signing service name for AWS SigV4 ([#117](https://github.com/opensearch-project/opensearch-php/pull/117)) ### Changed -- Added Windows and macOS jobs to CI -- Added Coverage reporting to Codecov -- Support a custom signing service name for AWS SigV4 ([#117](https://github.com/opensearch-project/opensearch-php/pull/117)) - ### Deprecated ### Removed ### Fixed -- Updated backport helper version and add fix to backport workflow when tag is applied before closing PR ([#131](https://github.com/opensearch-project/opensearch-php/pull/131)) + +- Fixed backport workflow when tag is applied before closing PR ([#131](https://github.com/opensearch-project/opensearch-php/pull/131)) - Fixed host urls with trailing slash in the url ([#130](https://github.com/opensearch-project/opensearch-php/pull/140)) - Fixed point-in-time APIs ([#142](https://github.com/opensearch-project/opensearch-php/pull/142)) -- Fixed bug in ClientBuilder where basic authentication is overridden by connection params ([#160](https://github.com/opensearch-project/opensearch-php/pull/160)) +- Fixed basic authentication being overridden by connection params in `ClientBuilder` ([#160](https://github.com/opensearch-project/opensearch-php/pull/160)) - Fixed PHP warning in `Connection::tryDeserializeError()` for some error responses ([#167](https://github.com/opensearch-project/opensearch-php/issues/167)) ### Security - [Unreleased]: https://github.com/opensearch-project/opensearch-php/compare/2.0...HEAD From 7d11085930df58c23d90830b0428e08e3a6305ac Mon Sep 17 00:00:00 2001 From: dblock Date: Thu, 18 Apr 2024 15:40:58 -0400 Subject: [PATCH 2/4] Display OpenSearch version in integration tests. Signed-off-by: dblock --- .github/wait_for_opensearch.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/wait_for_opensearch.php b/.github/wait_for_opensearch.php index 10c6303ac..fb252a246 100644 --- a/.github/wait_for_opensearch.php +++ b/.github/wait_for_opensearch.php @@ -11,7 +11,8 @@ try { $client = ClientBuilder::create()->build(); $client->ping(); - echo 'Is up and running' . PHP_EOL; + $info = $client->info(); + echo 'OpenSearch ' . $info['version']['number'] . ' is up and running' . PHP_EOL; exit(0); } catch (Throwable $e) { if ($retries === $maxRetries) { From 2a1552c013f4641735a255342a859fe3e928970f Mon Sep 17 00:00:00 2001 From: dblock Date: Thu, 18 Apr 2024 15:59:13 -0400 Subject: [PATCH 3/4] Added support for OPENSEARCH_INITIAL_ADMIN_PASSWORD with OpenSearch 2.12/13. Signed-off-by: dblock --- .github/workflows/test.yml | 97 +++++++++++++++++++++++++-- .github/workflows/test_unreleased.yml | 1 + CHANGELOG.md | 1 + tests/Utility.php | 5 +- 4 files changed, 98 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9f9f394b6..831aba370 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -58,12 +58,13 @@ jobs: runs-on: ubuntu-latest services: search-server: - image: opensearchproject/opensearch:2.6.0 + image: opensearchproject/opensearch:2.13.0 ports: - 9200:9200 env: discovery.type: single-node - plugins.security.disabled: 'true' + plugins.security.disabled: true + OPENSEARCH_INITIAL_ADMIN_PASSWORD: myStrongPassword123! steps: - name: Checkout @@ -130,8 +131,8 @@ jobs: run: | composer run unit - integration-test: - name: Tntegration Test + integration-test-elasticsearch: + name: Tntegration Test (Elasticsearch) runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -140,6 +141,48 @@ jobs: - ubuntu-latest search-server-image: - docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.0 + services: + search-server: + image: ${{ matrix.search-server-image }} + ports: + - 9200:9200 + env: + discovery.type: single-node + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Use PHP 8.2 + uses: shivammathur/setup-php@v2 + with: + php-version: 8.2 + extensions: yaml, zip, curl + env: + COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Install dependencies + run: | + composer install --prefer-dist + + - name: Wait for Search server + run: php ./.github/wait_for_opensearch.php + + - name: Integration tests + run: | + composer run integration + env: + OPENSEARCH_URL: 'http://localhost:9200' + + integration-test-opensearch: + name: Tntegration Test (OpenSearch) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + - ubuntu-latest + search-server-image: - opensearchproject/opensearch:1.1.0 - opensearchproject/opensearch:1.2.4 - opensearchproject/opensearch:1.3.15 @@ -156,6 +199,49 @@ jobs: - opensearchproject/opensearch:2.9.0 - opensearchproject/opensearch:2.10.0 - opensearchproject/opensearch:2.11.1 + services: + search-server: + image: ${{ matrix.search-server-image }} + ports: + - 9200:9200 + env: + discovery.type: single-node + plugins.security.disabled: true + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Use PHP 8.2 + uses: shivammathur/setup-php@v2 + with: + php-version: 8.2 + extensions: yaml, zip, curl + env: + COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Install dependencies + run: | + composer install --prefer-dist + + - name: Wait for Search server + run: php ./.github/wait_for_opensearch.php + + - name: Integration tests + run: | + composer run integration + env: + OPENSEARCH_URL: 'http://localhost:9200' + + integration-test-opensearch-strong-password: + name: Tntegration Test (OpenSearch w/Strong Password) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + - ubuntu-latest + search-server-image: - opensearchproject/opensearch:2.12.0 - opensearchproject/opensearch:2.13.0 services: @@ -165,7 +251,8 @@ jobs: - 9200:9200 env: discovery.type: single-node - plugins.security.disabled: '${{ contains(matrix.search-server-image, ''opensearch'') && ''true'' || '''' }}' + plugins.security.disabled: true + OPENSEARCH_INITIAL_ADMIN_PASSWORD: myStrongPassword123! steps: - name: Checkout diff --git a/.github/workflows/test_unreleased.yml b/.github/workflows/test_unreleased.yml index a1f5697af..ac681f6e5 100644 --- a/.github/workflows/test_unreleased.yml +++ b/.github/workflows/test_unreleased.yml @@ -51,6 +51,7 @@ jobs: working-directory: opensearch/distribution/archives/linux-tar/build/distributions run: | tar xf opensearch-min-* + echo "discovery.type: single-node" >> ./opensearch-*/config/opensearch.yml ./opensearch-*/bin/opensearch & for attempt in {1..20}; do sleep 5; if curl -s localhost:9200; then echo '=====> ready'; break; fi; echo '=====> waiting...'; done diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c1160153..d1c66a194 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added Windows and MacOS support ([#100](https://github.com/opensearch-project/opensearch-php/pull/100)) - Added code coverage reporting ([#100](https://github.com/opensearch-project/opensearch-php/pull/100)) - Added support for a custom signing service name for AWS SigV4 ([#117](https://github.com/opensearch-project/opensearch-php/pull/117)) +- Added support for OpenSearch 2.12 and 2.13 ([#180](https://github.com/opensearch-project/opensearch-php/pull/180)) ### Changed diff --git a/tests/Utility.php b/tests/Utility.php index 5297eadc9..78a440071 100644 --- a/tests/Utility.php +++ b/tests/Utility.php @@ -42,7 +42,10 @@ public static function getHost(): ?string if (false !== $url) { return $url; } - return 'https://admin:admin@localhost:9200'; + + $password = getenv('OPENSEARCH_INITIAL_ADMIN_PASSWORD') ?: 'admin'; + + return 'https://admin:' . $password . '@localhost:9200'; } /** From 42b6fe758bcc406fd132755a3e5f718b2509b83d Mon Sep 17 00:00:00 2001 From: dblock Date: Fri, 19 Apr 2024 08:20:17 -0400 Subject: [PATCH 4/4] Set discovery.type. Signed-off-by: dblock --- .github/workflows/test_unreleased.yml | 35 ++++++++++++--------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/.github/workflows/test_unreleased.yml b/.github/workflows/test_unreleased.yml index ac681f6e5..ac03cedb5 100644 --- a/.github/workflows/test_unreleased.yml +++ b/.github/workflows/test_unreleased.yml @@ -16,6 +16,21 @@ jobs: matrix: opensearch_ref: [ '1.x', '2.x', '2.0', 'main' ] steps: + - name: Checkout PHP Client + uses: actions/checkout@v2 + + - name: Use PHP 8.2 + uses: shivammathur/setup-php@v2 + with: + php-version: 8.2 + extensions: yaml, zip, curl + env: + COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Install dependencies + run: | + composer install --prefer-dist + - name: Checkout OpenSearch uses: actions/checkout@v3 with: @@ -51,29 +66,11 @@ jobs: working-directory: opensearch/distribution/archives/linux-tar/build/distributions run: | tar xf opensearch-min-* - echo "discovery.type: single-node" >> ./opensearch-*/config/opensearch.yml - ./opensearch-*/bin/opensearch & - for attempt in {1..20}; do sleep 5; if curl -s localhost:9200; then echo '=====> ready'; break; fi; echo '=====> waiting...'; done + ./opensearch-*/bin/opensearch -d - - name: Checkout PHP Client - uses: actions/checkout@v2 - - - name: Use PHP 8.2 - uses: shivammathur/setup-php@v2 - with: - php-version: 8.2 - extensions: yaml, zip, curl - env: - COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Install dependencies - run: | - composer install --prefer-dist - - name: Wait for Search server run: php ./.github/wait_for_opensearch.php - - name: Integration tests run: | composer run integration