From 96c8d4d5729e103fb1dc5aab4247db4bbabdb0d9 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Mon, 30 Oct 2023 14:14:26 -0400 Subject: [PATCH] Onboarding Jenkins prod docker images to github ci check (#1538) * Onboarding Jenkins prod docker images to github ci check Signed-off-by: Peter Zhu * Add suggested changes in the comments Signed-off-by: Peter Zhu --------- Signed-off-by: Peter Zhu --- .github/workflows/CI-workflow.yml | 113 ++++++++++++++++++++++-------- 1 file changed, 82 insertions(+), 31 deletions(-) diff --git a/.github/workflows/CI-workflow.yml b/.github/workflows/CI-workflow.yml index 8472803dfc..1933d9840f 100644 --- a/.github/workflows/CI-workflow.yml +++ b/.github/workflows/CI-workflow.yml @@ -14,15 +14,29 @@ permissions: contents: read jobs: - Build-ml: + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main + with: + product: opensearch + + Build-ml-linux: + needs: Get-CI-Image-Tag strategy: matrix: java: [11, 17, 20] - name: Build and Test MLCommons Plugin + name: Build and Test MLCommons Plugin on linux if: github.repository == 'opensearch-project/ml-commons' environment: ml-commons-cicd-env + outputs: + build-test-linux: ${{ steps.step-build-test-linux.outputs.build-test-linux }} runs-on: ubuntu-latest + container: + # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + # this image tag is subject to change as more dependencies and updates will arrive over time + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + # need to switch to root so that github actions can install runner binary on container without permission issues. + options: --user root steps: - name: Setup Java ${{ matrix.java }} @@ -41,43 +55,86 @@ jobs: ref: ${{ github.event.pull_request.head.sha }} - name: Build and Run Tests + id: step-build-test-linux run: | - export OPENAI_KEY=$(aws secretsmanager get-secret-value --secret-id github_openai_key --query SecretString --output text) - export COHERE_KEY=$(aws secretsmanager get-secret-value --secret-id github_cohere_key --query SecretString --output text) - echo "::add-mask::$OPENAI_KEY" - echo "::add-mask::$COHERE_KEY" - ./gradlew build - - name: Publish to Maven Local - run: | - ./gradlew publishToMavenLocal - - name: Multi Nodes Integration Testing - run: | - export OPENAI_KEY=$(aws secretsmanager get-secret-value --secret-id github_openai_key --query SecretString --output text) - export COHERE_KEY=$(aws secretsmanager get-secret-value --secret-id github_cohere_key --query SecretString --output text) - echo "::add-mask::$OPENAI_KEY" - echo "::add-mask::$COHERE_KEY" - ./gradlew integTest -PnumNodes=3 + chown -R 1000:1000 `pwd` + su `id -un 1000` -c 'whoami && java -version && + export OPENAI_KEY=`aws secretsmanager get-secret-value --secret-id github_openai_key --query SecretString --output text` && + export COHERE_KEY=`aws secretsmanager get-secret-value --secret-id github_cohere_key --query SecretString --output text` && + echo "::add-mask::$OPENAI_KEY" && + echo "::add-mask::$COHERE_KEY" && + echo "build and run tests" && ./gradlew build && + echo "Publish to Maven Local" && ./gradlew publishToMavenLocal && + echo "Multi Nodes Integration Testing" && ./gradlew integTest -PnumNodes=3' + plugin=`basename $(ls plugin/build/distributions/*.zip)` + echo $plugin + mv -v plugin/build/distributions/$plugin ./ + echo "build-test-linux=$plugin" >> $GITHUB_OUTPUT + + - name: Upload Coverage Report + uses: codecov/codecov-action@v1 + with: + flags: ml-commons + token: ${{ secrets.CODECOV_TOKEN }} + + - uses: actions/upload-artifact@v3 + with: + name: ml-plugin-linux-${{ matrix.java }} + path: ${{ steps.step-build-test-linux.outputs.build-test-linux }} + if-no-files-found: error + + + Test-ml-linux-docker: + needs: Build-ml-linux + strategy: + matrix: + java: [11, 17, 20] + + name: Test MLCommons Plugin on linux docker + if: github.repository == 'opensearch-project/ml-commons' + environment: ml-commons-cicd-env + runs-on: ubuntu-latest + + steps: + - uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ secrets.ML_ROLE }} + aws-region: us-west-2 + + - name: Checkout MLCommons + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - uses: actions/download-artifact@v3 + with: + name: ml-plugin-linux-${{ matrix.java }} - name: Pull and Run Docker run: | - plugin=`basename $(ls plugin/build/distributions/*.zip)` + plugin=${{ needs.Build-ml-linux.outputs.build-test-linux }} version=`echo $plugin|awk -F- '{print $3}'| cut -d. -f 1-3` plugin_version=`echo $plugin|awk -F- '{print $3}'| cut -d. -f 1-4` qualifier=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-1` - candidate_version=`echo $plugin|awk -F- '{print $5}'| cut -d. -f 1-1` - docker_version=$version-$qualifier - [[ -z $candidate_version ]] && candidate_version=$qualifier && qualifier="" + if [ -n "$qualifier" ] && [ "$qualifier" != "SNAPSHOT" ]; then + qualifier=-${qualifier} + else + qualifier="" + fi - echo plugin version plugin_version qualifier candidate_version docker_version - echo "($plugin) ($version) ($plugin_version) ($qualifier) ($candidate_version) ($docker_version)" + docker_version=$version$qualifier + + echo plugin version plugin_version qualifier docker_version + echo "($plugin) ($version) ($plugin_version) ($qualifier) ($docker_version)" + + pwd && ls -l ./$plugin - cd .. if docker pull opensearchstaging/opensearch:$docker_version then echo "FROM opensearchstaging/opensearch:$docker_version" >> Dockerfile echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-ml ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-ml; fi" >> Dockerfile - echo "ADD ml-commons/plugin/build/distributions/$plugin /tmp/" >> Dockerfile + echo "COPY $plugin /tmp/" >> Dockerfile echo "RUN /usr/share/opensearch/bin/opensearch-plugin install --batch file:/tmp/$plugin" >> Dockerfile docker build -t opensearch-ml:test . echo "imagePresent=true" >> $GITHUB_ENV @@ -107,12 +164,6 @@ jobs: ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster" fi - - name: Upload Coverage Report - uses: codecov/codecov-action@v1 - with: - flags: ml-commons - token: ${{ secrets.CODECOV_TOKEN }} - Build-ml-windows: strategy: matrix: