From 114057d81f548693c51de339930fb0745cbe14fa Mon Sep 17 00:00:00 2001 From: ajasnosz Date: Wed, 19 Jun 2024 14:04:37 +0200 Subject: [PATCH 1/3] chore: setup pcf environment --- .github/pre-req.sh | 13 ++- .github/update_manifest.sh | 4 +- .github/workflows/main.yml | 216 +++++++++++++++++++++++++++---------- 3 files changed, 177 insertions(+), 56 deletions(-) diff --git a/.github/pre-req.sh b/.github/pre-req.sh index bec30214..a69ac4db 100755 --- a/.github/pre-req.sh +++ b/.github/pre-req.sh @@ -10,7 +10,9 @@ sudo apt-get update sudo apt-get install apt-transport-https ca-certificates sudo apt-get install cf-cli #CF Login -cf login --skip-ssl-validation -a $API_ENDPOINT -u $API_USER -p $API_PASSWORD -o system -s system +API_PASSWORD_DEC=$(echo "$API_PASSWORD" | openssl aes-256-cbc -d -pbkdf2 -a -pass pass:"$ENCRYPT_KEY") +cf login --skip-ssl-validation -a "$API_ENDPOINT" -u "$API_USER" -p "$API_PASSWORD_DEC" + #Create splunk-ci org and space if [ "`cf o | grep "splunk-ci-org"`" == "splunk-ci-org" ]; then echo "splunk-ci-org org already exists" @@ -21,4 +23,13 @@ else cf target -o splunk-ci-org cf create-space splunk-ci-space cf target -o "splunk-ci-org" -s "splunk-ci-space" +fi + +gem install cf-uaac +uaac target "$API_UAA_ENDPOINT" --skip-ssl-validation +API_CLIENT_PASSWORD_DEC=$(echo "$API_CLIENT_PASSWORD" | openssl aes-256-cbc -d -pbkdf2 -a -pass pass:"$ENCRYPT_KEY") +uaac token client get "$API_USER" -s "$API_CLIENT_PASSWORD_DEC" + +if [ $(uaac client get "$CLIENT_ID" | grep -woc "$CLIENT_ID") -eq 0 ]; then + uaac client add "$CLIENT_ID" --name splunk-firehose --secret "$CLIENT_SECRET" --authorized_grant_types client_credentials,refresh_token --authorities doppler.firehose,cloud_controller.admin_read_only fi \ No newline at end of file diff --git a/.github/update_manifest.sh b/.github/update_manifest.sh index 7d3baaa0..f144062f 100755 --- a/.github/update_manifest.sh +++ b/.github/update_manifest.sh @@ -2,10 +2,12 @@ set -e #Set below params in github env variable settings # API_ENDPOINT, API_USER, API_PASSWORD, SPLUNK_TOKEN, SPLUNK_HOST, SPLUNK_INDEX, SPLUNK_METRIC_INDEX +API_PASSWORD_DEC=$(echo "$API_PASSWORD" | openssl aes-256-cbc -d -pbkdf2 -a -pass pass:"$ENCRYPT_KEY") +echo "$API_PASSWORD_DEC" #Update manifest for deployment sed -i 's@API_ENDPOINT:.*@'"API_ENDPOINT: $API_ENDPOINT"'@' scripts/ci_nozzle_manifest.yml sed -i 's@API_USER:.*@'"API_USER: $API_USER"'@' scripts/ci_nozzle_manifest.yml -sed -i 's@API_PASSWORD:.*@'"API_PASSWORD: $API_PASSWORD"'@' scripts/ci_nozzle_manifest.yml +sed -i 's@API_PASSWORD:.*@'"API_PASSWORD: $API_PASSWORD_DEC"'@' scripts/ci_nozzle_manifest.yml sed -i 's@CLIENT_ID:.*@'"CLIENT_ID: $CLIENT_ID"'@' scripts/ci_nozzle_manifest.yml sed -i 's@CLIENT_SECRET:.*@'"CLIENT_SECRET: $CLIENT_SECRET"'@' scripts/ci_nozzle_manifest.yml sed -i 's@SPLUNK_HOST:.*@'"SPLUNK_HOST: $SPLUNK_HOST"'@' scripts/ci_nozzle_manifest.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e7597637..5a1a4499 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -4,7 +4,7 @@ on: branches: [ "develop" ] pull_request: branches: [ "develop" ] - + workflow_dispatch: jobs: @@ -44,7 +44,7 @@ jobs: fossa test --debug env: FOSSA_API_KEY: ${{ secrets.FOSSA_API_KEY }} - + semgrep: runs-on: ubuntu-latest needs: @@ -59,10 +59,74 @@ jobs: with: publishToken: ${{ secrets.SEMGREP_PUBLISH_TOKEN }} + create-env: + needs: + - workflow_approval + runs-on: ubuntu-latest + timeout-minutes: 60 + outputs: + API_ENDPOINT: ${{ steps.get-credentials.outputs.API_ENDPOINT }} + API_UAA_ENDPOINT: ${{ steps.get-credentials.outputs.API_UAA_ENDPOINT }} + API_PASSWORD: ${{ steps.get-credentials.outputs.API_PASSWORD }} + API_CLIENT_PASSWORD: ${{ steps.get-credentials.outputs.API_CLIENT_PASSWORD }} + ENV_ID: ${{ steps.get-credentials.outputs.ENV_ID }} + steps: + - name: Install Selfservice + run: | + wget https://github.com/cf-platform-eng/selfservice/releases/download/0.2.9/selfservice + chmod +x selfservice + ls -latr + + - name: Get authorization token + run: | + echo "$(./selfservice auth $API_TOKEN | cut -c 8-)" >> "$GITHUB_ENV" + echo "::add-mask::$AUTH_TOKEN" + env: + API_TOKEN: ${{ secrets.API_TOKEN }} + + - name: Claim environment and wait + run: | + echo ENV_ID="$(./selfservice claimAndWait isv_ci_tas_srt_4_0 | jq -r '.id')" >> "$GITHUB_ENV" + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: '1.18.1' + + - name: Set up Brew + uses: Homebrew/actions/setup-homebrew@master + + - name: Install hammer and cf + run: | + brew tap pivotal/hammer https://github.com/pivotal/hammer + brew install hammer + + brew tap pivotal-cf/om https://github.com/pivotal-cf/om + brew install om + brew install cloudfoundry/tap/cf-cli + + - name: Credentials + id: get-credentials + run: | + ./selfservice get $ENV_ID | jq -r '.credentials' > env.json + echo API_ENDPOINT="https://api.$(jq -r '.sys_domain' ./env.json)" >> "$GITHUB_OUTPUT" + echo API_UAA_ENDPOINT="https://uaa.$(jq -r '.sys_domain' ./env.json)" >> "$GITHUB_OUTPUT" + API_PASSWORD="$(hammer -t ./env.json om credentials -- -p cf -t json -c .uaa.admin_credentials | jq -r '.password')" + API_PASSWORD_ENC="$(echo $API_PASSWORD | openssl aes-256-cbc -a -pbkdf2 -salt -pass pass:$ENCRYPT_KEY)" + echo API_PASSWORD=$API_PASSWORD_ENC >> "$GITHUB_OUTPUT" + API_CLIENT_PASSWORD="$(hammer -t ./env.json om credentials -- -p cf -t json -c .uaa.admin_client_credentials | jq -r '.password')" + API_CLIENT_PASSWORD_ENC="$(echo $API_CLIENT_PASSWORD | openssl aes-256-cbc -a -pbkdf2 -salt -pass pass:$ENCRYPT_KEY)" + echo API_CLIENT_PASSWORD=$API_CLIENT_PASSWORD_ENC >> "$GITHUB_OUTPUT" + echo ENV_ID=$ENV_ID >> "$GITHUB_OUTPUT" + env: + ENCRYPT_KEY: ${{ secrets.ENCRYPT_KEY }} + + build: runs-on: ubuntu-latest needs: - - workflow_approval + - create-env + steps: - name: Checkout uses: actions/checkout@v3 @@ -71,7 +135,7 @@ jobs: repository: ${{github.event.pull_request.head.repo.full_name}} - uses: actions/setup-go@v3 with: - go-version: '1.17.0' + go-version: '1.18.1' - run: go version # Install Dependencies @@ -79,7 +143,7 @@ jobs: run: ls -la - name: Install Dependencies run: | - go mod vendor + go mod vendor # Builder - name: Builder @@ -96,10 +160,11 @@ jobs: path: splunk-firehose-nozzle deploy_nozzle: - - env: - API_ENDPOINT: ${{ secrets.API_ENDPOINT }} - API_PASSWORD: ${{ secrets.API_PASSWORD }} + env: + API_ENDPOINT: ${{ needs.create-env.outputs.API_ENDPOINT }} + API_PASSWORD: ${{ needs.create-env.outputs.API_PASSWORD }} + API_CLIENT_PASSWORD: ${{ needs.create-env.outputs.API_CLIENT_PASSWORD }} + API_UAA_ENDPOINT: ${{ needs.create-env.outputs.API_UAA_ENDPOINT }} API_USER: ${{ secrets.API_USER }} CLIENT_ID: ${{ secrets.CLIENT_ID }} CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }} @@ -108,20 +173,26 @@ jobs: SPLUNK_HOST: ${{ secrets.SPLUNK_HOST }} SPLUNK_METRIC_INDEX: ${{ secrets.SPLUNK_METRIC_INDEX }} - needs: build + needs: + - build + - create-env runs-on: ubuntu-latest steps: - - name: Checkout uses: actions/checkout@v3 with: ref: ${{github.event.pull_request.head.sha}} repository: ${{github.event.pull_request.head.repo.full_name}} + - uses: actions/setup-go@v3 with: - go-version: '1.17.0' + go-version: '1.18.1' - run: go version + - uses: ruby/setup-ruby@v1 + with: + ruby-version: '3.3' + - run: ruby -v - uses: actions/download-artifact@v3 with: @@ -130,21 +201,29 @@ jobs: # Install dependencies - name: Install dependencies run: | - go mod vendor + go mod vendor # Deploy nozzle - name: Deploy nozzle run: | - .github/update_manifest.sh - .github/pre-req.sh - cf push -f scripts/ci_nozzle_manifest.yml -u process --random-route + .github/update_manifest.sh + .github/pre-req.sh + cf push -f scripts/ci_nozzle_manifest.yml -u process --random-route + env: + ENCRYPT_KEY: ${{ secrets.ENCRYPT_KEY }} + # Nozzle Log + - name: Nozzle Log + run: | + cf logs splunk-firehose-nozzle & + tile-builder: - - env: - API_ENDPOINT: ${{ secrets.API_ENDPOINT }} - API_PASSWORD: ${{ secrets.API_PASSWORD }} + env: + API_ENDPOINT: ${{ needs.create-env.outputs.API_ENDPOINT }} + API_PASSWORD: ${{ needs.create-env.outputs.API_PASSWORD }} + API_CLIENT_PASSWORD: ${{ needs.create-env.outputs.API_CLIENT_PASSWORD }} + API_UAA_ENDPOINT: ${{ needs.create-env.outputs.API_UAA_ENDPOINT }} API_USER: ${{ secrets.API_USER }} CLIENT_ID: ${{ secrets.CLIENT_ID }} CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }} @@ -153,37 +232,32 @@ jobs: SPLUNK_HOST: ${{ secrets.SPLUNK_HOST }} SPLUNK_METRIC_INDEX: ${{ secrets.SPLUNK_METRIC_INDEX }} - - needs: deploy_nozzle + needs: + - deploy_nozzle + - create-env runs-on: ubuntu-latest steps: - - name: Checkout uses: actions/checkout@v3 with: ref: ${{github.event.pull_request.head.sha}} repository: ${{github.event.pull_request.head.repo.full_name}} + - uses: actions/setup-go@v3 with: - go-version: '1.17.0' + go-version: '1.18.1' - run: go version - - uses: actions/download-artifact@v3 with: name: splunk-firehose-nozzle - + # Tile builder - name: Tile builder run: | - .github/tile-builder.sh - - # Nozzle Log - - name: Nozzle Log - run: | - cf logs splunk-firehose-nozzle & - + .github/tile-builder.sh + - name: Get tile name run: | echo "tile_name=$(ls tile/product | grep ".pivotal")" >> "$GITHUB_ENV" @@ -191,18 +265,21 @@ jobs: - name: Upload tile uses: actions/upload-artifact@v2 with: - name: ${{ env.tile_name }} - path: tile/product/${{ env.tile_name }} + name: ${{ env.tile_name }} + path: tile/product/${{ env.tile_name }} # Skip test for now! execute_tests: - - needs: tile-builder + needs: + - tile-builder + - create-env runs-on: ubuntu-latest - env: - API_ENDPOINT: ${{ secrets.API_ENDPOINT }} - API_PASSWORD: ${{ secrets.API_PASSWORD }} + env: + API_ENDPOINT: ${{ needs.create-env.outputs.API_ENDPOINT }} + API_PASSWORD: ${{ needs.create-env.outputs.API_PASSWORD }} + API_CLIENT_PASSWORD: ${{ needs.create-env.outputs.API_CLIENT_PASSWORD }} + API_UAA_ENDPOINT: ${{ needs.create-env.outputs.API_UAA_ENDPOINT }} API_USER: ${{ secrets.API_USER }} CLIENT_ID: ${{ secrets.CLIENT_ID }} CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }} @@ -215,58 +292,89 @@ jobs: SPLUNK_HOST: ${{ secrets.SPLUNK_HOST }} steps: - - name: Checkout uses: actions/checkout@v3 with: ref: ${{github.event.pull_request.head.sha}} repository: ${{github.event.pull_request.head.repo.full_name}} + - uses: actions/setup-go@v3 with: - go-version: '1.17.0' + go-version: '1.18.1' - run: go version + - uses: ruby/setup-ruby@v1 + with: + ruby-version: '3.3' + - run: ruby -v - uses: actions/download-artifact@v3 with: name: splunk-firehose-nozzle - # Install dependencies - name: Install dependencies run: | go mod vendor chmod +x splunk-firehose-nozzle - + # Deploy data-gen - name: Deploy data-gen run: | .github/pre-req.sh cf push -f scripts/data_gen_manifest.yml -u process -p tools/data_gen --random-route sleep 10 - + cf apps + env: + ENCRYPT_KEY: ${{ secrets.ENCRYPT_KEY }} # Nozzle Log - name: Nozzle Log run: | - cf logs splunk-firehose-nozzle & + cf logs splunk-firehose-nozzle & # Prepare test environment - name: Prepare test environment run: | .github/pre-functional-test.sh - # Executing tests - name: Executing tests - run: | - .github/functional-test.sh - + run: | + .github/functional-test.sh + # Teardown - name: Teardown if: always() run: | - echo "Teardown deployment env" - cf delete splunk-firehose-nozzle -f - cf delete data_gen -f - cf delete-org splunk-ci-org -f + echo "Teardown deployment env" + cf target -o "splunk-ci-org" -s "splunk-ci-space" + cf delete splunk-firehose-nozzle -f + cf delete data_gen -f + cf delete-org splunk-ci-org -f + + + teardown-env: + runs-on: ubuntu-latest + needs: + - create-env + - execute_tests + if: ${{ always() }} + env: + ENV_ID: ${{ needs.create-env.outputs.ENV_ID }} + steps: + - name: install-selfservice + run: | + wget https://github.com/cf-platform-eng/selfservice/releases/download/0.2.9/selfservice + chmod +x selfservice + ls -latr + + - name: auth + run: | + echo "$(./selfservice auth $API_TOKEN | cut -c 8-)" >> "$GITHUB_ENV" + env: + API_TOKEN: ${{ secrets.API_TOKEN }} + + - name: release environment + run: | + ./selfservice release $ENV_ID \ No newline at end of file From 0aa2e5efd3469d73dfe6f11ac26bba5778a30d20 Mon Sep 17 00:00:00 2001 From: ajasnosz Date: Tue, 25 Jun 2024 13:46:10 +0200 Subject: [PATCH 2/3] chore: rename variables --- .github/pre-req.sh | 8 ++++---- .github/workflows/main.yml | 28 ++++++++++++++++------------ 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/.github/pre-req.sh b/.github/pre-req.sh index a69ac4db..4692815e 100755 --- a/.github/pre-req.sh +++ b/.github/pre-req.sh @@ -10,8 +10,8 @@ sudo apt-get update sudo apt-get install apt-transport-https ca-certificates sudo apt-get install cf-cli #CF Login -API_PASSWORD_DEC=$(echo "$API_PASSWORD" | openssl aes-256-cbc -d -pbkdf2 -a -pass pass:"$ENCRYPT_KEY") -cf login --skip-ssl-validation -a "$API_ENDPOINT" -u "$API_USER" -p "$API_PASSWORD_DEC" +API_PASSWORD_DECRYPTED=$(echo "$API_PASSWORD" | openssl aes-256-cbc -d -pbkdf2 -a -pass pass:"$ENCRYPT_KEY") +cf login --skip-ssl-validation -a "$API_ENDPOINT" -u "$API_USER" -p "$API_PASSWORD_DECRYPTED" #Create splunk-ci org and space if [ "`cf o | grep "splunk-ci-org"`" == "splunk-ci-org" ]; then @@ -27,8 +27,8 @@ fi gem install cf-uaac uaac target "$API_UAA_ENDPOINT" --skip-ssl-validation -API_CLIENT_PASSWORD_DEC=$(echo "$API_CLIENT_PASSWORD" | openssl aes-256-cbc -d -pbkdf2 -a -pass pass:"$ENCRYPT_KEY") -uaac token client get "$API_USER" -s "$API_CLIENT_PASSWORD_DEC" +API_CLIENT_PASSWORD_DECRYPTED=$(echo "$API_CLIENT_PASSWORD" | openssl aes-256-cbc -d -pbkdf2 -a -pass pass:"$ENCRYPT_KEY") +uaac token client get "$API_USER" -s "$API_CLIENT_PASSWORD_DECRYPTED" if [ $(uaac client get "$CLIENT_ID" | grep -woc "$CLIENT_ID") -eq 0 ]; then uaac client add "$CLIENT_ID" --name splunk-firehose --secret "$CLIENT_SECRET" --authorized_grant_types client_credentials,refresh_token --authorities doppler.firehose,cloud_controller.admin_read_only diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5a1a4499..912a9931 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -7,6 +7,10 @@ on: workflow_dispatch: +env: + GO_VERSION: '1.18.1' + RUBY_VERSION: '3.3' + jobs: workflow_approval: name: Approve workflow @@ -91,7 +95,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: '1.18.1' + go-version: ${{ env.GO_VERSION }} - name: Set up Brew uses: Homebrew/actions/setup-homebrew@master @@ -112,11 +116,11 @@ jobs: echo API_ENDPOINT="https://api.$(jq -r '.sys_domain' ./env.json)" >> "$GITHUB_OUTPUT" echo API_UAA_ENDPOINT="https://uaa.$(jq -r '.sys_domain' ./env.json)" >> "$GITHUB_OUTPUT" API_PASSWORD="$(hammer -t ./env.json om credentials -- -p cf -t json -c .uaa.admin_credentials | jq -r '.password')" - API_PASSWORD_ENC="$(echo $API_PASSWORD | openssl aes-256-cbc -a -pbkdf2 -salt -pass pass:$ENCRYPT_KEY)" - echo API_PASSWORD=$API_PASSWORD_ENC >> "$GITHUB_OUTPUT" + API_PASSWORD_ENCRYPTED="$(echo $API_PASSWORD | openssl aes-256-cbc -a -pbkdf2 -salt -pass pass:$ENCRYPT_KEY)" + echo API_PASSWORD=$API_PASSWORD_ENCRYPTED >> "$GITHUB_OUTPUT" API_CLIENT_PASSWORD="$(hammer -t ./env.json om credentials -- -p cf -t json -c .uaa.admin_client_credentials | jq -r '.password')" - API_CLIENT_PASSWORD_ENC="$(echo $API_CLIENT_PASSWORD | openssl aes-256-cbc -a -pbkdf2 -salt -pass pass:$ENCRYPT_KEY)" - echo API_CLIENT_PASSWORD=$API_CLIENT_PASSWORD_ENC >> "$GITHUB_OUTPUT" + API_CLIENT_PASSWORD_ENCRYPTED="$(echo $API_CLIENT_PASSWORD | openssl aes-256-cbc -a -pbkdf2 -salt -pass pass:$ENCRYPT_KEY)" + echo API_CLIENT_PASSWORD=$API_CLIENT_PASSWORD_ENCRYPTED >> "$GITHUB_OUTPUT" echo ENV_ID=$ENV_ID >> "$GITHUB_OUTPUT" env: ENCRYPT_KEY: ${{ secrets.ENCRYPT_KEY }} @@ -135,7 +139,7 @@ jobs: repository: ${{github.event.pull_request.head.repo.full_name}} - uses: actions/setup-go@v3 with: - go-version: '1.18.1' + go-version: ${{ env.GO_VERSION }} - run: go version # Install Dependencies @@ -186,12 +190,12 @@ jobs: - uses: actions/setup-go@v3 with: - go-version: '1.18.1' + go-version: ${{ env.GO_VERSION }} - run: go version - uses: ruby/setup-ruby@v1 with: - ruby-version: '3.3' + ruby-version: ${{ env.RUBY_VERSION }} - run: ruby -v - uses: actions/download-artifact@v3 @@ -246,7 +250,7 @@ jobs: - uses: actions/setup-go@v3 with: - go-version: '1.18.1' + go-version: ${{ env.GO_VERSION }} - run: go version - uses: actions/download-artifact@v3 @@ -300,12 +304,12 @@ jobs: - uses: actions/setup-go@v3 with: - go-version: '1.18.1' + go-version: ${{ env.GO_VERSION }} - run: go version - uses: ruby/setup-ruby@v1 with: - ruby-version: '3.3' + ruby-version: ${{ env.RUBY_VERSION }} - run: ruby -v - uses: actions/download-artifact@v3 @@ -359,7 +363,7 @@ jobs: needs: - create-env - execute_tests - if: ${{ always() }} + if: always() env: ENV_ID: ${{ needs.create-env.outputs.ENV_ID }} steps: From f915c281592c670365ab86b605b0f36962a96c0c Mon Sep 17 00:00:00 2001 From: ajasnosz Date: Fri, 5 Jul 2024 11:17:53 +0200 Subject: [PATCH 3/3] chore: fix teardown --- .github/workflows/main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 912a9931..6fe753af 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -351,11 +351,15 @@ jobs: - name: Teardown if: always() run: | + API_PASSWORD_DEC=$(echo "$API_PASSWORD" | openssl aes-256-cbc -d -pbkdf2 -a -pass pass:"$ENCRYPT_KEY") + cf login --skip-ssl-validation -a "$API_ENDPOINT" -u "$API_USER" -p "$API_PASSWORD_DEC" echo "Teardown deployment env" cf target -o "splunk-ci-org" -s "splunk-ci-space" cf delete splunk-firehose-nozzle -f cf delete data_gen -f cf delete-org splunk-ci-org -f + env: + ENCRYPT_KEY: ${{ secrets.ENCRYPT_KEY }} teardown-env: