NNF Integration-Test #626
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: NNF Integration-Test | |
| on: | |
| schedule: | |
| # Time is in UTC - start at 10:00pm CST or 11:00pm CDT | |
| - cron: "00 04 * * *" | |
| workflow_dispatch: | |
| # These inputs are only used when running manually | |
| inputs: | |
| system: | |
| description: Which system to run on | |
| type: choice | |
| required: true | |
| default: "htx-1" | |
| options: | |
| - htx-1 | |
| procs: | |
| description: Number of parallel workflows to run | |
| required: true | |
| default: 1 | |
| type: number | |
| testTarget: | |
| description: Which int-test target to run | |
| type: choice | |
| required: true | |
| default: "simple" | |
| options: | |
| - sanity | |
| - simple | |
| - dm | |
| - container | |
| - full | |
| rounds: | |
| description: How many times to run the test target | |
| type: number | |
| required: true | |
| default: 1 | |
| hTimeout: | |
| description: High Timeout duration | |
| type: string | |
| required: true | |
| default: "5m" | |
| lTimeout: | |
| description: Low Timeout duration | |
| type: string | |
| required: true | |
| default: "2m" | |
| runSystemTest: | |
| description: Run system-test | |
| type: boolean | |
| required: true | |
| default: false | |
| runDMSystemTest: | |
| description: Run dm-system-test | |
| type: boolean | |
| required: true | |
| default: false | |
| notifySlack: | |
| description: Report start/results to slack | |
| type: boolean | |
| required: true | |
| default: false | |
| env: | |
| # When running as a cron job, these values will be used since there will be no inputs. These | |
| # fields match the inputs above. | |
| TEST_TARGET: full | |
| PROCS: 1 | |
| HTIMEOUT: 5m | |
| LTIMEOUT: 2m | |
| SYSTEM: htx-1 | |
| ROUNDS: 1 | |
| NOTIFY_SLACK: "true" | |
| # These env vars are used for either type of run | |
| SLACK_CHANNEL_ID: C043U7PRGGM # rabbit-notifications | |
| # SLACK_CHANNEL_ID: C06SMGMM3PY # rabbit-notifications-sandbox | |
| RUN_URL: "<https://github.com/NearNodeFlash/nnf-integration-test/actions/runs/${{ github.run_id }}|#${{ github.run_number }}>" | |
| jobs: | |
| int-test: | |
| runs-on: ${{ inputs.system || 'htx-1' }} | |
| outputs: | |
| test-result: ${{ steps.int-test.outcome }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Version | |
| run: ./git-version-gen | |
| # If this is a manual run (workflow_dispatch trigger), then override the env vars with the input values. | |
| - name: Override TEST_TARGET with manual input | |
| if: github.event_name == 'workflow_dispatch' | |
| run: | | |
| echo "TEST_TARGET=${{ inputs.testTarget }}" >> $GITHUB_ENV | |
| - name: Override PROCS with manual input | |
| if: github.event_name == 'workflow_dispatch' | |
| run: | | |
| echo "PROCS=${{ inputs.procs }}" >> $GITHUB_ENV | |
| - name: Override ROUNDS with manual input | |
| if: github.event_name == 'workflow_dispatch' | |
| run: | | |
| echo "ROUNDS=${{ inputs.rounds }}" >> $GITHUB_ENV | |
| - name: Override LTIMEOUT with manual input | |
| if: github.event_name == 'workflow_dispatch' | |
| run: | | |
| echo "LTIMEOUT=${{ inputs.lTimeout }}" >> $GITHUB_ENV | |
| - name: Override HTIMEOUT with manual input | |
| if: github.event_name == 'workflow_dispatch' | |
| run: | | |
| echo "HTIMEOUT=${{ inputs.hTimeout }}" >> $GITHUB_ENV | |
| - name: Override NOTIFY_SLACK with manual input | |
| if: github.event_name == 'workflow_dispatch' | |
| run: | | |
| echo "NOTIFY_SLACK=${{ inputs.notifySlack }}" >> $GITHUB_ENV | |
| # Optionally send a slack message | |
| - name: Start Slack Message | |
| if: ${{ env.NOTIFY_SLACK == 'true' }} | |
| uses: archive/github-actions-slack@master | |
| id: start-message | |
| with: | |
| slack-function: send-message | |
| slack-bot-user-oauth-access-token: ${{ secrets.SLACK_BOT_USER_OAUTH_ACCESS_TOKEN }} | |
| slack-channel: ${{ env.SLACK_CHANNEL_ID }} | |
| slack-optional-blocks: >- | |
| [ | |
| { | |
| "type": "header", | |
| "text": { | |
| "type": "plain_text", | |
| "text": "NNF Integration Test", | |
| "emoji": true | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "Run ${{ env.RUN_URL }}\n_triggered via ${{ github.event_name }}_" | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "fields": [ | |
| { | |
| "type": "mrkdwn", | |
| "text": "*System*\n${{ env.SYSTEM }}" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Test Type*\n${{ env.TEST_TARGET }}" | |
| } | |
| ] | |
| }, | |
| { | |
| "type": "section", | |
| "fields": [ | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Num Procs*\n${{ env.PROCS }}" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Rounds*\n${{ env.ROUNDS }}" | |
| } | |
| ] | |
| }, | |
| { | |
| "type": "section", | |
| "fields": [ | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Status*\n:hourglass: In Progress" | |
| } | |
| ] | |
| }, | |
| { | |
| "type": "divider" | |
| } | |
| ] | |
| # Setup and Run the Test | |
| - name: Setup Go | |
| uses: actions/setup-go@v3 | |
| with: | |
| go-version-file: "go.mod" | |
| cache: true | |
| - name: Install Ginkgo and Ginkgo CLI | |
| run: make init | |
| - name: Run int-test (P=${{ env.P }} ${{ env.TEST_TARGET }}) ${{ env.ROUNDS }} time(s) on ${{ env.SYSTEM }} | |
| id: int-test | |
| shell: bash | |
| env: | |
| P: ${{ env.PROCS }} | |
| HTIMEOUT: ${{ env.HTIMEOUT }} | |
| LTIMEOUT: ${{ env.LTIMEOUT }} | |
| run: | | |
| for i in {1..${{ env.ROUNDS }}}; do | |
| echo "" | |
| echo "--------------------" | |
| echo "Attempt start: $i" | |
| echo "--------------------" | |
| echo "" | |
| make ${{ env.TEST_TARGET }} | |
| echo "" | |
| echo "@@@@@@@@@@@@@@@@@@@@" | |
| echo "Attempt end: $i" | |
| echo "@@@@@@@@@@@@@@@@@@@@" | |
| echo "" | |
| done | |
| # Optionally update the slack message with a failure | |
| - name: Failure Slack Message | |
| if: ${{ env.NOTIFY_SLACK == 'true' && failure() }} | |
| uses: archive/github-actions-slack@master | |
| with: | |
| slack-function: update-message | |
| slack-bot-user-oauth-access-token: ${{ secrets.SLACK_BOT_USER_OAUTH_ACCESS_TOKEN }} | |
| slack-channel: ${{ env.SLACK_CHANNEL_ID }} | |
| slack-update-message-ts: ${{ fromJson(steps.start-message.outputs.slack-result).response.message.ts }} | |
| slack-optional-blocks: >- | |
| [ | |
| { | |
| "type": "header", | |
| "text": { | |
| "type": "plain_text", | |
| "text": "NNF Integration Test", | |
| "emoji": true | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "Run ${{ env.RUN_URL }}\n_triggered via ${{ github.event_name }}_" | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "fields": [ | |
| { | |
| "type": "mrkdwn", | |
| "text": "*System*\n${{ env.SYSTEM }}" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Test Type*\n${{ env.TEST_TARGET }}" | |
| } | |
| ] | |
| }, | |
| { | |
| "type": "section", | |
| "fields": [ | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Num Procs*\n${{ env.PROCS }}" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Rounds*\n${{ env.ROUNDS }}" | |
| } | |
| ] | |
| }, | |
| { | |
| "type": "section", | |
| "fields": [ | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Status*\n:red_circle: Test Failed" | |
| } | |
| ] | |
| }, | |
| { | |
| "type": "divider" | |
| } | |
| ] | |
| # Optionally update the slack message with success | |
| - name: Success Slack Message | |
| if: ${{ env.NOTIFY_SLACK == 'true' && success() }} | |
| uses: archive/github-actions-slack@master | |
| with: | |
| slack-function: update-message | |
| slack-bot-user-oauth-access-token: ${{ secrets.SLACK_BOT_USER_OAUTH_ACCESS_TOKEN }} | |
| slack-channel: ${{ env.SLACK_CHANNEL_ID }} | |
| slack-update-message-ts: ${{ fromJson(steps.start-message.outputs.slack-result).response.message.ts }} | |
| slack-optional-blocks: >- | |
| [ | |
| { | |
| "type": "header", | |
| "text": { | |
| "type": "plain_text", | |
| "text": "NNF Integration Test", | |
| "emoji": true | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "Run ${{ env.RUN_URL }}\n_triggered via ${{ github.event_name }}_" | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "fields": [ | |
| { | |
| "type": "mrkdwn", | |
| "text": "*System*\n${{ env.SYSTEM }}" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Test Type*\n${{ env.TEST_TARGET }}" | |
| } | |
| ] | |
| }, | |
| { | |
| "type": "section", | |
| "fields": [ | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Num Procs*\n${{ env.PROCS }}" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Rounds*\n${{ env.ROUNDS }}" | |
| } | |
| ] | |
| }, | |
| { | |
| "type": "section", | |
| "fields": [ | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Status*\n:green_circle: Test Passed" | |
| } | |
| ] | |
| }, | |
| { | |
| "type": "divider" | |
| } | |
| ] | |
| # Only run system test after int-test is successful. If int-test fails, it can leave around | |
| # artifacts. | |
| system-test: | |
| needs: int-test | |
| if: ${{ github.event_name == 'schedule' || inputs.runSystemTest == true }} | |
| runs-on: ${{ inputs.system || 'htx-1' }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Version | |
| run: ./git-version-gen | |
| - name: View Permissions | |
| run: id | |
| - name: Install Bats | |
| run: | | |
| cd system-test | |
| make init | |
| - name: Create env file | |
| run: | | |
| cd system-test | |
| touch env | |
| echo "GLOBAL_LUSTRE_ROOT?=/lus/global" >> env | |
| echo "DM_PROFILE?=no-xattr" >> env | |
| echo "N ?=4" >> env | |
| echo "J ?=1" >> env | |
| - name: Run system-test | |
| run: | | |
| cd system-test | |
| make test | |
| # Ensure dm-system-test runs after system-test. system-test is optional, so we need to use | |
| # always() to ensure this runs even when system-test does not. | |
| dm-system-test: | |
| needs: system-test | |
| if: ${{ always() && ( github.event_name == 'schedule' || inputs.runDMSystemTest == true ) }} | |
| runs-on: ${{ inputs.system || 'htx-1' }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Version | |
| run: ./git-version-gen | |
| - name: View Permissions | |
| run: id | |
| - name: Install Bats | |
| run: | | |
| cd system-test | |
| make init | |
| - name: Create env file | |
| run: | | |
| cd system-test | |
| touch env | |
| echo "GLOBAL_LUSTRE_ROOT?=/lus/global" >> env | |
| echo "DM_PROFILE?=no-xattr" >> env | |
| echo "N ?=2" >> env | |
| echo "J ?=2" >> env | |
| - name: Run dm-system-test | |
| run: | | |
| cd system-test | |
| make dm |