openmdao_latest_workflow.yml

# Run OpenMDAO tests on latest/pre-release versions
name: OpenMDAO Latest

on:

  # Run the workflow every Monday and Thursday at 0400 UTC
  schedule:
    - cron: '0 4 * * 1,4'

  # Allow running the workflow manually from the Actions tab
  workflow_dispatch:

   inputs:
      run_name:
        description: 'Name of workflow run as it will appear under Actions tab (optional):'
        type: string
        required: false
        default: ''
      job_name:
        description: 'Select a job from the matrix to run (default: all jobs)'
        type: choice
        options:
          - ''
          - 'Ubuntu Latest, NumPy<2'
          - 'Ubuntu Latest, NumPy 2 (No PETSc/pyOptSparse)'
          - 'MacOS Latest, NumPy<2'
          - 'MacOS Latest, NumPy 2 (No PETSc/pyOptSparse)'
        required: false
        default: ''
      debug_enabled:
        description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
        type: boolean
        required: false
        default: false

run-name:  ${{ inputs.run_name }}

permissions: {}

jobs:

  tests:

    timeout-minutes: 120

    strategy:
      fail-fast: false
      matrix:
        include:
          # test latest versions on ubuntu
          # Include pyOptSparse, which requires NumPy<2
          - NAME: Ubuntu Latest, NumPy<2
            OS: ubuntu-latest
            PY: '3.12'
            NUMPY: '<2'
            PETSc: true
            PYOPTSPARSE: true
            SNOPT: true
            BANDIT: true
            BUILD_DOCS: true
            PUBLISH_DOCS: true

          # test latest versions on ubuntu
          # Exclude pyOptSparse, which requires NumPy<2
          # NumPy>=2.1 is required with Python 3.13
          - NAME: Ubuntu Latest, NumPy 2 (No pyOptSparse)
            OS: ubuntu-latest
            PY: '3.13'
            NUMPY: '>=2.1'
            PETSc: true

          # test latest versions on macos
          # Include pyOptSparse, which requires NumPy<2
          - NAME: MacOS Latest, NumPy<2
            OS: macos-13
            PY: '3.12'
            NUMPY: '<2'
            PETSc: true
            PYOPTSPARSE: true

          # test latest versions on macos
          # Exclude pyOptSparse, which require NumPy<2
          # NumPy>=2.1 is required with Python 3.13
          - NAME: MacOS Latest, NumPy 2 (No pyOptSparse)
            OS: macos-13
            PY: '3.13'
            NUMPY: '>=2.1'
            PETSc: true

    runs-on: ${{ matrix.OS }}

    name: ${{ matrix.NAME }}

    defaults:
      run:
        shell: bash -l {0}

    steps:
      - name: Display run details
        run: |
          echo "============================================================="
          echo "Run #${GITHUB_RUN_NUMBER}"
          echo "Run ID: ${GITHUB_RUN_ID}"
          echo "Testing: ${GITHUB_REPOSITORY}"
          echo "Branch: ${GITHUB_BASE_REF}"
          echo "Triggered by: ${GITHUB_EVENT_NAME}"
          echo "Initiated by: ${GITHUB_ACTOR}"
          echo "============================================================="

      - name: Exit if this job was not selected
        if: github.event_name == 'workflow_dispatch' &&  inputs.job_name != '' &&  inputs.job_name != matrix.NAME
        uses: actions/github-script@v7
        with:
          script: core.setFailed('The ${{ matrix.NAME }} job was not included in the run, exiting...');

      - name: Create SSH key
        if: (matrix.SNOPT || matrix.BUILD_DOCS)
        env:
          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
          SSH_KNOWN_HOSTS: ${{ secrets.SSH_KNOWN_HOSTS }}
        run: |
          mkdir -p ~/.ssh/
          echo "$SSH_PRIVATE_KEY" > ~/.ssh/id_rsa
          sudo chmod 600 ~/.ssh/id_rsa
          echo "$SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts

      - name: Checkout code
        uses: actions/checkout@v4

      - name: Setup conda
        uses: conda-incubator/setup-miniconda@v3
        with:
          python-version: ${{ matrix.PY }}
          miniforge-version: latest

      - name: Install MacOS-specific dependencies
        if: matrix.OS == 'macos-13'
        run: |
          conda install compilers swig

      - name: Install NumPy/SciPy
        run: |
          echo "============================================================="
          echo "Install latest versions of NumPy/SciPy"
          echo "============================================================="
          python -m pip install --upgrade --pre "numpy${{ matrix.NUMPY }}"
          python -m pip install --upgrade --pre scipy

          # remember versions so we can check them later
          NUMPY_VER=`python -c "import numpy; print(numpy.__version__)"`
          SCIPY_VER=`python -c "import scipy; print(scipy.__version__)"`
          echo "NUMPY_VER=$NUMPY_VER" >> $GITHUB_ENV
          echo "SCIPY_VER=$SCIPY_VER" >> $GITHUB_ENV

      - name: Install PETSc
        if: ${{ matrix.PETSC }}
        run: |
          echo "============================================================="
          echo "Install latest PETSc"
          echo "============================================================="
          if [[ "${{ matrix.NUMPY }}" == "<2" ]]; then
            conda install mpich mpi4py petsc4py=3.21.2 -q -y
          else
            conda install mpich mpi4py petsc4py -q -y
          fi

          echo "============================================================="
          echo "Check MPI and PETSc installation"
          echo "============================================================="
          export OMPI_MCA_rmaps_base_oversubscribe=1
          export OMPI_MCA_btl=^openib

          echo "OMPI_MCA_rmaps_base_oversubscribe=1" >> $GITHUB_ENV
          echo "OMPI_MCA_btl=^openib" >> $GITHUB_ENV

          echo "-----------------------"
          echo "Quick test of mpi4py:"
          mpirun -n 3 python -c "from mpi4py import MPI; print(f'Rank: {MPI.COMM_WORLD.rank}')"
          echo "-----------------------"
          echo "Quick test of petsc4py:"
          mpirun -n 3 python -c "import numpy; from mpi4py import MPI; comm = MPI.COMM_WORLD; \
                              import petsc4py; petsc4py.init(); \
                              x = petsc4py.PETSc.Vec().createWithArray(numpy.ones(5)*comm.rank, comm=comm);  \
                              print(x.getArray())"
          echo "-----------------------"

      - name: Install latest versions of [all] openmdao dependencies
        run: |
          source .github/scripts/install_latest_deps.sh

      - name: Install pyOptSparse
        if: ${{ matrix.PYOPTSPARSE }}
        run: |
          echo "============================================================="
          echo "Define useful functions"
          echo "============================================================="
          function latest_version() {
            local REPO_URL=$1/releases/latest
            local LATEST_URL=`curl -fsSLI -o /dev/null -w %{url_effective} $REPO_URL`
            local LATEST_VER=`echo $LATEST_URL | awk '{split($0,a,"/tag/"); print a[2]}'`
            echo $LATEST_VER
          }
          function latest_branch() {
            local LATEST_VER=$(latest_version $1)
            echo git+$1@$LATEST_VER
          }

          echo "============================================================="
          echo "Install latest pyoptsparse"
          echo "============================================================="
          python -m pip install git+https://github.com/OpenMDAO/build_pyoptsparse
          BRANCH="-b $(latest_version https://github.com/mdolab/pyoptsparse)"
          if [[ "${{ matrix.SNOPT }}" ]]; then
            if [[ "${{ secrets.SNOPT_LOCATION_77 }}" ]]; then
              echo "  > Secure copying SNOPT 7.7 over SSH"
              mkdir SNOPT
              scp -qr ${{ secrets.SNOPT_LOCATION_77 }} SNOPT
              SNOPT="-s SNOPT/src"
            else
              echo "SNOPT source is not available"
            fi
          fi
          build_pyoptsparse $BRANCH $SNOPT

      - name: Install OpenMDAO
        run: |
          echo "============================================================="
          echo "Install OpenMDAO"
          echo "============================================================="
          if [[ "${{ matrix.NUMPY }}" == "" ]]; then
            sed -i.bak 's/numpy<2/numpy/g' pyproject.toml
            grep numpy pyproject.toml
          fi
          python -m pip install .

      - name: Display environment info
        id: env_info
        if: always()
        continue-on-error: true
        run: |
          conda info
          conda list

          echo "============================================================="
          echo "Check installed versions of Python, Numpy and Scipy"
          echo "============================================================="
          echo 'errors<<EOF' >> $GITHUB_OUTPUT

          FINAL_VER=`python -c "import platform; print(platform.python_version())"`
          if [[ ! "$FINAL_VER" == "${{ matrix.PY }}"* ]]; then
            echo "Python version was changed from ${{ matrix.PY }} to $FINAL_VER" >> $GITHUB_OUTPUT
          fi

          FINAL_VER=`python -c "import numpy; print(numpy.__version__)"`
          if [[ ! "$FINAL_VER" == "$NUMPY_VER"* ]]; then
            echo "NumPy version was changed from $NUMPY_VER to $FINAL_VER" >> $GITHUB_OUTPUT
          fi

          FINAL_VER=`python -c "import scipy; print(scipy.__version__)"`
          if [[ ! "$FINAL_VER" == "$SCIPY_VER"* ]]; then
            echo "SciPy version was changed from $SCIPY_VER to $FINAL_VER" >> $GITHUB_OUTPUT
          fi

          echo 'EOF' >> $GITHUB_OUTPUT

          grep changed $GITHUB_OUTPUT || echo ""

      # Enable tmate debugging of manually-triggered workflows if the input option was provided
      #
      # To access the terminal through the web-interface:
      #    1. Click on the web-browser link printed out in this action from the github
      #       workflow terminal
      #    2. Press cntrl + c in the new tab that opens up to reveal the terminal
      #    3. To activate the conda environment run:
      #        $ source $CONDA/etc/profile.d/conda.sh
      #        $ conda activate test
      - name: Setup tmate session
        uses: mxschmitt/action-tmate@v3
        if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
        with:
          limit-access-to-actor: true

      - name: Run tests
        id: run_tests
        continue-on-error: true
        run: |
          echo "============================================================="
          echo "Run tests (from directory other than repo root)"
          echo "============================================================="
          cd $HOME
          RPT_FILE=`pwd`/deprecations.txt
          echo "RPT_FILE=$RPT_FILE" >> $GITHUB_ENV
          testflo -n 1 openmdao --timeout=240 --deprecations_report=$RPT_FILE --exclude=test_warnings_filters

      - name: Build docs
        id: build_docs
        if: matrix.BUILD_DOCS
        continue-on-error: true
        run: |
          export OPENMDAO_REPORTS=0
          export PYDEVD_DISABLE_FILE_VALIDATION=1

          cd openmdao/docs
          if [[ "${{ secrets.SNOPT_LOCATION_77 }}" ]]; then
            echo "============================================================="
            echo "Building docs with SNOPT examples."
            echo "============================================================="
          else
            echo "============================================================="
            echo "Disabling SNOPT cells in notebooks."
            echo "============================================================="
            python openmdao_book/other/disable_snopt_cells.py
          fi

          echo "============================================================="
          echo "Starting ipcluster to run MPI under notebooks"
          echo "============================================================="
          ./ipcluster_start.sh
          sleep 12

          echo "============================================================="
          echo "Building the docs"
          echo "============================================================="
          ./build_all_docs.sh

      - name: Display doc build reports
        if: steps.build_docs.outcome == 'failure'
        continue-on-error: true
        run: |
          for f in $(find /home/runner/work/OpenMDAO/OpenMDAO/openmdao/docs/openmdao_book/_build/html/reports -name '*.log'); do
            echo "============================================================="
            echo $f
            echo "============================================================="
            cat $f
          done

      - name: Publish docs
        if: matrix.PUBLISH_DOCS
        env:
          DOCS_LOCATION: ${{ secrets.DOCS_LOCATION }}
        run: |
          if [[ "${#DOCS_LOCATION}" ]]; then
            echo "============================================================="
            echo "Install version of openssl compatible with hosting service"
            echo "============================================================="
            conda install -c conda-forge 'openssl=3.0'

            echo "============================================================="
            echo "Fetch tags to get docs version"
            echo "============================================================="
            git fetch --prune --unshallow --tags

            echo "============================================================="
            echo "Publish docs"
            echo "============================================================="
            cd openmdao/docs
            python upload_doc_version.py openmdao_book/_build/html/ ${{ secrets.DOCS_LOCATION }}
          else
            echo "Docs destination not available."
          fi

      - name: Deprecations Report
        id: deprecations_report
        continue-on-error: true
        run: |
          echo "============================================================="
          echo "Display deprecations report"
          echo "============================================================="
          cat $RPT_FILE

          echo 'summary<<EOF' >> $GITHUB_OUTPUT
          head -n 6 $RPT_FILE | cut -d':' -f 1 >> $GITHUB_OUTPUT
          echo 'EOF' >> $GITHUB_OUTPUT

          grep '^0 unique deprecation warnings' $RPT_FILE

      - name: Scan for security issues
        if: matrix.BANDIT
        id: bandit
        continue-on-error: true
        run: |
          python -m pip install bandit
          echo "============================================================="
          echo "Run bandit scan for high/medium severity issues"
          echo "============================================================="
          cd ${{ github.workspace }}
          python -m bandit -c bandit.yml -ll -r openmdao

      - name: Slack security issue
        if: steps.bandit.outcome == 'failure'
        uses: act10ns/slack@v2.0.0
        with:
          webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }}
          status: 'warning'
          message:
            Security issue found on `${{ matrix.NAME }}` build.
            ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

      - name: Check NumPy 2.0 Compatibility
        run: |
          echo "============================================================="
          echo "Check OpenMDAO code for NumPy 2.0 compatibility"
          echo "See: https://numpy.org/devdocs/numpy_2_0_migration_guide.html"
          echo "============================================================="
          python -m pip install ruff
          cd ${{ github.workspace }}
          ruff check . --select NPY201

      - name: Slack env change
        if: steps.env_info.outputs.errors != ''
        uses: act10ns/slack@v2.0.0
        with:
          webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }}
          status: 'warning'
          message: |
            Environment change detected on `${{ matrix.NAME }}` build.
            Python, NumPy or SciPy was not the requested version:
            ```${{steps.env_info.outputs.errors}}```
            ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

      - name: Slack unit test failure
        if: steps.run_tests.outcome == 'failure'
        uses: act10ns/slack@v2.0.0
        with:
          webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }}
          status: ${{ steps.run_tests.outcome }}
          message:
            Unit testing failed on `${{ matrix.NAME }}` build.
            ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

      - name: Slack doc build failure
        if: steps.build_docs.outcome == 'failure'
        uses: act10ns/slack@v2.0.0
        with:
          webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }}
          status: ${{ steps.build_docs.outcome }}
          message: |
            Doc build failed on `${{ matrix.NAME }}` build.
            ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

      - name: Slack deprecation warnings
        if: steps.deprecations_report.outcome == 'failure'
        uses: act10ns/slack@v2.0.0
        with:
          webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }}
          status: 'warning'
          message: |
            Deprecations were detected on `${{ matrix.NAME }}` build.
            ```${{ steps.deprecations_report.outputs.summary }}```
            ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

      - name: Fail the workflow if tests or doc build failed
        if: steps.run_tests.outcome == 'failure' || steps.build_docs.outcome == 'failure'
        uses: actions/github-script@v3
        with:
          script: |
              let test_fail = ${{ steps.run_tests.outcome == 'failure' }};
              let docs_fail = ${{ steps.build_docs.outcome == 'failure' }};
              if (test_fail && docs_fail) {
                  core.setFailed('Tests and doc build failed.');
              }
              else if (test_fail) {
                  core.setFailed('Tests failed.');
              }
              else if (docs_fail) {
                  core.setFailed('Doc build failed.');
              }