python-versions.yml

name: Python Versions

on:
  push:
    branches:
      - "develop"
      - "feature/**"
  pull_request:
    types: [opened, synchronize, reopened]
  # Allows workflow to be called from other workflows
  workflow_call:
    inputs:
      ref:
        required: true
        type: string
      force-canary:
        description: |
          Forces the current build to be canary.
          Canary builds test all Python versions and do not use constraints.
        default: false
        type: boolean
      constraints-branch:
        description: "The name of the branch from which the constraints files will be downloaded or compared with."
        default: "constraints-develop"
        type: string
    secrets:
      PARAMETER_PASSWORD:
        description: "Token passed from caller workflows for snowflake integration tests"
        required: true

# Avoid duplicate workflows on same branch
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-python
  cancel-in-progress: true

defaults:
  run:
    shell: bash --login -eo pipefail {0}

env:
  FORCE_COLOR: "1"

jobs:
  build_info:
    runs-on: ubuntu-latest

    name: "Build info"

    steps:
      - name: Checkout Streamlit code
        uses: actions/checkout@v3
        with:
          ref: ${{ inputs.ref }}
          persist-credentials: false
          submodules: "recursive"
          fetch-depth: 2
      - name: Set Python version vars
        id: build_info
        uses: ./.github/actions/build_info
        with:
          force-canary: ${{ inputs.force-canary || false }}

    outputs:
      PYTHON_VERSIONS: ${{ steps.build_info.outputs.PYTHON_VERSIONS }}
      PYTHON_MIN_VERSION: ${{ steps.build_info.outputs.PYTHON_MIN_VERSION }}
      PYTHON_MAX_VERSION: ${{ steps.build_info.outputs.PYTHON_MAX_VERSION }}
      USE_CONSTRAINTS_FILE: ${{ steps.build_info.outputs.USE_CONSTRAINTS_FILE }}

  py_version:
    needs:
      - build_info

    runs-on: ubuntu-latest

    strategy:
      fail-fast: false
      matrix:
        python_version: "${{ fromJson(needs.build_info.outputs.PYTHON_VERSIONS) }}"

    # TODO: Should we add developer-friendly name for this job also?
    #   This will unfortunately require a branch protection update.

    env:
      PYTHON_VERSION: >-
        ${{
          (
            matrix.python_version == 'min' && needs.build_info.outputs.PYTHON_MIN_VERSION ||
            (matrix.python_version == 'max' && needs.build_info.outputs.PYTHON_MAX_VERSION || matrix.python_version)
          )
        }}
      USE_CONSTRAINTS_FILE: "${{ fromJson(needs.build_info.outputs.USE_CONSTRAINTS_FILE )}}"
      CONSTRAINTS_BRANCH: ${{ inputs.constraints-branch || 'constraints-develop' }}

    steps:
      - name: Checkout Streamlit code
        uses: actions/checkout@v3
        with:
          ref: ${{ inputs.ref }}
          persist-credentials: false
          submodules: "recursive"

      - name: Set up Python ${{ env.PYTHON_VERSION }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ env.PYTHON_VERSION }}
      - name: Setup virtual env
        uses: ./.github/actions/make_init
      - name: Run make develop
        run: make develop
      - name: Run Linters
        run: |
          PRE_COMMIT_NO_CONCURRENCY=true pre-commit run --show-diff-on-failure --color=always --all-files
      - name: Run Type Checkers
        run: scripts/mypy --report
      - name: Run Python Tests
        run: make pytest
      - name: Run Integration Tests
        run: make integration-tests
      - name: CLI Smoke Tests
        run: make cli-smoke-tests
      - name: Set CONSTRAINTS_FILE env variable
        if: ${{ always() }}
        run: |
          mkdir -p /tmp/constraints
          echo "CONSTRAINTS_FILE=/tmp/constraints/constraints-${PYTHON_VERSION}.txt" >> $GITHUB_ENV
      - name: Generate constraint file for Python ${{ env.PYTHON_VERSION }}
        if: ${{ always() }}
        run: |
          pip freeze | grep -v "\-e git" | tee "${CONSTRAINTS_FILE}"
      - name: Diff constraint file
        if: ${{ always() }}
        run: |
          CONSTRAINT_URL="https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/${CONSTRAINTS_BRANCH}/constraints-${PYTHON_VERSION}.txt"
          diff -y <(echo "Old"; curl -s "${CONSTRAINT_URL}") <(echo "New"; cat "${CONSTRAINTS_FILE}") || true
      - name: Upload constraints file
        uses: actions/upload-artifact@v3
        with:
          name: constraints
          path: ${{ env.CONSTRAINTS_FILE }}
          if-no-files-found: error

  py_constraint:
    needs:
      - py_version
    permissions:
      # Additional permission needed to upload constraints
      contents: write

    runs-on: ubuntu-latest
    if: |
      github.repository == 'streamlit/streamlit' && (
      (github.event_name == 'push' && github.ref_name == 'develop') ||
      (github.event_name == 'schedule')
      )

    name: Upload constraints

    env:
      TARGET_BRANCH: constraints-${{ github.ref_name }}

    steps:
      - name: Checkout branch "${{ env.TARGET_BRANCH }}"
        uses: actions/checkout@v3
        with:
          ref: ${{ env.TARGET_BRANCH }}
          # Save the access token to the local git config, so
          # later git commands can work.
          persist-credentials: true

      - uses: actions/download-artifact@v3
        with:
          name: constraints
          path: .

      - name: Commit and push constraint files
        run: |
          git add .
          git config --local user.email "core+streamlitbot-github@streamlit.io"
          git config --local user.name "Automated GitHub Actions commit"
          if ! git diff --cached --color --exit-code --ignore-matching-lines="^#.*"
          then
            git commit --all --message "Updating constraints. Github run id:${GITHUB_RUN_ID}

            This update in constraints is automatically committed by the CI based on
            '${GITHUB_REF}' in the '${GITHUB_REPOSITORY}' repository with commit sha ${GITHUB_SHA}.

            The action that build those constraints can be found at https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/
            "
            git push "origin" "HEAD:${TARGET_BRANCH}";
          else
            echo "No changes"
          fi
        env:
          TARGET_BRANCH: constraints-${{ github.ref_name }}

  py_snowflake:
    runs-on: ubuntu-latest

    # Runs triggered by PRs from forks or by dependabot won't run this job, since that PR wouldn't have secrets access
    # See: https://docs.github.com/en/code-security/dependabot/working-with-dependabot/automating-dependabot-with-github-actions
    # Runs triggered by Release/RC are workflow_dispatch events ; Nightly is a schedule event
    if: |
      github.repository == 'streamlit/streamlit' && (
      (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]') ||
      (github.event_name == 'push') ||
      (github.event_name == 'workflow_dispatch') ||
      (github.event_name == 'schedule')
      )

    name: >
      Python 3.8: Python tests for Snowflake

    steps:
      - name: Checkout Streamlit code
        uses: actions/checkout@v3
        with:
          ref: ${{ inputs.ref }}
          persist-credentials: false
          submodules: "recursive"
      - name: Set up Python 3.8
        uses: actions/setup-python@v4
        with:
          python-version: "3.8"
      - name: Decrypt credentials
        run: ./.github/scripts/decrypt_credentials.sh
        env:
          PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }}
      - name: Setup virtual env
        uses: ./.github/actions/make_init
      - name: Run make develop
        run: make develop
      - name: Run Type Checkers
        run: scripts/mypy --report
      - name: Run Python Tests for Snowflake
        run: make pytest-snowflake