[WIP] Add duration scripts and logic from conda repo

.github/workflows/durations.yml

@@ -0,0 +1,63 @@
+name: Update Durations
+
+on:
+ # every Sunday at 00:00 UTC
+ # https://crontab.guru/#0_0_*_*_0
+ schedule:
+ - cron: '0 0 * * 0'
+
+ # https://docs.github.com/en/webhooks-and-events/webhooks/webhook-events-and-payloads#workflow_dispatch
+ workflow_dispatch:
+
+jobs:
+ update-durations:
+ runs-on: ubuntu-latest
+ permissions:
+ # necessary to open PR
+ # https://github.com/peter-evans/create-pull-request#action-inputs
+ contents: write
+ pull-requests: write
+ steps:
+ - uses: actions/checkout@v3
+
+ - name: download recent artifacts
+ run: |
+ gh run list \
+ --branch main \
+ --workflow tests \
+ --limit 10 \
+ --json databaseId \
+ --jq '.[].databaseId' \
+ | xargs \
+ -n 1 \
+ gh run download \
+ --dir ${{ runner.temp }}/artifacts/ \
+ --pattern '*-all' \
+ || true
+ env:
+ GITHUB_TOKEN: ${{ github.token }}
+
+ - uses: actions/setup-python@v4
+ with:
+ python-version: '3.11'
+
+ - name: combine recent durations from artifacts
+ run: python ./tools/durations/combine.py ${{ runner.temp }}/artifacts/
+
+ - name: create updated durations PR
+ uses: peter-evans/create-pull-request@v5
+ with:
+ push-to-fork: conda-bot/conda
+ token: ${{ secrets.DURATIONS_TOKEN }}
+ branch: update-durations
+ delete-branch: true
+ commit-message: Update test durations
+ author: Conda Bot <[email protected]>
+ committer: Conda Bot <[email protected]>
+ title: 🤖 Update test durations
+ body: |
+ Aggregate recent test durations for each test and update the durations file.
+
+ [durations.yml]: ${{ github.server_url }}/${{ github.repository }}/blob/main/.github/workflows/durations.yml
+
+ This PR was created automatically by the [`durations.yml`][durations.yml] workflow.

.github/workflows/tests.yml

@@ -77,28 +77,37 @@ jobs:
  fail-fast: false
  matrix:
  # test all lower versions (w/ stable conda) and upper version (w/ canary conda)
- python-version: ['3.9', '3.10', '3.11']
- conda-version: [release]
- test-type: [serial, parallel]
- include:
- # minimum Python/conda combo
- - python-version: '3.8'
+ python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
+ conda-version: [23.5.0, release, canary]
+ test-group: [1, 2, 3]
+ exclude:
+ # testing oldest supported conda with Python 3.8
+ - python-version: '3.9'
  conda-version: 23.5.0
- test-type: serial
- - python-version: '3.8'
+ - python-version: '3.10'
+ conda-version: 23.5.0
+ - python-version: '3.11'
  conda-version: 23.5.0
- test-type: parallel
- # maximum Python/conda combo
  - python-version: '3.12'
- conda-version: canary
- test-type: serial
+ conda-version: 23.5.0
+ # testing stable conda with Python 3.9-3.11
+ - python-version: '3.8'
+ conda-version: release
  - python-version: '3.12'
+ conda-version: release
+ # testing canary conda with Python 3.12
+ - python-version: '3.8'
+ conda-version: canary
+ - python-version: '3.9'
+ conda-version: canary
+ - python-version: '3.10'
+ conda-version: canary
+ - python-version: '3.11'
  conda-version: canary
- test-type: parallel
  env:
  CONDA_CHANNEL_LABEL: ${{ matrix.conda-version == 'canary' && 'conda-canary/label/dev::' || '' }}
  CONDA_VERSION: ${{ contains('canary|release', matrix.conda-version) && 'conda' || format('conda={0}', matrix.conda-version) }}
- PYTEST_MARKER: ${{ matrix.test-type == 'serial' && 'serial' || 'not serial' }}
+ PYTEST_SPLITS: 3
 
  steps:
  - name: Checkout Source
@@ -107,7 +116,7 @@ jobs:
  fetch-depth: 0
 
  - name: Hash + Timestamp
- run: echo "HASH=${{ runner.os }}-${{ runner.arch }}-Py${{ matrix.python-version }}-${{ matrix.conda-version }}-${{ matrix.test-type }}-$(date -u "+%Y%m")" >> $GITHUB_ENV
+ run: echo "HASH=${{ runner.os }}-${{ runner.arch }}-Py${{ matrix.python-version }}-${{ matrix.conda-version }}-${{ matrix.test-group }}-$(date -u "+%Y%m")" >> $GITHUB_ENV
 
  - name: Cache Conda
  uses: actions/cache@v4
@@ -143,8 +152,10 @@ jobs:
  - name: Run Tests
  run: pytest
  --cov=conda_build
+ --durations-path=tools/durations/${{ runner.os }}.json
+ --group=${{ matrix.test-group }}
+ --splits=${{ env.PYTEST_SPLITS }}
  -n auto
- -m "${{ env.PYTEST_MARKER }}"
 
  - name: Upload Coverage
  uses: codecov/codecov-action@v4
@@ -158,6 +169,7 @@ jobs:
  name: test-results-${{ env.HASH }}
  path: |
  .coverage
+ tools/durations/${{ runner.os }}.json
  test-report.xml
  retention-days: 1 # temporary, combined in aggregate below
 
@@ -172,20 +184,18 @@ jobs:
  fail-fast: false
  matrix:
  # test lower version (w/ stable conda) and upper version (w/ canary conda)
- python-version: ['3.8']
- conda-version: [release]
- test-type: [serial, parallel]
- include:
- - python-version: '3.12'
+ python-version: ['3.8', '3.12']
+ conda-version: [release, canary]
+ test-group: [1, 2, 3]
+ exclude:
+ - python-version: '3.8'
  conda-version: canary
- test-type: serial
  - python-version: '3.12'
- conda-version: canary
- test-type: parallel
+ conda-version: release
  env:
  ErrorActionPreference: Stop # powershell exit on first error
  CONDA_CHANNEL_LABEL: ${{ matrix.conda-version == 'canary' && 'conda-canary/label/dev' || 'defaults' }}
- PYTEST_MARKER: ${{ matrix.test-type == 'serial' && 'serial' || 'not serial and not slow' }}
+ PYTEST_SPLITS: 3
 
  steps:
  - name: Checkout Source
@@ -195,7 +205,7 @@ jobs:
 
  - name: Hash + Timestamp
  shell: bash # use bash to run date command
- run: echo "HASH=${{ runner.os }}-${{ runner.arch }}-Py${{ matrix.python-version }}-${{ matrix.conda-version }}-${{ matrix.test-type }}-$(date -u "+%Y%m")" >> $GITHUB_ENV
+ run: echo "HASH=${{ runner.os }}-${{ runner.arch }}-Py${{ matrix.python-version }}-${{ matrix.conda-version }}-${{ matrix.test-group }}-$(date -u "+%Y%m")" >> $GITHUB_ENV
 
  - name: Cache Conda
  uses: actions/cache@v4
@@ -237,8 +247,10 @@ jobs:
  run: pytest
  --cov=conda_build
  --basetemp=${{ runner.temp }}
+ --durations-path=tools\durations\${{ runner.os }}.json
+ --group=${{ matrix.test-group }}
+ --splits=${{ env.PYTEST_SPLITS }}
  -n auto
- -m "${{ env.PYTEST_MARKER }}"
 
  - name: Upload Coverage
  uses: codecov/codecov-action@v4
@@ -252,6 +264,7 @@ jobs:
  name: test-results-${{ env.HASH }}
  path: |
  .coverage
+ tools\durations\${{ runner.os }}.json
  test-report.xml
  retention-days: 1 # temporary, combined in aggregate below
 
@@ -270,19 +283,17 @@ jobs:
  fail-fast: false
  matrix:
  # test lower version (w/ stable conda) and upper version (w/ canary conda)
- python-version: ['3.8']
- conda-version: [release]
- test-type: [serial, parallel]
- include:
- - python-version: '3.12'
+ python-version: ['3.8', '3.12']
+ conda-version: [release, canary]
+ test-group: [1, 2, 3]
+ exclude:
+ - python-version: '3.8'
  conda-version: canary
- test-type: serial
  - python-version: '3.12'
- conda-version: canary
- test-type: parallel
+ conda-version: release
  env:
  CONDA_CHANNEL_LABEL: ${{ matrix.conda-version == 'canary' && 'conda-canary/label/dev' || 'defaults' }}
- PYTEST_MARKER: ${{ matrix.test-type == 'serial' && 'serial' || 'not serial' }}
+ PYTEST_SPLITS: 3
 
  steps:
  - name: Checkout Source
@@ -291,7 +302,7 @@ jobs:
  fetch-depth: 0
 
  - name: Hash + Timestamp
- run: echo "HASH=${{ runner.os }}-${{ runner.arch }}-Py${{ matrix.python-version }}-${{ matrix.conda-version }}-${{ matrix.test-type }}-$(date -u "+%Y%m")" >> $GITHUB_ENV
+ run: echo "HASH=${{ runner.os }}-${{ runner.arch }}-Py${{ matrix.python-version }}-${{ matrix.conda-version }}-${{ matrix.test-group }}-$(date -u "+%Y%m")" >> $GITHUB_ENV
 
  - name: Cache Conda
  uses: actions/cache@v4
@@ -330,8 +341,10 @@ jobs:
  - name: Run Tests
  run: pytest
  --cov=conda_build
+ --durations-path=tools/durations/${{ runner.os }}.json
+ --group=${{ matrix.test-group }}
+ --splits=${{ env.PYTEST_SPLITS }}
  -n auto
- -m "${{ env.PYTEST_MARKER }}"
 
  - name: Upload Coverage
  uses: codecov/codecov-action@v4
@@ -345,6 +358,7 @@ jobs:
  name: test-results-${{ env.HASH }}
  path: |
  .coverage
+ tools/durations/${{ runner.os }}.json
  test-report.xml
  retention-days: 1 # temporary, combined in aggregate below
 

tests/cli/test_main_render.py

@@ -1,7 +1,10 @@
 # Copyright (C) 2014 Anaconda, Inc
 # SPDX-License-Identifier: BSD-3-Clause
+from __future__ import annotations
+
 import os
 import sys
+from typing import TYPE_CHECKING
 
 import pytest
 import yaml
@@ -12,6 +15,12 @@
 
 from ..utils import metadata_dir
 
+if TYPE_CHECKING:
+ from pytest import CaptureFixture
+
+ from conda_build.config import Config
+ from conda_build.metadata import MetaData
+
 
 def test_render_add_channel():
  """This recipe requires the conda_build_test_requirement package, which is
@@ -67,7 +76,10 @@ def test_render_without_channel_fails(tmp_path):
 
 
 def test_render_output_build_path(
- testing_workdir, testing_config, testing_metadata, capfd, caplog
+ testing_workdir: str,
+ testing_config: Config,
+ testing_metadata: MetaData,
+ capfd: CaptureFixture,
 ):
  api.output_yaml(testing_metadata, "meta.yaml")
  args = ["--output", testing_workdir]
@@ -83,7 +95,10 @@ def test_render_output_build_path(
 
 
 def test_render_output_build_path_and_file(
- testing_workdir, testing_config, testing_metadata, capfd, caplog
+ testing_workdir: str,
+ testing_config: Config,
+ testing_metadata: MetaData,
+ capfd: CaptureFixture,
 ):
  api.output_yaml(testing_metadata, "meta.yaml")
  rendered_filename = "out.yaml"

tests/conftest.py

@@ -55,16 +55,12 @@ def testing_workdir(monkeypatch: MonkeyPatch, tmp_path: Path) -> Iterator[str]:
 
 
 @pytest.fixture(scope="function")
-def testing_homedir() -> Iterator[Path]:
+def testing_homedir(monkeypatch: MonkeyPatch) -> Iterator[Path]:
  """Create a temporary testing directory in the users home directory; cd into dir before test, cd out after."""
- saved = Path.cwd()
  try:
  with tempfile.TemporaryDirectory(dir=Path.home(), prefix=".pytest_") as home:
- os.chdir(home)
-
- yield home
-
- os.chdir(saved)
+ monkeypatch.chdir(home)
+ yield Path(home)
  except OSError:
  pytest.xfail(
  f"failed to create temporary directory () in {'%HOME%' if on_win else '${HOME}'} "
@@ -73,7 +69,7 @@ def testing_homedir() -> Iterator[Path]:
 
 
 @pytest.fixture(scope="function")
-def testing_config(testing_workdir):
+def testing_config(testing_workdir: str) -> Config:
  def boolify(v):
  return v == "true"
 

tests/requirements-ci.txt

@@ -1,4 +1,5 @@
 anaconda-client
+conda-forge::pytest-split
 conda-forge::xdoctest
 conda-verify
 contextlib2