From d844679d763713e646034348eb62dd7c26313a3a Mon Sep 17 00:00:00 2001 From: Kristoffer Andersson Date: Thu, 25 Jan 2024 11:39:26 +0100 Subject: [PATCH] ci: fix CI Test Plan: - run tests - run CI Reviewers: Pull Request: https://github.com/spraakbanken/sparv-ocr-suggestion-plugin/pull/2 --- .github/actions/setup-python-env/action.yml | 36 +++--- .github/workflows/ci.yml | 99 ++++++++++------ Makefile | 9 +- pyproject.toml | 5 +- src/sparv_ocr_suggestion/__init__.py | 6 +- tests/conftest.py | 2 +- tests/requirements-testing.txt | 120 ++++++++++++++++++++ 7 files changed, 212 insertions(+), 65 deletions(-) create mode 100644 tests/requirements-testing.txt diff --git a/.github/actions/setup-python-env/action.yml b/.github/actions/setup-python-env/action.yml index e1d8d7b..2329507 100644 --- a/.github/actions/setup-python-env/action.yml +++ b/.github/actions/setup-python-env/action.yml @@ -10,39 +10,23 @@ inputs: required: false description: "The python version to use" default: "3.11" - rye-version: - required: false - description: "The version of rye to use" - default: "latest" outputs: python-version: description: "Python version used" value: ${{ steps.set-python-version.outputs.python-version }} - rye-version: - description: "rye version used" - value: ${{ steps.set-rye-version.outputs.poetry-version }} runs: using: "composite" steps: - - name: Install rye - id: setup-rye - uses: eifinger/setup-rye@f8f2ca26808cdffef0b77c4fddf4305a5e5b010a + - name: Setup Python + id: setup-python + uses: actions/setup-python@v5 with: - version: ${{ inputs.rye-version }} - - - name: Pin python version - run: rye pin ${{ inputs.python-version }} - shell: bash + python-version: ${{ inputs.python-version }} - name: Output python version id: set-python-version - run: echo "python-version=$(rye run python -c 'import sys; print(".".join(map(str, sys.version_info[0:3])))')" >> $GITHUB_OUTPUT - shell: bash - - - name: Output rye version - id: set-rye-version - run: echo "rye-version=${{ steps.setup-rye.outputs.rye-version}}" >> $GITHUB_OUTPUT + run: echo "python-version=${{ steps.setup-python.outputs.python-version }}" >> $GITHUB_OUTPUT shell: bash - name: Load cached venv @@ -54,5 +38,13 @@ runs: - name: Install dependencies if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' - run: rye sync --features=ci + run: | + test -d .venv || python -m venv .venv + pip install wheel + pip install -r tests/requirements-testing.txt + pip install -e . + shell: bash + + - name: list .venv + run: ls -R .venv shell: bash diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 90b1338..381b4d0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,6 +2,8 @@ name: CI on: push: + branches: + - main pull_request: merge_group: @@ -10,6 +12,7 @@ permissions: env: MINIMUM_PYTHON_VERSION: "3.9" + CI: "github-actions" jobs: lint: @@ -38,12 +41,31 @@ jobs: uses: actions/checkout@v4 #---------------------------------------------- - # ----- install & configure python & rye ----- + # ----- setup python ----- #---------------------------------------------- - name: Set up the environment - uses: ./.github/actions/setup-python-env + uses: actions/setup-python@v5 + id: setup-python with: python-version: ${{ env.MINIMUM_PYTHON_VERSION }} + + + - name: Load cached venv + id: cached-venv + uses: actions/cache@v3 + with: + path: .venv + key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/pyproject.toml') }} + + - name: Install dependencies + if: steps.cached-venv.outputs.cache-hit != 'true' + run: | + test -d .venv || python -m venv .venv + pip install wheel + pip install -r tests/requirements-testing.txt + pip install -e . + shell: bash + - run: make info #---------------------------------------------- # Run matrix task #---------------------------------------------- @@ -61,10 +83,7 @@ jobs: - "3.9" - "3.10" - "3.11" - - "3.12" - -# continue-on-error: >- -# ${{ contains(matrix.python-version, 'dev') && true || false }} + # - "3.12" not supported runs-on: ${{ matrix.os }}-latest steps: @@ -75,12 +94,30 @@ jobs: uses: actions/checkout@v4 #---------------------------------------------- - # ----- install & configure python & rye ----- + # ----- setup python ----- #---------------------------------------------- - name: Set up the environment - uses: ./.github/actions/setup-python-env + uses: actions/setup-python@v5 + id: setup-python with: python-version: ${{ matrix.python-version }} + + + - name: Load cached venv + id: cached-venv + uses: actions/cache@v3 + with: + path: .venv + key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/pyproject.toml') }} + + - name: Install dependencies + if: steps.cached-venv.outputs.cache-hit != 'true' + run: | + test -d .venv || python -m venv .venv + pip install wheel + pip install -r tests/requirements-testing.txt + pip install -e . + shell: bash #---------------------------------------------- @@ -89,16 +126,11 @@ jobs: - name: Run tests for coverage run: make test-w-coverage - - name: copy coverage - run: | - mkdir coverage - cp -v .coverage coverage/.coverage.${{ matrix.os }}-py${{ matrix.python-version }} - - - name: store coverage files + - name: Upload coverage artifact uses: actions/upload-artifact@v4 with: - name: coverage - path: coverage + name: .coverage-${{ matrix.os }}-${{ matrix.python-version }} + path: .coverage upload-coverage: @@ -125,18 +157,17 @@ jobs: - name: Install dependencies run: pip install coverage - - name: download coverage files + - name: Download coverage artifacts uses: actions/download-artifact@v4 with: - name: coverage - path: coverage + pattern: .coverage-* - name: convert coverage to xml run: | - coverage combine --data-file=coverage/.coverage - coverage xml -i --data-file=coverage/.coverage - ls - ls coverage + ls -aR + coverage combine .coverage*/.coverage + coverage xml -i + ls -a - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 @@ -176,19 +207,21 @@ jobs: uses: actions/checkout@v4 #---------------------------------------------- - # ----- install & configure rye ----- + # ----- setup python ----- #---------------------------------------------- - - name: Install rye - id: setup-rye - uses: eifinger/setup-rye@5b24b104a68da1e2ea0073aa191a2bab66f55d7c + - name: Set up the environment + uses: actions/setup-python@v5 + id: setup-python + with: + python-version: ${{ env.MINIMUM_PYTHON_VERSION }} - - name: Set up python ${{ env.MINIMUM_PYTHON_VERSION }} - run: rye pin ${{ env.MINIMUM_PYTHON_VERSION }} + - name: Set up build environment + run: pip install build #---------------------------------------------- # ----- build distribution ----- #---------------------------------------------- - name: Build distribution - run: rye build + run: python -m build #---------------------------------------------- # ----- upload artifacts ----- @@ -222,10 +255,10 @@ jobs: name: pypi_files path: dist - - run: rm -r src/sparv_bert_neighbour + - run: rm -r src/sparv_ocr_suggestion - run: pip install typing-extensions - - run: pip install -r tests/requirements-test.txt - - run: pip install sparv-bert-neighbour-plugin --no-index --no-deps --find-links dist --force-reinstall + - run: pip install -r tests/requirements-testing.txt + - run: pip install sparv-ocr-suggestion-plugin --no-index --no-deps --find-links dist --force-reinstall - run: pytest publish: diff --git a/Makefile b/Makefile index 98de317..312e169 100644 --- a/Makefile +++ b/Makefile @@ -34,13 +34,17 @@ PROJECT_SRC := "src/sparv_ocr_suggestion" ifeq (${VIRTUAL_ENV},) VENV_NAME = .venv - INVENV = rye run + ifeq (${CI},) + INVENV = rye run + else + INVENV = export VIRTUAL_ENV="${VENV_NAME}"; export PATH="${VENV_NAME}/bin:${PATH}"; unset PYTHON_HOME; + endif else VENV_NAME = ${VIRTUAL_ENV} INVENV = endif -default_cov := "--cov=src/json_streams" +default_cov := "--cov=${PROJECT_SRC}" cov_report := "term-missing" cov := ${default_cov} @@ -50,6 +54,7 @@ tests := tests info: @echo "Platform: ${PLATFORM}" @echo "INVENV: '${INVENV}'" + @echo "CI: '${CI}'" dev: install-dev diff --git a/pyproject.toml b/pyproject.toml index c9bf1d2..d4632c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,9 +32,6 @@ build-backend = "hatchling.build" [project.entry-points."sparv.plugin"] sparv_ocr_suggestion = "sparv_ocr_suggestion" -[project.optional-dependencies] -ci = ["ruff>=0.1.1"] - [project.urls] Homepage = "https://github.com/spraakbanken/sparv-ocr-suggestion-plugin" Repository = "https://github.com/spraakbanken/sparv-ocr-suggestion-plugin" @@ -44,10 +41,10 @@ Repository = "https://github.com/spraakbanken/sparv-ocr-suggestion-plugin" managed = true dev-dependencies = [ "pytest>=7.4.2", - "black>=23.10.1", "mypy>=1.8.0", "bump2version>=1.0.1", "pytest-cov>=4.1.0", + "ruff>=0.1.14", ] [tool.hatch.build.targets.sdist] diff --git a/src/sparv_ocr_suggestion/__init__.py b/src/sparv_ocr_suggestion/__init__.py index eb0a006..c69afdf 100644 --- a/src/sparv_ocr_suggestion/__init__.py +++ b/src/sparv_ocr_suggestion/__init__.py @@ -1,4 +1,4 @@ -from typing import Iterable, Optional +from typing import Optional from sparv.api import ( # type: ignore [import-untyped] annotator, Output, @@ -7,7 +7,7 @@ Config, ) -from transformers import pipeline, T5ForConditionalGeneration, AutoTokenizer +from transformers import pipeline, T5ForConditionalGeneration, AutoTokenizer # type: ignore [import-untyped] __description__ = "Calculating word neighbours by mask a word in a BERT model." @@ -79,7 +79,7 @@ def __init__(self, *, tokenizer, model) -> None: "text2text-generation", model=model, tokenizer=tokenizer ) - def calculate_suggestions(self, text: str) -> str: + def calculate_suggestions(self, text: str) -> list[Optional[str]]: logger.warning("Analyzing '%s'", text) suggested_text = self.pipeline(text) logger.warning("Output: '%s'", suggested_text) diff --git a/tests/conftest.py b/tests/conftest.py index bb2939b..b5487ae 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,6 @@ import pytest -from transformers import T5ForConditionalGeneration, AutoTokenizer +from transformers import T5ForConditionalGeneration, AutoTokenizer # type: ignore [import-untyped] from sparv_ocr_suggestion import ( DEFAULT_TOKENIZER_NAME, OcrSuggestor, diff --git a/tests/requirements-testing.txt b/tests/requirements-testing.txt new file mode 100644 index 0000000..b333545 --- /dev/null +++ b/tests/requirements-testing.txt @@ -0,0 +1,120 @@ +# generated by rye +# use `rye lock` or `rye sync` to update this lockfile +# +# last locked with the following flags: +# pre: false +# features: [] +# all-features: false + +-e file:. +appdirs==1.4.4 +argcomplete==3.1.6 +attrs==23.2.0 +black==23.12.1 +bump2version==1.0.1 +certifi==2023.11.17 +cffi==1.16.0 +charset-normalizer==3.3.2 +click==8.1.7 +configargparse==1.7 +connection-pool==0.0.3 +coverage==7.4.0 +cryptography==41.0.7 +datrie==0.8.2 +docutils==0.20.1 +docx2python==1.27.1 +dpath==2.1.6 +emoji==2.10.0 +fastjsonschema==2.19.1 +filelock==3.13.1 +fsspec==2023.12.2 +gitdb==4.0.11 +gitpython==3.1.41 +huggingface-hub==0.20.3 +humanfriendly==10.0 +idna==3.6 +importlib-metadata==6.8.0 +iniconfig==2.0.0 +jinja2==3.1.3 +joblib==1.3.2 +jsonschema==4.20.0 +jsonschema-specifications==2023.12.1 +jupyter-core==5.7.1 +markdown-it-py==3.0.0 +markupsafe==2.1.4 +mdurl==0.1.2 +mpmath==1.3.0 +mypy==1.8.0 +mypy-extensions==1.0.0 +nbformat==5.9.2 +networkx==3.2.1 +nltk==3.8.1 +numpy==1.26.3 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==8.9.2.26 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.18.1 +nvidia-nvjitlink-cu12==12.3.101 +nvidia-nvtx-cu12==12.1.105 +packaging==23.2 +pathspec==0.12.1 +pdfminer-six==20221105 +pdfplumber==0.10.3 +pillow==10.2.0 +plac==1.4.2 +platformdirs==4.1.0 +pluggy==1.3.0 +prompt-toolkit==3.0.43 +protobuf==3.20.3 +psutil==5.9.8 +pulp==2.8.0 +pycountry==22.3.5 +pycparser==2.21 +pygments==2.17.2 +pypdfium2==4.26.0 +pytest==7.4.4 +pytest-cov==4.1.0 +python-dateutil==2.8.2 +python-json-logger==2.0.7 +pyyaml==6.0.1 +questionary==1.10.0 +referencing==0.32.1 +regex==2023.12.25 +requests==2.31.0 +reretry==0.11.8 +rich==13.7.0 +rpds-py==0.17.1 +ruff==0.1.14 +safetensors==0.4.1 +six==1.16.0 +smart-open==6.4.0 +smmap==5.0.1 +snakemake==7.32.3 +sparv-pipeline==5.2.0 +stanza==1.5.1 +stopit==1.1.2 +sympy==1.12 +tabulate==0.9.0 +throttler==1.2.2 +tokenizers==0.15.0 +toposort==1.10 +torch==2.1.2 +tqdm==4.66.1 +traitlets==5.14.1 +transformers==4.37.0 +triton==2.1.0 +typing-extensions==4.9.0 +typing-inspect==0.9.0 +urllib3==2.1.0 +wcwidth==0.2.13 +wrapt==1.16.0 +yte==1.5.4 +zipp==3.17.0 +# The following packages are considered to be unsafe in a requirements file: +setuptools==69.0.3