ci: fix CI

Test Plan: - run tests - run CI Reviewers: Pull Request: #2
spraakbanken · Jan 25, 2024 · d844679 · d844679
1 parent 8c6fe5c
commit d844679
Show file tree

Hide file tree

Showing 7 changed files with 212 additions and 65 deletions.
diff --git a/.github/actions/setup-python-env/action.yml b/.github/actions/setup-python-env/action.yml
@@ -10,39 +10,23 @@ inputs:
     required: false
     description: "The python version to use"
     default: "3.11"
-  rye-version:
-    required: false
-    description: "The version of rye to use"
-    default: "latest"
 outputs:
   python-version:
     description: "Python version used"
     value: ${{ steps.set-python-version.outputs.python-version }}
-  rye-version:
-    description: "rye version used"
-    value: ${{ steps.set-rye-version.outputs.poetry-version }}
 
 runs:
   using: "composite"
   steps:
-    - name: Install rye
-      id: setup-rye
-      uses: eifinger/setup-rye@f8f2ca26808cdffef0b77c4fddf4305a5e5b010a
+    - name: Setup Python
+      id: setup-python
+      uses: actions/setup-python@v5
       with:
-        version: ${{ inputs.rye-version }}
-
-    - name: Pin python version
-      run: rye pin ${{ inputs.python-version }}
-      shell: bash
+        python-version: ${{ inputs.python-version }}
 
     - name: Output python version
       id: set-python-version
-      run: echo "python-version=$(rye run python -c 'import sys; print(".".join(map(str, sys.version_info[0:3])))')" >> $GITHUB_OUTPUT
-      shell: bash
-
-    - name: Output rye version
-      id: set-rye-version
-      run: echo "rye-version=${{ steps.setup-rye.outputs.rye-version}}" >> $GITHUB_OUTPUT
+      run: echo "python-version=${{ steps.setup-python.outputs.python-version }}" >> $GITHUB_OUTPUT
       shell: bash
 
     - name: Load cached venv
@@ -54,5 +38,13 @@ runs:
 
     - name: Install dependencies
       if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-      run: rye sync --features=ci
+      run: |
+        test -d .venv || python -m venv .venv
+        pip install wheel
+        pip install -r tests/requirements-testing.txt
+        pip install -e .
+      shell: bash
+
+    - name: list .venv
+      run: ls -R .venv
       shell: bash
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -2,6 +2,8 @@ name: CI
 
 on:
   push:
+    branches:
+      - main
   pull_request:
   merge_group:
 
@@ -10,6 +12,7 @@ permissions:
 
 env:
   MINIMUM_PYTHON_VERSION: "3.9"
+  CI: "github-actions"
 
 jobs:
   lint:
@@ -38,12 +41,31 @@ jobs:
         uses: actions/checkout@v4
 
       #----------------------------------------------
-      #  -----  install & configure python & rye  -----
+      #  -----  setup python   -----
       #----------------------------------------------
       - name: Set up the environment
-        uses: ./.github/actions/setup-python-env
+        uses: actions/setup-python@v5
+        id: setup-python
         with:
           python-version: ${{ env.MINIMUM_PYTHON_VERSION }}
+
+
+      - name: Load cached venv
+        id: cached-venv
+        uses: actions/cache@v3
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/pyproject.toml') }}
+
+      - name: Install dependencies
+        if: steps.cached-venv.outputs.cache-hit != 'true'
+        run: |
+          test -d .venv || python -m venv .venv
+          pip install wheel
+          pip install -r tests/requirements-testing.txt
+          pip install -e .
+        shell: bash
+      - run: make info
       #----------------------------------------------
       # Run matrix task
       #----------------------------------------------
@@ -61,10 +83,7 @@ jobs:
           - "3.9"
           - "3.10"
           - "3.11"
-          - "3.12"
-
-#     continue-on-error: >-
-#       ${{ contains(matrix.python-version, 'dev') && true || false }}
+          # - "3.12" not supported
 
     runs-on: ${{ matrix.os }}-latest
     steps:
@@ -75,12 +94,30 @@ jobs:
         uses: actions/checkout@v4
 
       #----------------------------------------------
-      #  -----  install & configure python & rye  -----
+      #  -----  setup python   -----
       #----------------------------------------------
       - name: Set up the environment
-        uses: ./.github/actions/setup-python-env
+        uses: actions/setup-python@v5
+        id: setup-python
         with:
           python-version: ${{ matrix.python-version }}
+
+
+      - name: Load cached venv
+        id: cached-venv
+        uses: actions/cache@v3
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/pyproject.toml') }}
+
+      - name: Install dependencies
+        if: steps.cached-venv.outputs.cache-hit != 'true'
+        run: |
+          test -d .venv || python -m venv .venv
+          pip install wheel
+          pip install -r tests/requirements-testing.txt
+          pip install -e .
+        shell: bash
 
 
       #----------------------------------------------
@@ -89,16 +126,11 @@ jobs:
       - name: Run tests for coverage
         run: make test-w-coverage
 
-      - name: copy coverage
-        run: |
-          mkdir coverage
-          cp -v .coverage coverage/.coverage.${{ matrix.os }}-py${{ matrix.python-version }}
-
-      - name: store coverage files
+      - name: Upload coverage artifact
         uses: actions/upload-artifact@v4
         with:
-          name: coverage
-          path: coverage
+          name: .coverage-${{ matrix.os }}-${{ matrix.python-version }}
+          path: .coverage
 
 
   upload-coverage:
@@ -125,18 +157,17 @@ jobs:
       - name: Install dependencies
         run: pip install coverage
 
-      - name: download coverage files
+      - name: Download coverage artifacts
         uses: actions/download-artifact@v4
         with:
-          name: coverage
-          path: coverage
+          pattern: .coverage-*
 
       - name: convert coverage to xml
         run: |
-          coverage combine --data-file=coverage/.coverage
-          coverage xml -i --data-file=coverage/.coverage
-          ls
-          ls coverage
+          ls -aR
+          coverage combine .coverage*/.coverage
+          coverage xml -i 
+          ls -a
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
@@ -176,19 +207,21 @@ jobs:
         uses: actions/checkout@v4
 
       #----------------------------------------------
-      #  -----  install & configure rye  -----
+      #  -----  setup python   -----
       #----------------------------------------------
-      - name: Install rye
-        id: setup-rye
-        uses: eifinger/setup-rye@5b24b104a68da1e2ea0073aa191a2bab66f55d7c
+      - name: Set up the environment
+        uses: actions/setup-python@v5
+        id: setup-python
+        with:
+          python-version: ${{ env.MINIMUM_PYTHON_VERSION }}
 
-      - name: Set up python ${{ env.MINIMUM_PYTHON_VERSION }}
-        run: rye pin ${{ env.MINIMUM_PYTHON_VERSION }}
+      - name: Set up build environment
+        run: pip install build
       #----------------------------------------------
       #  -----  build distribution -----
       #----------------------------------------------
       - name: Build distribution
-        run: rye build
+        run: python -m build
 
       #----------------------------------------------
       #  -----  upload artifacts  -----
@@ -222,10 +255,10 @@ jobs:
           name: pypi_files
           path: dist
 
-      - run: rm -r src/sparv_bert_neighbour
+      - run: rm -r src/sparv_ocr_suggestion
       - run: pip install typing-extensions
-      - run: pip install -r tests/requirements-test.txt
-      - run: pip install sparv-bert-neighbour-plugin --no-index --no-deps --find-links dist --force-reinstall
+      - run: pip install -r tests/requirements-testing.txt
+      - run: pip install sparv-ocr-suggestion-plugin --no-index --no-deps --find-links dist --force-reinstall
       - run: pytest
 
   publish:

diff --git a/Makefile b/Makefile
@@ -34,13 +34,17 @@ PROJECT_SRC := "src/sparv_ocr_suggestion"
 
 ifeq (${VIRTUAL_ENV},)
   VENV_NAME = .venv
-  INVENV = rye run
+  ifeq (${CI},)
+    INVENV = rye run
+  else
+    INVENV = export VIRTUAL_ENV="${VENV_NAME}"; export PATH="${VENV_NAME}/bin:${PATH}"; unset PYTHON_HOME;
+  endif
 else
   VENV_NAME = ${VIRTUAL_ENV}
   INVENV =
 endif
 
-default_cov := "--cov=src/json_streams"
+default_cov := "--cov=${PROJECT_SRC}"
 cov_report := "term-missing"
 cov := ${default_cov}
 
@@ -50,6 +54,7 @@ tests := tests
 info:
 	@echo "Platform: ${PLATFORM}"
 	@echo "INVENV: '${INVENV}'"
+	@echo "CI: '${CI}'"
 
 dev: install-dev
 

diff --git a/pyproject.toml b/pyproject.toml
@@ -32,9 +32,6 @@ build-backend = "hatchling.build"
 [project.entry-points."sparv.plugin"]
 sparv_ocr_suggestion = "sparv_ocr_suggestion"
 
-[project.optional-dependencies]
-ci = ["ruff>=0.1.1"]
-
 [project.urls]
 Homepage = "https://github.com/spraakbanken/sparv-ocr-suggestion-plugin"
 Repository = "https://github.com/spraakbanken/sparv-ocr-suggestion-plugin"
@@ -44,10 +41,10 @@ Repository = "https://github.com/spraakbanken/sparv-ocr-suggestion-plugin"
 managed = true
 dev-dependencies = [
     "pytest>=7.4.2",
-    "black>=23.10.1",
     "mypy>=1.8.0",
     "bump2version>=1.0.1",
     "pytest-cov>=4.1.0",
+    "ruff>=0.1.14",
 ]
 
 [tool.hatch.build.targets.sdist]

diff --git a/src/sparv_ocr_suggestion/__init__.py b/src/sparv_ocr_suggestion/__init__.py
@@ -1,4 +1,4 @@
-from typing import Iterable, Optional
+from typing import Optional
 from sparv.api import (  # type: ignore [import-untyped]
     annotator,
     Output,
@@ -7,7 +7,7 @@
     Config,
 )
 
-from transformers import pipeline, T5ForConditionalGeneration, AutoTokenizer
+from transformers import pipeline, T5ForConditionalGeneration, AutoTokenizer  # type: ignore [import-untyped]
 
 __description__ = "Calculating word neighbours by mask a word in a BERT model."
 
@@ -79,7 +79,7 @@ def __init__(self, *, tokenizer, model) -> None:
             "text2text-generation", model=model, tokenizer=tokenizer
         )
 
-    def calculate_suggestions(self, text: str) -> str:
+    def calculate_suggestions(self, text: str) -> list[Optional[str]]:
         logger.warning("Analyzing '%s'", text)
         suggested_text = self.pipeline(text)
         logger.warning("Output: '%s'", suggested_text)

diff --git a/tests/conftest.py b/tests/conftest.py
@@ -1,6 +1,6 @@
 import pytest
 
-from transformers import T5ForConditionalGeneration, AutoTokenizer
+from transformers import T5ForConditionalGeneration, AutoTokenizer  # type: ignore [import-untyped]
 from sparv_ocr_suggestion import (
     DEFAULT_TOKENIZER_NAME,
     OcrSuggestor,