From 6f3b6cfdfd3810a92959b4b313285de9faedeef1 Mon Sep 17 00:00:00 2001 From: Keyvan Khademi Date: Thu, 21 Mar 2024 07:53:27 -0700 Subject: [PATCH] fix: add type check and fix the type issues (#26) * feat: add pyright type check to pre commit * chore: update the dependencies required to fix the type issues * fix: all the type issues * fix: install dependencies before running pre commit in ci * fix: python version in ci * fix: pre-commit issue on ci * fix: refine typing and fix grammar in error message The 'Any' type was removed from the `on_notify_price_sched` callback function in the `Pythd` class, specifying it to not return any value. In `publisher.py`, a grammatical error was corrected in the exception message for an unknown provider. --------- Co-authored-by: Keyvan --- .github/workflows/pre-commit.yaml | 13 +- .pre-commit-config.yaml | 4 + example_publisher/publisher.py | 14 +- example_publisher/pythd.py | 21 +- .../test_pyth_replicator_manual_aggregate.py | 3 +- poetry.lock | 197 +++++++++++++----- pyproject.toml | 5 +- 7 files changed, 184 insertions(+), 73 deletions(-) diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml index a56cde4..ec30efb 100644 --- a/.github/workflows/pre-commit.yaml +++ b/.github/workflows/pre-commit.yaml @@ -10,5 +10,14 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - uses: pre-commit/action@v2.0.3 + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + - name: Run image + uses: abatilo/actions-poetry@v2 + with: + poetry-version: '1.3.2' + - name: Install dependencies + run: poetry install + - name: Run pre-commit + run: poetry run pre-commit run --all-files diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d10d65d..e16d78a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,3 +14,7 @@ repos: rev: 6.0.0 hooks: - id: flake8 +- repo: https://github.com/fsouza/mirrors-pyright + rev: v1.1.354 + hooks: + - id: pyright diff --git a/example_publisher/publisher.py b/example_publisher/publisher.py index 0063e34..64c2e68 100644 --- a/example_publisher/publisher.py +++ b/example_publisher/publisher.py @@ -32,12 +32,20 @@ def __init__(self, config: Config) -> None: if not getattr(self.config, self.config.provider_engine): raise ValueError(f"Missing {self.config.provider_engine} config") - if self.config.provider_engine == "coin_gecko": + if ( + self.config.provider_engine == "coin_gecko" + and config.coin_gecko is not None + ): self.provider = CoinGecko(config.coin_gecko) - elif self.config.provider_engine == "pyth_replicator": + elif ( + self.config.provider_engine == "pyth_replicator" + and config.pyth_replicator is not None + ): self.provider: Provider = PythReplicator(config.pyth_replicator) else: - raise ValueError(f"Unknown provider {self.config.provider_engine}") + raise ValueError( + f"Unknown provider {self.config.provider_engine}, possibly the env variables are not set." + ) self.pythd: Pythd = Pythd( address=config.pythd.endpoint, diff --git a/example_publisher/pythd.py b/example_publisher/pythd.py index 5c8a7de..930899b 100644 --- a/example_publisher/pythd.py +++ b/example_publisher/pythd.py @@ -2,8 +2,8 @@ from dataclasses import dataclass, field import sys import traceback -from dataclasses_json import config, dataclass_json -from typing import Awaitable, Callable, List +from dataclasses_json import config, DataClassJsonMixin +from typing import Callable, Coroutine, List from structlog import get_logger from jsonrpc_websocket import Server @@ -15,22 +15,19 @@ TRADING = "trading" -@dataclass_json @dataclass -class Price: +class Price(DataClassJsonMixin): account: str exponent: int = field(metadata=config(field_name="price_exponent")) -@dataclass_json @dataclass -class Metadata: +class Metadata(DataClassJsonMixin): symbol: str -@dataclass_json @dataclass -class Product: +class Product(DataClassJsonMixin): account: str metadata: Metadata = field(metadata=config(field_name="attr_dict")) prices: List[Price] = field(metadata=config(field_name="price")) @@ -38,14 +35,16 @@ class Product: class Pythd: def __init__( - self, address: str, on_notify_price_sched: Callable[[SubscriptionId], Awaitable] + self, + address: str, + on_notify_price_sched: Callable[[SubscriptionId], Coroutine[None, None, None]], ) -> None: self.address = address - self.server: Server = None + self.server: Server self.on_notify_price_sched = on_notify_price_sched self._tasks = set() - async def connect(self) -> Server: + async def connect(self): self.server = Server(self.address) self.server.notify_price_sched = self._notify_price_sched task = await self.server.ws_connect() diff --git a/example_publisher/tests/test_pyth_replicator_manual_aggregate.py b/example_publisher/tests/test_pyth_replicator_manual_aggregate.py index b0d2a9c..a003744 100644 --- a/example_publisher/tests/test_pyth_replicator_manual_aggregate.py +++ b/example_publisher/tests/test_pyth_replicator_manual_aggregate.py @@ -1,9 +1,10 @@ import random +from typing import List from example_publisher.providers.pyth_replicator import manual_aggregate def test_manual_aggregate_works(): - prices = [1, 2, 3, 4, 5, 6, 8, 10, 12, 14] + prices: List[float] = [1, 2, 3, 4, 5, 6, 8, 10, 12, 14] random.shuffle(prices) agg_price, agg_confidence_interval = manual_aggregate(prices) diff --git a/poetry.lock b/poetry.lock index 3f1332b..531b351 100644 --- a/poetry.lock +++ b/poetry.lock @@ -263,21 +263,6 @@ files = [ [package.extras] tests = ["PyHamcrest (>=2.0.2)", "mypy", "pytest (>=4.6)", "pytest-benchmark", "pytest-cov", "pytest-flake8"] -[[package]] -name = "cattrs" -version = "22.2.0" -description = "Composable complex class support for attrs and dataclasses." -optional = false -python-versions = ">=3.7" -files = [ - {file = "cattrs-22.2.0-py3-none-any.whl", hash = "sha256:bc12b1f0d000b9f9bee83335887d532a1d3e99a833d1bf0882151c97d3e68c21"}, - {file = "cattrs-22.2.0.tar.gz", hash = "sha256:f0eed5642399423cf656e7b66ce92cdc5b963ecafd041d1b24d136fdde7acf6d"}, -] - -[package.dependencies] -attrs = ">=20" -exceptiongroup = {version = "*", markers = "python_version < \"3.11\""} - [[package]] name = "certifi" version = "2022.12.7" @@ -365,6 +350,17 @@ files = [ [package.dependencies] pycparser = "*" +[[package]] +name = "cfgv" +version = "3.4.0" +description = "Validate configuration and produce human readable error messages." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cfgv-3.4.0-py2.py3-none-any.whl", hash = "sha256:b7265b1f29fd3316bfcd2b330d63d024f2bfd8bcb8b0272f8e19a504856c48f9"}, + {file = "cfgv-3.4.0.tar.gz", hash = "sha256:e52591d4c5f5dead8e0f673fb16db7949d2cfb3f7da4582893288f0ded8fe560"}, +] + [[package]] name = "charset-normalizer" version = "3.0.1" @@ -489,22 +485,18 @@ files = [ [[package]] name = "dataclasses-json" -version = "0.5.7" -description = "Easily serialize dataclasses to and from JSON" +version = "0.6.4" +description = "Easily serialize dataclasses to and from JSON." optional = false -python-versions = ">=3.6" +python-versions = ">=3.7,<4.0" files = [ - {file = "dataclasses-json-0.5.7.tar.gz", hash = "sha256:c2c11bc8214fbf709ffc369d11446ff6945254a7f09128154a7620613d8fda90"}, - {file = "dataclasses_json-0.5.7-py3-none-any.whl", hash = "sha256:bc285b5f892094c3a53d558858a88553dd6a61a11ab1a8128a0e554385dcc5dd"}, + {file = "dataclasses_json-0.6.4-py3-none-any.whl", hash = "sha256:f90578b8a3177f7552f4e1a6e535e84293cd5da421fcce0642d49c0d7bdf8df2"}, + {file = "dataclasses_json-0.6.4.tar.gz", hash = "sha256:73696ebf24936560cca79a2430cbc4f3dd23ac7bf46ed17f38e5e5e7657a6377"}, ] [package.dependencies] -marshmallow = ">=3.3.0,<4.0.0" -marshmallow-enum = ">=1.5.1,<2.0.0" -typing-inspect = ">=0.4.0" - -[package.extras] -dev = ["flake8", "hypothesis", "ipython", "mypy (>=0.710)", "portray", "pytest (>=6.2.3)", "simplejson", "types-dataclasses"] +marshmallow = ">=3.18.0,<4.0.0" +typing-inspect = ">=0.4.0,<1" [[package]] name = "dill" @@ -520,6 +512,17 @@ files = [ [package.extras] graph = ["objgraph (>=1.7.2)"] +[[package]] +name = "distlib" +version = "0.3.8" +description = "Distribution utilities" +optional = false +python-versions = "*" +files = [ + {file = "distlib-0.3.8-py2.py3-none-any.whl", hash = "sha256:034db59a0b96f8ca18035f36290806a9a6e6bd9d1ff91e45a7f172eb17e51784"}, + {file = "distlib-0.3.8.tar.gz", hash = "sha256:1530ea13e350031b6312d8580ddb6b27a104275a31106523b8f123787f494f64"}, +] + [[package]] name = "dnspython" version = "2.3.0" @@ -573,6 +576,22 @@ typing-extensions = ">=4.8.0" [package.extras] all = ["email-validator (>=2.0.0)", "httpx (>=0.23.0)", "itsdangerous (>=1.1.0)", "jinja2 (>=2.11.2)", "orjson (>=3.2.1)", "pydantic-extra-types (>=2.0.0)", "pydantic-settings (>=2.0.0)", "python-multipart (>=0.0.7)", "pyyaml (>=5.3.1)", "ujson (>=4.0.1,!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0)", "uvicorn[standard] (>=0.12.0)"] +[[package]] +name = "filelock" +version = "3.13.1" +description = "A platform independent file lock." +optional = false +python-versions = ">=3.8" +files = [ + {file = "filelock-3.13.1-py3-none-any.whl", hash = "sha256:57dbda9b35157b05fb3e58ee91448612eb674172fab98ee235ccb0b5bee19a1c"}, + {file = "filelock-3.13.1.tar.gz", hash = "sha256:521f5f56c50f8426f5e03ad3b281b490a87ef15bc6c526f168290f0c7148d44e"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.24)"] +testing = ["covdefaults (>=2.3)", "coverage (>=7.3.2)", "diff-cover (>=8)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)", "pytest-timeout (>=2.2)"] +typing = ["typing-extensions (>=4.8)"] + [[package]] name = "flake8" version = "6.0.0" @@ -731,6 +750,20 @@ files = [ [package.extras] test = ["Cython (>=0.29.24,<0.30.0)"] +[[package]] +name = "identify" +version = "2.5.35" +description = "File identification library for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "identify-2.5.35-py2.py3-none-any.whl", hash = "sha256:c4de0081837b211594f8e877a6b4fad7ca32bbfc1a9307fdd61c28bfe923f13e"}, + {file = "identify-2.5.35.tar.gz", hash = "sha256:10a7ca245cfcd756a554a7288159f72ff105ad233c7c4b9c6f0f4d108f5f6791"}, +] + +[package.extras] +license = ["ukkonen"] + [[package]] name = "idna" version = "3.4" @@ -877,20 +910,6 @@ docs = ["alabaster (==0.7.12)", "autodocsumm (==0.2.9)", "sphinx (==5.3.0)", "sp lint = ["flake8 (==5.0.4)", "flake8-bugbear (==22.10.25)", "mypy (==0.990)", "pre-commit (>=2.4,<3.0)"] tests = ["pytest", "pytz", "simplejson"] -[[package]] -name = "marshmallow-enum" -version = "1.5.1" -description = "Enum field for Marshmallow" -optional = false -python-versions = "*" -files = [ - {file = "marshmallow-enum-1.5.1.tar.gz", hash = "sha256:38e697e11f45a8e64b4a1e664000897c659b60aa57bfa18d44e226a9920b6e58"}, - {file = "marshmallow_enum-1.5.1-py2.py3-none-any.whl", hash = "sha256:57161ab3dbfde4f57adeb12090f39592e992b9c86d206d02f6bd03ebec60f072"}, -] - -[package.dependencies] -marshmallow = ">=2.0.0" - [[package]] name = "mccabe" version = "0.7.0" @@ -996,6 +1015,20 @@ files = [ {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, ] +[[package]] +name = "nodeenv" +version = "1.8.0" +description = "Node.js virtual environment builder" +optional = false +python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" +files = [ + {file = "nodeenv-1.8.0-py2.py3-none-any.whl", hash = "sha256:df865724bb3c3adc86b3876fa209771517b0cfe596beff01a92700e0e8be4cec"}, + {file = "nodeenv-1.8.0.tar.gz", hash = "sha256:d51e0c37e64fbf47d017feac3145cdbb58836d7eee8c6f6d3b6880c5456227d2"}, +] + +[package.dependencies] +setuptools = "*" + [[package]] name = "numpy" version = "1.24.2" @@ -1085,6 +1118,24 @@ files = [ dev = ["pre-commit", "tox"] testing = ["pytest", "pytest-benchmark"] +[[package]] +name = "pre-commit" +version = "3.6.2" +description = "A framework for managing and maintaining multi-language pre-commit hooks." +optional = false +python-versions = ">=3.9" +files = [ + {file = "pre_commit-3.6.2-py2.py3-none-any.whl", hash = "sha256:ba637c2d7a670c10daedc059f5c49b5bd0aadbccfcd7ec15592cf9665117532c"}, + {file = "pre_commit-3.6.2.tar.gz", hash = "sha256:c3ef34f463045c88658c5b99f38c1e297abdcc0ff13f98d3370055fbbfabc67e"}, +] + +[package.dependencies] +cfgv = ">=2.0.0" +identify = ">=1.0.0" +nodeenv = ">=0.11.1" +pyyaml = ">=5.1" +virtualenv = ">=20.10.0" + [[package]] name = "py" version = "1.11.0" @@ -1491,6 +1542,22 @@ urllib3 = ">=1.21.1,<1.27" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +[[package]] +name = "setuptools" +version = "69.2.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.2.0-py3-none-any.whl", hash = "sha256:c21c49fb1042386df081cb5d86759792ab89efca84cf114889191cd09aacc80c"}, + {file = "setuptools-69.2.0.tar.gz", hash = "sha256:0ff4183f8f42cd8fa3acea16c45205521a4ef28f73c6391d8a25e92893134f2e"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + [[package]] name = "sniffio" version = "1.3.1" @@ -1571,28 +1638,30 @@ files = [ [[package]] name = "typed-settings" -version = "2.0.2" +version = "24.2.0" description = "Typed settings based on attrs classes" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "typed_settings-2.0.2-py3-none-any.whl", hash = "sha256:773cd3ae905db465983e422df0d74a39327c4288d957a0eea7391c127bd6c1fb"}, - {file = "typed_settings-2.0.2.tar.gz", hash = "sha256:0181c0d71152d20f7d725a06223be2f25a0a4bf43f02d7b22e22c7f2b7fed8da"}, + {file = "typed_settings-24.2.0-py3-none-any.whl", hash = "sha256:6788ecdb3bf42b561c8a61630588d26d5a3d09dda47c8027e29e6698796252d4"}, + {file = "typed_settings-24.2.0.tar.gz", hash = "sha256:06ea2c7c89428020fe87b780ffafe813df3375445a4f77a293e772b01a4947e6"}, ] [package.dependencies] -attrs = ">=22.1" -cattrs = ">=22.2" tomli = {version = ">=2", markers = "python_version < \"3.11\""} [package.extras] -all = ["typed-settings[click,option-groups]"] -click = ["click (>=7,<9)"] -dev = ["nox", "safety", "typed-settings[docs,lint,test]"] -docs = ["furo", "sphinx", "typed-settings[all]"] -lint = ["autoflake", "bandit", "flake8 (>=4.0.0,<5.0.0)", "flake8-bandit", "flake8-black", "flake8-bugbear", "flake8-isort", "mypy", "typed-settings[all]", "types-toml"] -option-groups = ["click (>=7,<9)", "click-option-group"] -test = ["coverage[toml] (>=5.3)", "pytest (>=7.2.0)", "pytest-cov", "rich-click (>=1.6)", "typed-settings[all]", "typing-extensions"] +all = ["typed-settings[attrs,cattrs,click,jinja,option-groups,pydantic]"] +attrs = ["attrs (>=23.1)"] +cattrs = ["cattrs (>=22.2)"] +click = ["click (>=7)"] +dev = ["nox", "pip-audit", "typed-settings[docs,lint,test]"] +docs = ["furo (>=2023.9)", "myst-parser (>=2.0)", "sphinx (>=7.2)", "sphinx-copybutton (>=0.5.2)", "sphinx-inline-tabs (>=2023.4.21)", "typed-settings[all]"] +jinja = ["jinja2"] +lint = ["black", "mypy", "ruff", "typed-settings[all]", "types-toml"] +option-groups = ["click (>=7)", "click-option-group"] +pydantic = ["pydantic (>=2)"] +test = ["coverage[toml] (>=5.3)", "pytest (>=7.2.0)", "pytest-cov", "rich-click (>=1.6)", "sybil (>=6)", "typed-settings[all]", "typing-extensions"] [[package]] name = "typing-extensions" @@ -1706,6 +1775,26 @@ files = [ docs = ["Sphinx (>=4.1.2,<4.2.0)", "sphinx-rtd-theme (>=0.5.2,<0.6.0)", "sphinxcontrib-asyncio (>=0.3.0,<0.4.0)"] test = ["Cython (>=0.29.36,<0.30.0)", "aiohttp (==3.9.0b0)", "aiohttp (>=3.8.1)", "flake8 (>=5.0,<6.0)", "mypy (>=0.800)", "psutil", "pyOpenSSL (>=23.0.0,<23.1.0)", "pycodestyle (>=2.9.0,<2.10.0)"] +[[package]] +name = "virtualenv" +version = "20.21.1" +description = "Virtual Python Environment builder" +optional = false +python-versions = ">=3.7" +files = [ + {file = "virtualenv-20.21.1-py3-none-any.whl", hash = "sha256:09ddbe1af0c8ed2bb4d6ed226b9e6415718ad18aef9fa0ba023d96b7a8356049"}, + {file = "virtualenv-20.21.1.tar.gz", hash = "sha256:4c104ccde994f8b108163cf9ba58f3d11511d9403de87fb9b4f52bf33dbc8668"}, +] + +[package.dependencies] +distlib = ">=0.3.6,<1" +filelock = ">=3.4.1,<4" +platformdirs = ">=2.4,<4" + +[package.extras] +docs = ["furo (>=2023.3.27)", "proselint (>=0.13)", "sphinx (>=6.1.3)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=22.12)"] +test = ["covdefaults (>=2.3)", "coverage (>=7.2.3)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23.1)", "pytest (>=7.3.1)", "pytest-env (>=0.8.1)", "pytest-freezegun (>=0.4.2)", "pytest-mock (>=3.10)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)"] + [[package]] name = "watchfiles" version = "0.21.0" @@ -2062,4 +2151,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "aa62f4f82930606c84e89509537d6065c54f34f1a9989eba4b914a5c385e90ea" +content-hash = "6116d6e7b26e7b1e2643c59b9f41c488c2a419a37373797d504c291cc3982157" diff --git a/pyproject.toml b/pyproject.toml index 10ef708..19aa8b3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,12 +18,12 @@ norecursedirs= ".git,.venv" python = "^3.10" structlog = "^22.3.0" click = "^8.1.3" -dataclasses-json = "^0.5.7" +dataclasses-json = "^0.6.4" attr = "^0.3.2" numpy = "^1.24.2" jsonrpc-websocket = "^3.1.4" pycoingecko = "^2.2.0" -typed-settings = "2.0.2" +typed-settings = "24.2.0" pythclient = "^0.1.4," fastapi = "^0.110.0" uvicorn = {extras = ["standard"], version = "^0.28.0"} @@ -33,6 +33,7 @@ pylint = "^2.16.2" pep8 = "^1.7.1" flake8 = "^6.0.0" pytest = "^6.2.0" +pre-commit = "^3.6.2" [build-system] requires = ["poetry-core"]