diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..d7db3ee --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,41 @@ +name: ci + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + check-latest: true + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e ".[dev]" + - name: Check ruff + run: | + ruff check + ruff format --check + - name: Test with pytest and report coverage + run: | + coverage run --branch -m pytest + coverage report + - name: Check types with pyright + run: | + pyright --pythonversion ${{ matrix.python-version }} + - name: Check tach + run: tach check \ No newline at end of file diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..b7a704b --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,39 @@ +# This workflow will upload a Python Package using Twine when a release is created +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Upload Python Package + +on: + release: + types: [published] + +permissions: + contents: read + +jobs: + deploy: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build + - name: Build package + run: python -m build + - name: Publish package + uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/api/src/app.py b/api/src/app.py index d8d2071..02dead4 100644 --- a/api/src/app.py +++ b/api/src/app.py @@ -1,7 +1,8 @@ from __future__ import annotations + from typing import Any -from fastapi import FastAPI, HTTPException, Request, Response +from fastapi import FastAPI, Request, Response from src import settings from src.deploy.routes import router as deploy_router @@ -14,10 +15,10 @@ @app.middleware("http") async def auth_check(request: Request, call_next: Any): - print(settings.CLIENT_SECRET, request.headers.get("X-Client-Secret")) + print(settings.CLIENT_SECRET, request.headers.get("X-Client-Secret")) # type: ignore if ( request.url.path not in AUTH_EXEMPT - and settings.CLIENT_SECRET != request.headers.get("X-Client-Secret") + and settings.CLIENT_SECRET != request.headers.get("X-Client-Secret") # type: ignore ): return Response(status_code=403) @@ -28,4 +29,5 @@ async def auth_check(request: Request, call_next: Any): def healthcheck(): return {"ok": True} + app.include_router(deploy_router) diff --git a/api/src/build/python_lambda.py b/api/src/build/python_lambda.py index 5f763a1..8b9d6b5 100644 --- a/api/src/build/python_lambda.py +++ b/api/src/build/python_lambda.py @@ -1,25 +1,38 @@ -from pathlib import Path +from __future__ import annotations + import subprocess import sys +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from pathlib import Path PIP_PLATFORM = "manylinux2014_x86_64" -def install_deps_to_dir(dependencies: list[str], python_version: str, output_dir: Path) -> None: +def install_deps_to_dir( + dependencies: list[str], python_version: str, output_dir: Path +) -> None: dependencies.append("gauge-serverless") output_dir.mkdir(parents=True, exist_ok=True) pip_command = [ - sys.executable, "-m", "pip", + sys.executable, + "-m", + "pip", "install", - "--platform", PIP_PLATFORM, - "--implementation", "cp", - "--python-version", python_version, - "--only-binary", ":all:", - "--target", str(output_dir), + "--platform", + PIP_PLATFORM, + "--implementation", + "cp", + "--python-version", + python_version, + "--only-binary", + ":all:", + "--target", + str(output_dir), "--upgrade", - "--no-deps" + "--no-deps", ] pip_command.extend(dependencies) diff --git a/api/src/build/zip.py b/api/src/build/zip.py index 45b4c4a..6ef5862 100644 --- a/api/src/build/zip.py +++ b/api/src/build/zip.py @@ -1,6 +1,11 @@ +from __future__ import annotations + import tempfile +from typing import TYPE_CHECKING from zipfile import ZipFile, is_zipfile -from pathlib import Path + +if TYPE_CHECKING: + from pathlib import Path def write_to_zipfile(content: bytes, output_path: Path) -> None: @@ -13,18 +18,20 @@ def write_to_zipfile(content: bytes, output_path: Path) -> None: output_path.write_bytes(content) -def write_extended_zipfile(existing_zipfile: Path, additional_paths: list[Path], output_path: Path) -> None: - with ZipFile(existing_zipfile, 'r') as existing_zip: - with ZipFile(output_path, 'w') as new_zip: +def write_extended_zipfile( + existing_zipfile: Path, additional_paths: list[Path], output_path: Path +) -> None: + with ZipFile(existing_zipfile, "r") as existing_zip: + with ZipFile(output_path, "w") as new_zip: for item in existing_zip.infolist(): data = existing_zip.read(item.filename) new_zip.writestr(item, data) - + for path in additional_paths: if path.is_file(): new_zip.write(path, path.name) elif path.is_dir(): - for file_path in path.rglob('*'): + for file_path in path.rglob("*"): if file_path.is_file(): # When adding a directory, only the contents are appended to the zip # e.g. adding "build/" will not create paths beginning with "build/" in the zip diff --git a/api/src/deploy/lambda_deploy.py b/api/src/deploy/lambda_deploy.py index fa11ed0..a100fc2 100644 --- a/api/src/deploy/lambda_deploy.py +++ b/api/src/deploy/lambda_deploy.py @@ -1,13 +1,16 @@ from __future__ import annotations import json -from pathlib import Path +from typing import TYPE_CHECKING import boto3 from botocore.exceptions import ClientError from src import settings +if TYPE_CHECKING: + from pathlib import Path + LAMBDA_RUNTIMES = ["python3.12", "python3.11", "python3.10", "python3.9", "python3.8"] @@ -27,7 +30,7 @@ def deploy_python_lambda_function( handler: str = "lambda_function.lambda_handler", ): # Initialize the Lambda client - lambda_client = boto3.client("lambda", region_name=settings.AWS_DEFAULT_REGION) + lambda_client = boto3.client("lambda", region_name=settings.AWS_DEFAULT_REGION) # type: ignore lambda_runtime = translate_python_version_to_lambda_runtime(python_version) try: @@ -37,21 +40,21 @@ def deploy_python_lambda_function( try: # Try to get the function configuration - lambda_client.get_function(FunctionName=function_name) + lambda_client.get_function(FunctionName=function_name) # type: ignore # If we reach here, the function exists, so we update it - response = lambda_client.update_function_code( + response = lambda_client.update_function_code( # type: ignore FunctionName=function_name, ZipFile=bytes_content ) print(f"Updated existing Lambda function: {function_name}") except ClientError as e: - if e.response["Error"]["Code"] == "ResourceNotFoundException": + if e.response["Error"]["Code"] == "ResourceNotFoundException": # type: ignore # The function doesn't exist, so we create it - response = lambda_client.create_function( + response = lambda_client.create_function( # type: ignore FunctionName=function_name, Runtime=lambda_runtime, - Role=settings.LAMBDA_ROLE_ARN, + Role=settings.LAMBDA_ROLE_ARN, # type: ignore Handler=handler, Code=dict(ZipFile=bytes_content), ) diff --git a/api/src/deploy/routes.py b/api/src/deploy/routes.py index efad7da..78d17f3 100644 --- a/api/src/deploy/routes.py +++ b/api/src/deploy/routes.py @@ -3,7 +3,7 @@ import json import tempfile from pathlib import Path -from typing import Annotated +from typing import Annotated # type: ignore from fastapi import APIRouter, File, Form, HTTPException, UploadFile from fastapi.responses import JSONResponse @@ -27,45 +27,59 @@ class DeploymentConfig(BaseModel): UPLOADED_BUNDLE_FILENAME = "uploaded_bundle.zip" + @router.post("/deploy/") async def deploy_zip( - file: Annotated[UploadFile, File()], - json_data: Annotated[str, Form()] + file: Annotated[UploadFile, File()], # type: ignore + json_data: Annotated[str, Form()], # type: ignore ): try: - deployment_data = json.loads(json_data) + deployment_data = json.loads(json_data) # type: ignore deployments: list[DeploymentConfig] = [ DeploymentConfig( name=name, path=deployment["reference"], python_version=deployment["python_version"], - requirements=deployment["dependencies"] + requirements=deployment["dependencies"], ) for name, deployment in deployment_data.items() ] except Exception: - raise HTTPException(status_code=422, detail="Couldn't process deployments data.") + raise HTTPException( + status_code=422, detail="Couldn't process deployments data." + ) with tempfile.TemporaryDirectory() as tmp_dir: tmp_dir = Path(tmp_dir) zipfile_path = tmp_dir / UPLOADED_BUNDLE_FILENAME try: - write_to_zipfile(await file.read(), output_path=zipfile_path) + write_to_zipfile(await file.read(), output_path=zipfile_path) # type: ignore except ValueError as err: return JSONResponse(status_code=400, content={"error": str(err)}) for deployment in deployments: # TODO: validate deployment name build_path = tmp_dir / deployment.name build_path.mkdir(parents=True, exist_ok=True) - build_lambda_handler(symbol_path=deployment.path, output_path=build_path / "lambda_function.py") - install_deps_to_dir(dependencies=deployment.requirements, python_version=deployment.python_version, output_dir=build_path) + build_lambda_handler( + symbol_path=deployment.path, + output_path=build_path / "lambda_function.py", + ) + install_deps_to_dir( + dependencies=deployment.requirements, + python_version=deployment.python_version, + output_dir=build_path, + ) deployment_package_path = tmp_dir / f"{deployment.name}.zip" - write_extended_zipfile(existing_zipfile=zipfile_path, additional_paths=[build_path], output_path=deployment_package_path) + write_extended_zipfile( + existing_zipfile=zipfile_path, + additional_paths=[build_path], + output_path=deployment_package_path, + ) deploy_python_lambda_function( function_name=deployment.name, zip_file=deployment_package_path, - python_version=deployment.python_version + python_version=deployment.python_version, ) # TODO: after each deployment is done, update record in RDS return {"status": "OK"} diff --git a/api/src/settings.py b/api/src/settings.py index 40b878e..4f86842 100644 --- a/api/src/settings.py +++ b/api/src/settings.py @@ -1,3 +1,4 @@ +# type: ignore from __future__ import annotations from environs import Env diff --git a/api/src/transform/build_lambda_handler.py b/api/src/transform/build_lambda_handler.py index 7324876..e2756db 100644 --- a/api/src/transform/build_lambda_handler.py +++ b/api/src/transform/build_lambda_handler.py @@ -18,7 +18,7 @@ def build_lambda_handler(symbol_path: str, output_path: Path): try: mod_path, target_symbol = symbol_path.split(":") - except: + except ValueError: # not enough values to unpack/too many values to unpack raise ValueError( f"Could not resolve module path and target symbol from: '{symbol_path}'" ) diff --git a/gauge/__init__.py b/gauge/__init__.py new file mode 100644 index 0000000..09deb03 --- /dev/null +++ b/gauge/__init__.py @@ -0,0 +1,5 @@ +from __future__ import annotations + +from gauge.sdk.main import endpoint + +__all__ = ["endpoint"] diff --git a/cli/__init__.py b/gauge/cli/__init__.py similarity index 100% rename from cli/__init__.py rename to gauge/cli/__init__.py diff --git a/cli/console.py b/gauge/cli/console.py similarity index 91% rename from cli/console.py rename to gauge/cli/console.py index bb15197..230cfc0 100644 --- a/cli/console.py +++ b/gauge/cli/console.py @@ -1,11 +1,14 @@ from __future__ import annotations -from collections.abc import Generator from contextlib import contextmanager from datetime import datetime +from typing import TYPE_CHECKING from rich.console import Console +if TYPE_CHECKING: + from collections.abc import Generator + @contextmanager def log_task(start_message: str, end_message: str) -> Generator[None, None, None]: diff --git a/cli/deploy.py b/gauge/cli/deploy.py similarity index 94% rename from cli/deploy.py rename to gauge/cli/deploy.py index ac7ddcb..88dd5f0 100644 --- a/cli/deploy.py +++ b/gauge/cli/deploy.py @@ -15,20 +15,20 @@ import requests from rich.console import Console -from cli.console import log_error, log_task +from gauge.cli.console import log_error, log_task API_URL = os.environ.get("GAUGE_API_URL", "http://localhost:8000") -class InnerDict(TypedDict): - module: str +class DeployType(TypedDict): + module: str | None reference: str function: str python_version: str dependencies: list[str] -DeployConfigType = Dict[str, InnerDict] +DeployConfigType = Dict[str, DeployType] class DeployHandler: @@ -65,9 +65,9 @@ def bundle(self, temp_dir: str) -> Path: def upload(self, zip_path: Path, deployments: DeployConfigType) -> None: print(deployments) - gauge_client_id = os.environ.get( - "GAUGE_CLIENT_ID" - ) or input("Input your GAUGE_CLIENT_ID: ") + gauge_client_id = os.environ.get("GAUGE_CLIENT_ID") or input( + "Input your GAUGE_CLIENT_ID: " + ) with log_task( start_message="Uploading bundle...", end_message="Bundle uploaded" ): @@ -115,7 +115,7 @@ def register_deployments(self) -> DeployConfigType: for name, obj in inspect.getmembers(module): if inspect.isfunction(obj) and hasattr(obj, "_gauge_register"): try: - name, config = obj._gauge_register() + name, config = obj._gauge_register() # type: ignore if name in results: raise ValueError(f"Duplicate endpoint {name}") results[name] = { @@ -129,7 +129,7 @@ def register_deployments(self) -> DeployConfigType: break else: print(f"Couldn't load module from {file_path}") - return results + return results # type: ignore def deploy(self): console = Console() diff --git a/cli/main.py b/gauge/cli/main.py similarity index 96% rename from cli/main.py rename to gauge/cli/main.py index c0cc69f..b0f126c 100644 --- a/cli/main.py +++ b/gauge/cli/main.py @@ -2,7 +2,7 @@ import argparse -from cli.deploy import DeployHandler +from gauge.cli.deploy import DeployHandler def deploy(file_path_str: str) -> None: diff --git a/cli/tests/__init__.py b/gauge/cli/tests/__init__.py similarity index 100% rename from cli/tests/__init__.py rename to gauge/cli/tests/__init__.py diff --git a/cli/tests/sample.py b/gauge/cli/tests/sample.py similarity index 83% rename from cli/tests/sample.py rename to gauge/cli/tests/sample.py index af01e00..caaa24a 100644 --- a/cli/tests/sample.py +++ b/gauge/cli/tests/sample.py @@ -1,6 +1,6 @@ from __future__ import annotations -from sdk.main import endpoint +from gauge.sdk.main import endpoint @endpoint(name="name1") diff --git a/cli/tests/test_deploy.py b/gauge/cli/tests/test_deploy.py similarity index 98% rename from cli/tests/test_deploy.py rename to gauge/cli/tests/test_deploy.py index f95d1e0..48fa821 100644 --- a/cli/tests/test_deploy.py +++ b/gauge/cli/tests/test_deploy.py @@ -5,7 +5,7 @@ import pytest -from cli.deploy import DeployHandler +from gauge.cli.deploy import DeployHandler @pytest.fixture diff --git a/cli/tests/test_main.py b/gauge/cli/tests/test_main.py similarity index 95% rename from cli/tests/test_main.py rename to gauge/cli/tests/test_main.py index c7d5dcb..05d6569 100644 --- a/cli/tests/test_main.py +++ b/gauge/cli/tests/test_main.py @@ -4,7 +4,7 @@ import pytest -from cli.main import create_parser +from gauge.cli.main import create_parser if TYPE_CHECKING: from pytest import CaptureFixture diff --git a/sdk/__init__.py b/gauge/sdk/__init__.py similarity index 100% rename from sdk/__init__.py rename to gauge/sdk/__init__.py diff --git a/sdk/config.py b/gauge/sdk/config.py similarity index 100% rename from sdk/config.py rename to gauge/sdk/config.py diff --git a/sdk/main.py b/gauge/sdk/main.py similarity index 100% rename from sdk/main.py rename to gauge/sdk/main.py diff --git a/sdk/tests/__init__.py b/gauge/sdk/tests/__init__.py similarity index 100% rename from sdk/tests/__init__.py rename to gauge/sdk/tests/__init__.py diff --git a/sdk/tests/test_sdk.py b/gauge/sdk/tests/test_sdk.py similarity index 96% rename from sdk/tests/test_sdk.py rename to gauge/sdk/tests/test_sdk.py index 507fe09..54f3dd9 100644 --- a/sdk/tests/test_sdk.py +++ b/gauge/sdk/tests/test_sdk.py @@ -1,6 +1,6 @@ from __future__ import annotations -from sdk.main import endpoint +from gauge import endpoint def test_default_endpoint_decorator(): diff --git a/pyproject.toml b/pyproject.toml index 5654198..cf420ab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "gauge-serverless" -version = "0.0.2" +version = "0.0.5" authors = [ { name="Caelean Barnes", email="caeleanb@gmail.com" }, { name="Evan Doyle", email="evanmdoyle@gmail.com" }, @@ -35,6 +35,41 @@ dependencies = [ ] keywords = ['python', 'lambda', 'aws', 'serverless', 'fastapi'] +[project.optional-dependencies] +dev = [ + # Core deps (pinned) + "pyyaml==6.0.1", + "pydantic==2.8.2; python_version > '3.7'", + "rich==13.7.1", + "fastapi==0.111.1", + "boto3-stubs-lite==1.34.145", + "boto3-stubs==1.34.145", + "tach==0.8.1", + # Setup + "pip==24.0", + # Code Quality + "pyright==1.1.372", + "ruff==0.5.2", + # Build/Release + "setuptools==69.5.1; python_version > '3.7'", + "twine==5.1.1; python_version > '3.7'", + "build==1.2.1; python_version > '3.7'", + # Tests + "pytest==8.2.2; python_version > '3.7'", + "pytest-mock==3.14.0; python_version > '3.7'", + "coverage==7.6.0; python_version > '3.7'", + # python version 3.7 pinned dependencies + "botocore==1.33.13; python_version == '3.7'", + "boto3-stubs-lite==1.33.13; python_version == '3.7'", + "boto3-stubs==1.33.13; python_version == '3.7'", + "pydantic==2.5.3; python_version == '3.7'", + "setuptools==47.1.0; python_version == '3.7'", + "twine==4.0.2; python_version == '3.7'", + "build==1.1.1; python_version == '3.7'", + "pytest==7.4.4; python_version == '3.7'", + "pytest-mock==3.11.1; python_version == '3.7'", + "coverage==7.2.7; python_version == '3.7'", +] [project.urls] @@ -43,6 +78,7 @@ Issues = "https://github.com/gauge-sh/tach/gauge-serverless" [tool.ruff] target-version = "py38" +exclude = ["**/__pycache__", ".venv", "api/src/transform/template.py"] lint.extend-select = ["I", "TCH", "UP"] [tool.ruff.lint.isort] @@ -60,18 +96,18 @@ runtime-evaluated-decorators = [ exempt-modules = ["typing", "typing_extensions"] [tool.pyright] -exclude = ["**/__pycache__", ".venv"] +exclude = ["**/__pycache__", ".venv", "api/src/transform/template.py"] strict = ["."] pythonVersion = "3.8" [project.scripts] -gauge = "cli.main:main" +gauge = "gauge.cli.main:main" [build-system] requires = ["setuptools", "wheel"] build-backend = "setuptools.build_meta" -[tool.setuptools] -packages = ["cli", "sdk"] -#include = ["README.md", "LICENSE"] \ No newline at end of file +[tool.setuptools.packages.find] +where = ["."] +include = ["gauge*"] diff --git a/tach.yml b/tach.yml index d80cd7d..d7c1be7 100644 --- a/tach.yml +++ b/tach.yml @@ -1,13 +1,20 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/gauge-sh/tach/v0.8.1/public/tach-yml-schema.json modules: - - path: cli + - path: api depends_on: [] - - path: sdk + - path: gauge + depends_on: + - gauge.sdk + - path: gauge.cli + depends_on: [] + - path: gauge.sdk depends_on: [] exclude: - .*__pycache__ - .*egg-info + - api/src/transform/template.py - docs - - tests + - gauge/cli/tests + - gauge/sdk/tests - venv source_root: .