Skip to content

Commit

Permalink
WIP
Browse files Browse the repository at this point in the history
  • Loading branch information
@itssimon
itssimon committed Feb 26, 2024
1 parent 3fd00ef commit d3c270d
Show file tree
Hide file tree
Showing 23 changed files with 226 additions and 1,301 deletions.
59 changes: 9 additions & 50 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@
</picture>
</p>

<p align="center"><b>Your refreshingly simple REST API companion.</b></p>
<p align="center"><b>API monitoring made easy.</b></p>

<p align="center"><i>Apitally is a simple and affordable API monitoring and API key management solution with a focus on data privacy. It is easy to set up and use for new and existing API projects using Python or Node.js.</i></p>
<p align="center"><i>Apitally is a simple and affordable API monitoring solution with a focus on data privacy. It is easy to set up and use for new and existing API projects using Python or Node.js.</i></p>

<p align="center">🔗 <b><a href="https://apitally.io" target="_blank">apitally.io</a></b></p>

Expand Down Expand Up @@ -38,10 +38,7 @@ the 📚 [documentation](https://docs.apitally.io).

- Middleware for different frameworks to capture metadata about API endpoints,
requests and responses (no sensitive data is captured)
- Non-blocking clients that aggregate and send captured data to Apitally and
optionally synchronize API key hashes in 1 minute intervals
- Functions to easily secure endpoints with API key authentication and
permission checks
- Non-blocking clients that aggregate and send captured data to Apitally

## Install

Expand All @@ -52,8 +49,7 @@ example:
pip install apitally[fastapi]
```

The available extras are: `fastapi`, `starlette`, `flask`, `django_ninja` and
`django_rest_framework`.
The available extras are: `fastapi`, `starlette`, `flask` and `django`.

## Usage

Expand All @@ -78,24 +74,6 @@ app.add_middleware(
)
```

### Starlette

This is an example of how to add the Apitally middleware to a Starlette
application. For further instructions, see our
[setup guide for Starlette](https://docs.apitally.io/frameworks/starlette).

```python
from starlette.applications import Starlette
from apitally.starlette import ApitallyMiddleware

app = Starlette(routes=[...])
app.add_middleware(
ApitallyMiddleware,
client_id="your-client-id",
env="dev", # or "prod" etc.
)
```

### Flask

This is an example of how to add the Apitally middleware to a Flask application.
Expand All @@ -114,36 +92,17 @@ app.wsgi_app = ApitallyMiddleware(
)
```

### Django Ninja

This is an example of how to add the Apitally middleware to a Django Ninja
application. For further instructions, see our
[setup guide for Django Ninja](https://docs.apitally.io/frameworks/django-ninja).

In your Django `settings.py` file:

```python
MIDDLEWARE = [
"apitally.django_ninja.ApitallyMiddleware",
# Other middleware ...
]
APITALLY_MIDDLEWARE = {
"client_id": "your-client-id",
"env": "dev", # or "prod" etc.
}
```

### Django REST Framework
### Django

This is an example of how to add the Apitally middleware to a Django REST
Framework application. For further instructions, see our
[setup guide for Django REST Framework](https://docs.apitally.io/frameworks/django-rest-framework).
This is an example of how to add the Apitally middleware to a Django Ninja or
Django REST Framework application. For further instructions, see our
[setup guide for Django](https://docs.apitally.io/frameworks/django).

In your Django `settings.py` file:

```python
MIDDLEWARE = [
"apitally.django_rest_framework.ApitallyMiddleware",
"apitally.django.ApitallyMiddleware",
# Other middleware ...
]
APITALLY_MIDDLEWARE = {
Expand Down
47 changes: 4 additions & 43 deletions apitally/client/asyncio.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,20 +2,14 @@

import asyncio
import logging
import sys
import time
from functools import partial
from typing import Any, Dict, Optional, Tuple, Type
from typing import Any, Dict, Optional, Tuple

import backoff
import httpx

from apitally.client.base import (
MAX_QUEUE_TIME,
REQUEST_TIMEOUT,
ApitallyClientBase,
ApitallyKeyCacheBase,
)
from apitally.client.base import MAX_QUEUE_TIME, REQUEST_TIMEOUT, ApitallyClientBase
from apitally.client.logging import get_logger


Expand All @@ -31,19 +25,8 @@


class ApitallyClient(ApitallyClientBase):
def __init__(
self,
client_id: str,
env: str,
sync_api_keys: bool = False,
key_cache_class: Optional[Type[ApitallyKeyCacheBase]] = None,
) -> None:
super().__init__(
client_id=client_id,
env=env,
sync_api_keys=sync_api_keys,
key_cache_class=key_cache_class,
)
def __init__(self, client_id: str, env: str) -> None:
super().__init__(client_id=client_id, env=env)
self._stop_sync_loop = False
self._sync_loop_task: Optional[asyncio.Task[Any]] = None
self._requests_data_queue: asyncio.Queue[Tuple[float, Dict[str, Any]]] = asyncio.Queue()
Expand All @@ -62,8 +45,6 @@ async def _run_sync_loop(self) -> None:
time_start = time.perf_counter()
async with self.get_http_client() as client:
tasks = [self.send_requests_data(client)]
if self.sync_api_keys:
tasks.append(self.get_keys(client))
if not self._app_info_sent and not first_iteration:
tasks.append(self.send_app_info(client))
await asyncio.gather(*tasks)
Expand Down Expand Up @@ -113,19 +94,6 @@ async def send_requests_data(self, client: httpx.AsyncClient) -> None:
for item in failed_items:
self._requests_data_queue.put_nowait(item)

async def get_keys(self, client: httpx.AsyncClient) -> None:
if response_data := await self._get_keys(client): # Response data can be None if backoff gives up
self.handle_keys_response(response_data)
self._keys_updated_at = time.time()
elif self.key_registry.salt is None: # pragma: no cover
logger.critical("Initial Apitally API key sync failed")
# Exit because the application will not be able to authenticate requests
sys.exit(1)
elif (self._keys_updated_at is not None and time.time() - self._keys_updated_at > MAX_QUEUE_TIME) or (
self._keys_updated_at is None and time.time() - self._started_at > MAX_QUEUE_TIME
):
logger.error("Apitally API key sync has been failing for more than 1 hour")

@retry(raise_on_giveup=False)
async def _send_app_info(self, client: httpx.AsyncClient, payload: Dict[str, Any]) -> None:
logger.debug("Sending app info")
Expand All @@ -143,10 +111,3 @@ async def _send_requests_data(self, client: httpx.AsyncClient, payload: Dict[str
logger.debug("Sending requests data")
response = await client.post(url="/requests", json=payload)
response.raise_for_status()

@retry(raise_on_giveup=False)
async def _get_keys(self, client: httpx.AsyncClient) -> Dict[str, Any]:
logger.debug("Updating API keys")
response = await client.get(url="/keys")
response.raise_for_status()
return response.json()
121 changes: 3 additions & 118 deletions apitally/client/base.py
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,13 @@
from __future__ import annotations

import json
import os
import re
import threading
import time
from abc import ABC, abstractmethod
from collections import Counter
from dataclasses import dataclass, field
from datetime import datetime, timedelta
from hashlib import scrypt
from dataclasses import dataclass
from math import floor
from typing import Any, Dict, List, Optional, Tuple, Type, TypeVar, Union, cast
from typing import Any, Dict, List, Optional, Tuple, Type, TypeVar, cast
from uuid import UUID, uuid4

from apitally.client.logging import get_logger
Expand Down Expand Up @@ -41,13 +37,7 @@ def __new__(cls, *args, **kwargs) -> ApitallyClientBase:
cls._instance = super().__new__(cls)
return cls._instance

def __init__(
self,
client_id: str,
env: str,
sync_api_keys: bool = False,
key_cache_class: Optional[Type[ApitallyKeyCacheBase]] = None,
) -> None:
def __init__(self, client_id: str, env: str) -> None:
if hasattr(self, "client_id"):
raise RuntimeError("Apitally client is already initialized") # pragma: no cover
try:
Expand All @@ -59,23 +49,13 @@ def __init__(

self.client_id = client_id
self.env = env
self.sync_api_keys = sync_api_keys
self.instance_uuid = str(uuid4())
self.request_counter = RequestCounter()
self.validation_error_counter = ValidationErrorCounter()
self.key_registry = KeyRegistry()
self.key_cache = key_cache_class(client_id=client_id, env=env) if key_cache_class is not None else None

self._app_info_payload: Optional[Dict[str, Any]] = None
self._app_info_sent = False
self._started_at = time.time()
self._keys_updated_at: Optional[float] = None

if self.key_cache is not None and (key_data := self.key_cache.retrieve()):
try:
self.handle_keys_response(json.loads(key_data), cache=False)
except (json.JSONDecodeError, TypeError, KeyError): # pragma: no cover
logger.exception("Failed to load API keys from cache")

@classmethod
def get_instance(cls: Type[TApitallyClient]) -> TApitallyClient:
Expand Down Expand Up @@ -104,42 +84,13 @@ def get_info_payload(self, app_info: Dict[str, Any]) -> Dict[str, Any]:
def get_requests_payload(self) -> Dict[str, Any]:
requests = self.request_counter.get_and_reset_requests()
validation_errors = self.validation_error_counter.get_and_reset_validation_errors()
api_key_usage = self.key_registry.get_and_reset_usage_counts() if self.sync_api_keys else {}
return {
"instance_uuid": self.instance_uuid,
"message_uuid": str(uuid4()),
"requests": requests,
"validation_errors": validation_errors,
"api_key_usage": api_key_usage,
}

def handle_keys_response(self, response_data: Dict[str, Any], cache: bool = True) -> None:
self.key_registry.salt = response_data["salt"]
self.key_registry.update(response_data["keys"])

if cache and self.key_cache is not None:
self.key_cache.store(json.dumps(response_data, check_circular=False, allow_nan=False))


class ApitallyKeyCacheBase(ABC):
def __init__(self, client_id: str, env: str) -> None:
self.client_id = client_id
self.env = env

@property
def cache_key(self) -> str:
return f"apitally:keys:{self.client_id}:{self.env}"

@abstractmethod
def store(self, data: str) -> None:
"""Store the key data in cache as a JSON string."""
pass # pragma: no cover

@abstractmethod
def retrieve(self) -> str | bytes | bytearray | None:
"""Retrieve the stored key data from the cache as a JSON string."""
pass # pragma: no cover


@dataclass(frozen=True)
class RequestInfo:
Expand Down Expand Up @@ -267,69 +218,3 @@ def get_and_reset_validation_errors(self) -> List[Dict[str, Any]]:
)
self.error_counts.clear()
return data


@dataclass(frozen=True)
class KeyInfo:
key_id: int
api_key_id: int
name: str = ""
scopes: List[str] = field(default_factory=list)
expires_at: Optional[datetime] = None

@property
def is_expired(self) -> bool:
return self.expires_at is not None and self.expires_at < datetime.now()

def has_scopes(self, scopes: Union[List[str], str]) -> bool:
if isinstance(scopes, str):
scopes = [scopes]
if not isinstance(scopes, list):
raise ValueError("scopes must be a string or a list of strings")
return all(scope in self.scopes for scope in scopes)

@classmethod
def from_dict(cls, data: Dict[str, Any]) -> KeyInfo:
return cls(
key_id=data["key_id"],
api_key_id=data["api_key_id"],
name=data.get("name", ""),
scopes=data.get("scopes", []),
expires_at=(
datetime.now() + timedelta(seconds=data["expires_in_seconds"])
if data["expires_in_seconds"] is not None
else None
),
)


class KeyRegistry:
def __init__(self) -> None:
self.salt: Optional[str] = None
self.keys: Dict[str, KeyInfo] = {}
self.usage_counts: Counter[int] = Counter()
self._lock = threading.Lock()

def get(self, api_key: str) -> Optional[KeyInfo]:
hash = self.hash_api_key(api_key.strip())
with self._lock:
key = self.keys.get(hash)
if key is None or key.is_expired:
return None
self.usage_counts[key.api_key_id] += 1
return key

def hash_api_key(self, api_key: str) -> str:
if self.salt is None:
raise RuntimeError("Apitally API keys not initialized")
return scrypt(api_key.encode(), salt=bytes.fromhex(self.salt), n=256, r=4, p=1, dklen=32).hex()

def update(self, keys: Dict[str, Dict[str, Any]]) -> None:
with self._lock:
self.keys = {hash: KeyInfo.from_dict(data) for hash, data in keys.items()}

def get_and_reset_usage_counts(self) -> Dict[int, int]:
with self._lock:
data = dict(self.usage_counts)
self.usage_counts.clear()
return data
Loading

0 comments on commit d3c270d

Please sign in to comment.