Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add Valkey support #106

Open
wants to merge 11 commits into
base: master
Choose a base branch
from
7 changes: 7 additions & 0 deletions docs/docs/configs.md
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,13 @@ _Requires [redis](https://redis-py.readthedocs.io/en/stable/) package._

We use it to create `redis` connection

---
### [VALKEY](https://pantherpy.github.io/valkey)
> <b>Type:</b> `dict` (<b>Default:</b> `{}`)

_Requires [valkey](https://valkey-py.readthedocs.io/en/stable/) package._

We use it to create `valkey` connection

---
### [TIMEZONE](https://pantherpy.github.io/timezone)
Expand Down
4 changes: 2 additions & 2 deletions docs/docs/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
- Include Simple **File-Base** Database ([PantherDB](https://pypi.org/project/pantherdb/))
- Built-in Document-oriented Databases **ODM** (**MongoDB**, PantherDB)
- Built-in **Websocket** Support
- Built-in API **Caching** System (In Memory, **Redis**)
- Built-in API **Caching** System (In Memory, **Redis**, **Valkey**)
- Built-in **Authentication** Classes
- Built-in **Permission** Classes
- Built-in Visual API **Monitoring** (In Terminal)
Expand Down Expand Up @@ -58,7 +58,7 @@
<summary>3. <b>Install Panther</b></summary>
- ⬇ Normal Installation
<pre><b>$ pip install panther</b></pre>
- ⬇ Include full requirements (MongoDB, JWTAuth, Ruff, Redis, Websockets, Cryptography, bpython)
- ⬇ Include full requirements (MongoDB, JWTAuth, Ruff, Redis, Valkey, Websockets, Cryptography, bpython)
<pre>$ pip install panther[full]</pre>
</details>

Expand Down
3 changes: 3 additions & 0 deletions docs/docs/release_notes.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
### 4.3.3
- Support 'valkey' as a cache backend

### 4.3.2
- Support `Python 3.13`

Expand Down
42 changes: 42 additions & 0 deletions docs/docs/valkey.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
### Structure

```python
VALKEY = {
'class': 'address of the class',
'arg1': 'value1',
...
}
```

### ValkeyConnection

```python
VALKEY = {
'class': 'panther.db.connections.ValkeyConnection',
'host': ..., # default is localhost
'port': ..., # default is 6379
'db': ..., # default is 0
'websocket_db': ..., # default is 0
...
}
```

#### Notes
- The arguments are same as `valkey.Valkey.__init__()` except `websocket_db`
- You can specify which `db` is for your `websocket` connections


### How it works?

- Panther creates an async valkey connection depends on `VALKEY` block you defined in `configs`

- You can access to it from `from panther.db.connections import valkey`

- Example:
```python
from panther.db.connections import valkey

await valkey.set('name', 'Ali')
result = await valkey.get('name')
print(result)
```
24 changes: 20 additions & 4 deletions panther/_load_configs.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from panther.base_websocket import WebsocketConnections
from panther.cli.utils import import_error
from panther.configs import JWTConfig, config
from panther.db.connections import redis
from panther.db.connections import cache
from panther.db.queries.mongodb_queries import BaseMongoDBQuery
from panther.db.queries.pantherdb_queries import BasePantherDBQuery
from panther.exceptions import PantherError
Expand All @@ -20,6 +20,7 @@
__all__ = (
'load_configs_module',
'load_redis',
'load_valkey',
'load_startup',
'load_shutdown',
'load_timezone',
Expand Down Expand Up @@ -70,6 +71,21 @@ def load_redis(_configs: dict, /) -> None:
redis_class(**args, init=True)


def load_valkey(_configs: dict, /) -> None:
if valkey_config := _configs.get('VALKEY'):
# Check valkey module installation
try:
from valkey.asyncio import Valkey
except ImportError as e:
raise import_error(e, package='valkey')
valkey_class_path = valkey_config.get('class', 'panther.db.connections.ValkeyConnection')
valkey_class = import_class(valkey_class_path)
# We have to create another dict then pop the 'class' else we can't pass the tests
args = valkey_config.copy()
args.pop('class', None)
valkey_class(**args, init=True)


def load_startup(_configs: dict, /) -> None:
if startup := _configs.get('STARTUP'):
config.STARTUP = import_class(startup)
Expand Down Expand Up @@ -254,16 +270,16 @@ def load_urls(_configs: dict, /, urls: dict | None) -> None:


def load_websocket_connections():
"""Should be after `load_redis()`"""
"""Should be after `load_redis()/load_valkey()`"""
if config.HAS_WS:
# Check `websockets`
try:
import websockets
except ImportError as e:
raise import_error(e, package='websockets')

# Use the redis pubsub if `redis.is_connected`, else use the `multiprocessing.Manager`
pubsub_connection = redis.create_connection_for_websocket() if redis.is_connected else Manager()
# Use the redis/valkey pubsub if `cache.is_connected`, else use the `multiprocessing.Manager`
pubsub_connection = cache.create_connection_for_websocket() if cache.is_connected else Manager()
config.WEBSOCKET_CONNECTIONS = WebsocketConnections(pubsub_connection=pubsub_connection)


Expand Down
12 changes: 6 additions & 6 deletions panther/authentications.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
from panther.base_websocket import Websocket
from panther.cli.utils import import_error
from panther.configs import config
from panther.db.connections import redis
from panther.db.connections import cache
from panther.db.models import BaseUser, Model
from panther.exceptions import AuthenticationAPIError
from panther.request import Request
Expand Down Expand Up @@ -67,7 +67,7 @@ async def authentication(cls, request: Request | Websocket) -> Model:
msg = 'Authorization keyword is not valid'
raise cls.exception(msg) from None

if redis.is_connected and await cls._check_in_cache(token=token):
if cache.is_connected and await cls._check_in_cache(token=token):
msg = 'User logged out'
raise cls.exception(msg) from None

Expand Down Expand Up @@ -134,22 +134,22 @@ def login(cls, user_id: str) -> dict:
@classmethod
async def logout(cls, raw_token: str) -> None:
*_, token = raw_token.split()
if redis.is_connected:
if cache.is_connected:
payload = cls.decode_jwt(token=token)
remaining_exp_time = payload['exp'] - time.time()
await cls._set_in_cache(token=token, exp=int(remaining_exp_time))
else:
logger.error('`redis` middleware is required for `logout()`')
logger.error('`redis/valkey` middleware is required for `logout()`')

@classmethod
async def _set_in_cache(cls, token: str, exp: int) -> None:
key = generate_hash_value_from_string(token)
await redis.set(key, b'', ex=exp)
await cache.set(key, b'', ex=exp)

@classmethod
async def _check_in_cache(cls, token: str) -> bool:
key = generate_hash_value_from_string(token)
return bool(await redis.exists(key))
return bool(await cache.exists(key))

@staticmethod
def exception(message: str | JWTError | UnicodeEncodeError, /) -> type[AuthenticationAPIError]:
Expand Down
13 changes: 7 additions & 6 deletions panther/base_websocket.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,14 @@
from panther import status
from panther.base_request import BaseRequest
from panther.configs import config
from panther.db.connections import redis
from panther.db.connections import cache
from panther.exceptions import AuthenticationAPIError, InvalidPathVariableAPIError
from panther.monitoring import Monitoring
from panther.utils import Singleton, ULID

if TYPE_CHECKING:
from redis.asyncio import Redis
from valkey.asyncio import Valkey

logger = logging.getLogger('panther')

Expand All @@ -36,7 +37,7 @@ def publish(self, msg):


class WebsocketConnections(Singleton):
def __init__(self, pubsub_connection: Redis | SyncManager):
def __init__(self, pubsub_connection: Redis | Valkey | SyncManager):
self.connections = {}
self.connections_count = 0
self.pubsub_connection = pubsub_connection
Expand All @@ -46,7 +47,7 @@ def __init__(self, pubsub_connection: Redis | SyncManager):

async def __call__(self):
if isinstance(self.pubsub_connection, SyncManager):
# We don't have redis connection, so use the `multiprocessing.Manager`
# We don't have redis/valkey connection, so use the `multiprocessing.Manager`
self.pubsub: PubSub
queue = self.pubsub.subscribe()
logger.info("Subscribed to 'websocket_connections' queue")
Expand All @@ -62,7 +63,7 @@ async def __call__(self):
queue.put(None)
break
else:
# We have a redis connection, so use it for pubsub
# We have a redis/valkey connection, so use it for pubsub
self.pubsub = self.pubsub_connection.pubsub()
await self.pubsub.subscribe('websocket_connections')
logger.info("Subscribed to 'websocket_connections' channel")
Expand Down Expand Up @@ -103,8 +104,8 @@ async def _handle_received_message(self, received_message):
async def publish(self, connection_id: str, action: Literal['send', 'close'], data: any):
publish_data = {'connection_id': connection_id, 'action': action, 'data': data}

if redis.is_connected:
await redis.publish('websocket_connections', json.dumps(publish_data))
if cache.is_connected:
await cache.publish('websocket_connections', json.dumps(publish_data))
else:
self.pubsub.publish(publish_data)

Expand Down
40 changes: 20 additions & 20 deletions panther/caching.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import orjson as json

from panther.configs import config
from panther.db.connections import redis
from panther.db.connections import cache
from panther.request import Request
from panther.response import Response, ResponseDataTypes
from panther.throttling import throttling_storage
Expand Down Expand Up @@ -38,14 +38,14 @@ def throttling_cache_key(request: Request, duration: timedelta) -> str:

async def get_response_from_cache(*, request: Request, cache_exp_time: timedelta) -> CachedResponse | None:
"""
If redis.is_connected:
Get Cached Data From Redis
If redis.is_connected or valkey.is_connected:
Get Cached Data From Redis or Valkey
else:
Get Cached Data From Memory
"""
if redis.is_connected:
if cache.is_connected:
key = api_cache_key(request=request)
data = (await redis.get(key) or b'{}').decode()
data = (await cache.get(key) or b'{}').decode()
if cached_value := json.loads(data):
return CachedResponse(*cached_value)

Expand All @@ -57,15 +57,15 @@ async def get_response_from_cache(*, request: Request, cache_exp_time: timedelta

async def set_response_in_cache(*, request: Request, response: Response, cache_exp_time: timedelta | int) -> None:
"""
If redis.is_connected:
Cache The Data In Redis
If redis.is_connected or valkey.is_connected:
Cache The Data In Redis or Valkey
else:
Cache The Data In Memory
"""

cache_data: tuple[ResponseDataTypes, int] = (response.data, response.status_code)

if redis.is_connected:
if cache.is_connected:
key = api_cache_key(request=request)

cache_exp_time = cache_exp_time or config.DEFAULT_CACHE_EXP
Expand All @@ -77,32 +77,32 @@ async def set_response_in_cache(*, request: Request, response: Response, cache_e

if cache_exp_time is None:
logger.warning(
'your response are going to cache in redis forever '
'your response are going to cache in server forever '
'** set DEFAULT_CACHE_EXP in `configs` or set the `cache_exp_time` in `@API.get()` to prevent this **'
)
await redis.set(key, cache_data)
await cache.set(key, cache_data)
else:
await redis.set(key, cache_data, ex=cache_exp_time)
await cache.set(key, cache_data, ex=cache_exp_time)

else:
key = api_cache_key(request=request, cache_exp_time=cache_exp_time)
caches[key] = cache_data

if cache_exp_time:
logger.info('`cache_exp_time` is not very accurate when `redis` is not connected.')
logger.info('`cache_exp_time` is not very accurate when `redis/valkey` is not connected.')


async def get_throttling_from_cache(request: Request, duration: timedelta) -> int:
"""
If redis.is_connected:
Get Cached Data From Redis
If redis.is_connected or valkey.is_connected:
Get Cached Data From Redis or Valkey
else:
Get Cached Data From Memory
"""
key = throttling_cache_key(request=request, duration=duration)

if redis.is_connected:
data = (await redis.get(key) or b'0').decode()
if cache.is_connected:
data = (await cache.get(key) or b'0').decode()
return json.loads(data)

else:
Expand All @@ -111,15 +111,15 @@ async def get_throttling_from_cache(request: Request, duration: timedelta) -> in

async def increment_throttling_in_cache(request: Request, duration: timedelta) -> None:
"""
If redis.is_connected:
Increment The Data In Redis
If redis.is_connected or valkey.is_connected:
Increment The Data In Redis or Valkey
else:
Increment The Data In Memory
"""
key = throttling_cache_key(request=request, duration=duration)

if redis.is_connected:
await redis.incrby(key, amount=1)
if cache.is_connected:
await cache.incrby(key, amount=1)

else:
throttling_storage[key] += 1
11 changes: 10 additions & 1 deletion panther/cli/create_command.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
AUTO_REFORMAT_PART,
DATABASE_PANTHERDB_PART,
DATABASE_MONGODB_PART,
USER_MODEL_PART, REDIS_PART,
USER_MODEL_PART, REDIS_PART, VALKEY_PART
)
from panther.cli.utils import cli_error

Expand All @@ -35,6 +35,7 @@ def __init__(self):
self.database = '0'
self.database_encryption = False
self.redis = False
self.valkey = False
self.authentication = False
self.monitoring = True
self.log_queries = True
Expand Down Expand Up @@ -76,6 +77,11 @@ def __init__(self):
'message': 'Do You Want To Use Redis (Required `redis`)',
'is_boolean': True,
},
{
'field': 'valkey',
'message': 'Do You Want To Use Valkey (Required `valkey`)',
'is_boolean': True,
},
{
'field': 'authentication',
'message': 'Do You Want To Use JWT Authentication (Required `python-jose`)',
Expand Down Expand Up @@ -147,6 +153,8 @@ def _create_file(self, *, path: str, data: str):
database_encryption = 'True' if self.database_encryption else 'False'
database_extension = 'pdb' if self.database_encryption else 'json'
redis_part = REDIS_PART if self.redis else ''
valkey_part = VALKEY_PART if self.valkey else ''

if self.database == '0':
database_part = DATABASE_PANTHERDB_PART
elif self.database == '1':
Expand All @@ -163,6 +171,7 @@ def _create_file(self, *, path: str, data: str):
data = data.replace('{PANTHERDB_ENCRYPTION}', database_encryption) # Should be after `DATABASE`
data = data.replace('{PANTHERDB_EXTENSION}', database_extension) # Should be after `DATABASE`
data = data.replace('{REDIS}', redis_part)
data = data.replace('{VALKEY}', valkey_part)

data = data.replace('{PROJECT_NAME}', self.project_name.lower())
data = data.replace('{PANTHER_VERSION}', version())
Expand Down
Loading
Loading