diff --git a/docs/docs/configs.md b/docs/docs/configs.md index 194ff2e..1f1cf81 100644 --- a/docs/docs/configs.md +++ b/docs/docs/configs.md @@ -151,6 +151,13 @@ _Requires [redis](https://redis-py.readthedocs.io/en/stable/) package._ We use it to create `redis` connection +--- +### [VALKEY](https://pantherpy.github.io/valkey) +> Type: `dict` (Default: `{}`) + +_Requires [valkey](https://valkey-py.readthedocs.io/en/stable/) package._ + +We use it to create `valkey` connection --- ### [TIMEZONE](https://pantherpy.github.io/timezone) diff --git a/docs/docs/index.md b/docs/docs/index.md index e4f93d1..7a1e93e 100644 --- a/docs/docs/index.md +++ b/docs/docs/index.md @@ -18,7 +18,7 @@ - Include Simple **File-Base** Database ([PantherDB](https://pypi.org/project/pantherdb/)) - Built-in Document-oriented Databases **ODM** (**MongoDB**, PantherDB) - Built-in **Websocket** Support -- Built-in API **Caching** System (In Memory, **Redis**) +- Built-in API **Caching** System (In Memory, **Redis**, **Valkey**) - Built-in **Authentication** Classes - Built-in **Permission** Classes - Built-in Visual API **Monitoring** (In Terminal) @@ -58,7 +58,7 @@ 3. Install Panther - ⬇ Normal Installation 
$ pip install panther
- - ⬇ Include full requirements (MongoDB, JWTAuth, Ruff, Redis, Websockets, Cryptography, bpython) + - ⬇ Include full requirements (MongoDB, JWTAuth, Ruff, Redis, Valkey, Websockets, Cryptography, bpython) 
$ pip install panther[full]
diff --git a/docs/docs/release_notes.md b/docs/docs/release_notes.md index e6c0bac..61287fb 100644 --- a/docs/docs/release_notes.md +++ b/docs/docs/release_notes.md @@ -1,3 +1,6 @@ +### 4.3.3 +- Support 'valkey' as a cache backend + ### 4.3.2 - Support `Python 3.13` diff --git a/docs/docs/valkey.md b/docs/docs/valkey.md new file mode 100644 index 0000000..8365ac8 --- /dev/null +++ b/docs/docs/valkey.md @@ -0,0 +1,42 @@ +### Structure + +```python +VALKEY = { + 'class': 'address of the class', + 'arg1': 'value1', + ... +} +``` + +### ValkeyConnection + +```python +VALKEY = { + 'class': 'panther.db.connections.ValkeyConnection', + 'host': ..., # default is localhost + 'port': ..., # default is 6379 + 'db': ..., # default is 0 + 'websocket_db': ..., # default is 0 + ... +} +``` + +#### Notes +- The arguments are same as `valkey.Valkey.__init__()` except `websocket_db` +- You can specify which `db` is for your `websocket` connections + + +### How it works? + +- Panther creates an async valkey connection depends on `VALKEY` block you defined in `configs` + +- You can access to it from `from panther.db.connections import valkey` + +- Example: + ```python + from panther.db.connections import valkey + + await valkey.set('name', 'Ali') + result = await valkey.get('name') + print(result) + ``` diff --git a/panther/_load_configs.py b/panther/_load_configs.py index 763ff00..356bdc0 100644 --- a/panther/_load_configs.py +++ b/panther/_load_configs.py @@ -9,7 +9,7 @@ from panther.base_websocket import WebsocketConnections from panther.cli.utils import import_error from panther.configs import JWTConfig, config -from panther.db.connections import redis +from panther.db.connections import cache from panther.db.queries.mongodb_queries import BaseMongoDBQuery from panther.db.queries.pantherdb_queries import BasePantherDBQuery from panther.exceptions import PantherError @@ -20,6 +20,7 @@ __all__ = ( 'load_configs_module', 'load_redis', + 'load_valkey', 'load_startup', 'load_shutdown', 'load_timezone', @@ -70,6 +71,21 @@ def load_redis(_configs: dict, /) -> None: redis_class(**args, init=True) +def load_valkey(_configs: dict, /) -> None: + if valkey_config := _configs.get('VALKEY'): + # Check valkey module installation + try: + from valkey.asyncio import Valkey + except ImportError as e: + raise import_error(e, package='valkey') + valkey_class_path = valkey_config.get('class', 'panther.db.connections.ValkeyConnection') + valkey_class = import_class(valkey_class_path) + # We have to create another dict then pop the 'class' else we can't pass the tests + args = valkey_config.copy() + args.pop('class', None) + valkey_class(**args, init=True) + + def load_startup(_configs: dict, /) -> None: if startup := _configs.get('STARTUP'): config.STARTUP = import_class(startup) @@ -254,7 +270,7 @@ def load_urls(_configs: dict, /, urls: dict | None) -> None: def load_websocket_connections(): - """Should be after `load_redis()`""" + """Should be after `load_redis()/load_valkey()`""" if config.HAS_WS: # Check `websockets` try: @@ -262,8 +278,8 @@ def load_websocket_connections(): except ImportError as e: raise import_error(e, package='websockets') - # Use the redis pubsub if `redis.is_connected`, else use the `multiprocessing.Manager` - pubsub_connection = redis.create_connection_for_websocket() if redis.is_connected else Manager() + # Use the redis/valkey pubsub if `cache.is_connected`, else use the `multiprocessing.Manager` + pubsub_connection = cache.create_connection_for_websocket() if cache.is_connected else Manager() config.WEBSOCKET_CONNECTIONS = WebsocketConnections(pubsub_connection=pubsub_connection) diff --git a/panther/authentications.py b/panther/authentications.py index 27454ea..dccb820 100644 --- a/panther/authentications.py +++ b/panther/authentications.py @@ -7,7 +7,7 @@ from panther.base_websocket import Websocket from panther.cli.utils import import_error from panther.configs import config -from panther.db.connections import redis +from panther.db.connections import cache from panther.db.models import BaseUser, Model from panther.exceptions import AuthenticationAPIError from panther.request import Request @@ -67,7 +67,7 @@ async def authentication(cls, request: Request | Websocket) -> Model: msg = 'Authorization keyword is not valid' raise cls.exception(msg) from None - if redis.is_connected and await cls._check_in_cache(token=token): + if cache.is_connected and await cls._check_in_cache(token=token): msg = 'User logged out' raise cls.exception(msg) from None @@ -134,22 +134,22 @@ def login(cls, user_id: str) -> dict: @classmethod async def logout(cls, raw_token: str) -> None: *_, token = raw_token.split() - if redis.is_connected: + if cache.is_connected: payload = cls.decode_jwt(token=token) remaining_exp_time = payload['exp'] - time.time() await cls._set_in_cache(token=token, exp=int(remaining_exp_time)) else: - logger.error('`redis` middleware is required for `logout()`') + logger.error('`redis/valkey` middleware is required for `logout()`') @classmethod async def _set_in_cache(cls, token: str, exp: int) -> None: key = generate_hash_value_from_string(token) - await redis.set(key, b'', ex=exp) + await cache.set(key, b'', ex=exp) @classmethod async def _check_in_cache(cls, token: str) -> bool: key = generate_hash_value_from_string(token) - return bool(await redis.exists(key)) + return bool(await cache.exists(key)) @staticmethod def exception(message: str | JWTError | UnicodeEncodeError, /) -> type[AuthenticationAPIError]: diff --git a/panther/base_websocket.py b/panther/base_websocket.py index 106c940..9d439c0 100644 --- a/panther/base_websocket.py +++ b/panther/base_websocket.py @@ -9,13 +9,14 @@ from panther import status from panther.base_request import BaseRequest from panther.configs import config -from panther.db.connections import redis +from panther.db.connections import cache from panther.exceptions import AuthenticationAPIError, InvalidPathVariableAPIError from panther.monitoring import Monitoring from panther.utils import Singleton, ULID if TYPE_CHECKING: from redis.asyncio import Redis + from valkey.asyncio import Valkey logger = logging.getLogger('panther') @@ -36,7 +37,7 @@ def publish(self, msg): class WebsocketConnections(Singleton): - def __init__(self, pubsub_connection: Redis | SyncManager): + def __init__(self, pubsub_connection: Redis | Valkey | SyncManager): self.connections = {} self.connections_count = 0 self.pubsub_connection = pubsub_connection @@ -46,7 +47,7 @@ def __init__(self, pubsub_connection: Redis | SyncManager): async def __call__(self): if isinstance(self.pubsub_connection, SyncManager): - # We don't have redis connection, so use the `multiprocessing.Manager` + # We don't have redis/valkey connection, so use the `multiprocessing.Manager` self.pubsub: PubSub queue = self.pubsub.subscribe() logger.info("Subscribed to 'websocket_connections' queue") @@ -62,7 +63,7 @@ async def __call__(self): queue.put(None) break else: - # We have a redis connection, so use it for pubsub + # We have a redis/valkey connection, so use it for pubsub self.pubsub = self.pubsub_connection.pubsub() await self.pubsub.subscribe('websocket_connections') logger.info("Subscribed to 'websocket_connections' channel") @@ -103,8 +104,8 @@ async def _handle_received_message(self, received_message): async def publish(self, connection_id: str, action: Literal['send', 'close'], data: any): publish_data = {'connection_id': connection_id, 'action': action, 'data': data} - if redis.is_connected: - await redis.publish('websocket_connections', json.dumps(publish_data)) + if cache.is_connected: + await cache.publish('websocket_connections', json.dumps(publish_data)) else: self.pubsub.publish(publish_data) diff --git a/panther/caching.py b/panther/caching.py index b8c5db9..14d891c 100644 --- a/panther/caching.py +++ b/panther/caching.py @@ -6,7 +6,7 @@ import orjson as json from panther.configs import config -from panther.db.connections import redis +from panther.db.connections import cache from panther.request import Request from panther.response import Response, ResponseDataTypes from panther.throttling import throttling_storage @@ -38,14 +38,14 @@ def throttling_cache_key(request: Request, duration: timedelta) -> str: async def get_response_from_cache(*, request: Request, cache_exp_time: timedelta) -> CachedResponse | None: """ - If redis.is_connected: - Get Cached Data From Redis + If redis.is_connected or valkey.is_connected: + Get Cached Data From Redis or Valkey else: Get Cached Data From Memory """ - if redis.is_connected: + if cache.is_connected: key = api_cache_key(request=request) - data = (await redis.get(key) or b'{}').decode() + data = (await cache.get(key) or b'{}').decode() if cached_value := json.loads(data): return CachedResponse(*cached_value) @@ -57,15 +57,15 @@ async def get_response_from_cache(*, request: Request, cache_exp_time: timedelta async def set_response_in_cache(*, request: Request, response: Response, cache_exp_time: timedelta | int) -> None: """ - If redis.is_connected: - Cache The Data In Redis + If redis.is_connected or valkey.is_connected: + Cache The Data In Redis or Valkey else: Cache The Data In Memory """ cache_data: tuple[ResponseDataTypes, int] = (response.data, response.status_code) - if redis.is_connected: + if cache.is_connected: key = api_cache_key(request=request) cache_exp_time = cache_exp_time or config.DEFAULT_CACHE_EXP @@ -77,32 +77,32 @@ async def set_response_in_cache(*, request: Request, response: Response, cache_e if cache_exp_time is None: logger.warning( - 'your response are going to cache in redis forever ' + 'your response are going to cache in server forever ' '** set DEFAULT_CACHE_EXP in `configs` or set the `cache_exp_time` in `@API.get()` to prevent this **' ) - await redis.set(key, cache_data) + await cache.set(key, cache_data) else: - await redis.set(key, cache_data, ex=cache_exp_time) + await cache.set(key, cache_data, ex=cache_exp_time) else: key = api_cache_key(request=request, cache_exp_time=cache_exp_time) caches[key] = cache_data if cache_exp_time: - logger.info('`cache_exp_time` is not very accurate when `redis` is not connected.') + logger.info('`cache_exp_time` is not very accurate when `redis/valkey` is not connected.') async def get_throttling_from_cache(request: Request, duration: timedelta) -> int: """ - If redis.is_connected: - Get Cached Data From Redis + If redis.is_connected or valkey.is_connected: + Get Cached Data From Redis or Valkey else: Get Cached Data From Memory """ key = throttling_cache_key(request=request, duration=duration) - if redis.is_connected: - data = (await redis.get(key) or b'0').decode() + if cache.is_connected: + data = (await cache.get(key) or b'0').decode() return json.loads(data) else: @@ -111,15 +111,15 @@ async def get_throttling_from_cache(request: Request, duration: timedelta) -> in async def increment_throttling_in_cache(request: Request, duration: timedelta) -> None: """ - If redis.is_connected: - Increment The Data In Redis + If redis.is_connected or valkey.is_connected: + Increment The Data In Redis or Valkey else: Increment The Data In Memory """ key = throttling_cache_key(request=request, duration=duration) - if redis.is_connected: - await redis.incrby(key, amount=1) + if cache.is_connected: + await cache.incrby(key, amount=1) else: throttling_storage[key] += 1 diff --git a/panther/cli/create_command.py b/panther/cli/create_command.py index 8ce81f0..c0052a4 100644 --- a/panther/cli/create_command.py +++ b/panther/cli/create_command.py @@ -16,7 +16,7 @@ AUTO_REFORMAT_PART, DATABASE_PANTHERDB_PART, DATABASE_MONGODB_PART, - USER_MODEL_PART, REDIS_PART, + USER_MODEL_PART, REDIS_PART, VALKEY_PART ) from panther.cli.utils import cli_error @@ -35,6 +35,7 @@ def __init__(self): self.database = '0' self.database_encryption = False self.redis = False + self.valkey = False self.authentication = False self.monitoring = True self.log_queries = True @@ -76,6 +77,11 @@ def __init__(self): 'message': 'Do You Want To Use Redis (Required `redis`)', 'is_boolean': True, }, + { + 'field': 'valkey', + 'message': 'Do You Want To Use Valkey (Required `valkey`)', + 'is_boolean': True, + }, { 'field': 'authentication', 'message': 'Do You Want To Use JWT Authentication (Required `python-jose`)', @@ -147,6 +153,8 @@ def _create_file(self, *, path: str, data: str): database_encryption = 'True' if self.database_encryption else 'False' database_extension = 'pdb' if self.database_encryption else 'json' redis_part = REDIS_PART if self.redis else '' + valkey_part = VALKEY_PART if self.valkey else '' + if self.database == '0': database_part = DATABASE_PANTHERDB_PART elif self.database == '1': @@ -163,6 +171,7 @@ def _create_file(self, *, path: str, data: str): data = data.replace('{PANTHERDB_ENCRYPTION}', database_encryption) # Should be after `DATABASE` data = data.replace('{PANTHERDB_EXTENSION}', database_extension) # Should be after `DATABASE` data = data.replace('{REDIS}', redis_part) + data = data.replace('{VALKEY}', valkey_part) data = data.replace('{PROJECT_NAME}', self.project_name.lower()) data = data.replace('{PANTHER_VERSION}', version()) diff --git a/panther/cli/template.py b/panther/cli/template.py index 19eb429..e9c77f6 100644 --- a/panther/cli/template.py +++ b/panther/cli/template.py @@ -63,7 +63,7 @@ async def info_api(request: Request): BASE_DIR = Path(__name__).resolve().parent env = load_env(BASE_DIR / '.env') -SECRET_KEY = env['SECRET_KEY']{DATABASE}{REDIS}{USER_MODEL}{AUTHENTICATION}{MONITORING}{LOG_QUERIES}{AUTO_REFORMAT} +SECRET_KEY = env['SECRET_KEY']{DATABASE}{REDIS}{VALKEY}{USER_MODEL}{AUTHENTICATION}{MONITORING}{LOG_QUERIES}{AUTO_REFORMAT} # More Info: https://PantherPy.GitHub.io/urls/ URLs = 'core.urls.url_routing' @@ -134,7 +134,7 @@ async def info_api(request: Request): BASE_DIR = Path(__name__).resolve().parent env = load_env(BASE_DIR / '.env') -SECRET_KEY = env['SECRET_KEY']{DATABASE}{REDIS}{USER_MODEL}{AUTHENTICATION}{MONITORING}{LOG_QUERIES}{AUTO_REFORMAT} +SECRET_KEY = env['SECRET_KEY']{DATABASE}{REDIS}{VALKEY}{USER_MODEL}{AUTHENTICATION}{MONITORING}{LOG_QUERIES}{AUTO_REFORMAT} InfoThrottling = Throttling(rate=5, duration=timedelta(minutes=1)) @@ -206,6 +206,16 @@ async def info_api(request: Request): 'db': 0, }""" +VALKEY_PART = """ + +# More Info: https://PantherPy.GitHub.io/valkey/ +VALKEY = { + 'class': 'panther.db.connections.ValkeyConnection', + 'host': '127.0.0.1', + 'port': 6379, + 'db': 0, +}""" + USER_MODEL_PART = """ # More Info: https://PantherPy.GitHub.io/configs/#user_model diff --git a/panther/cli/utils.py b/panther/cli/utils.py index 7d06f88..71afc6d 100644 --- a/panther/cli/utils.py +++ b/panther/cli/utils.py @@ -110,13 +110,13 @@ def print_uvicorn_help_message(): def print_info(config: Config): - from panther.db.connections import redis + from panther.db.connections import cache mo = config.MONITORING lq = config.LOG_QUERIES bt = config.BACKGROUND_TASKS ws = config.HAS_WS - rd = redis.is_connected + rd = cache.is_connected bd = '{0:<39}'.format(str(config.BASE_DIR)) if len(bd) > 39: bd = f'{bd[:36]}...' @@ -148,7 +148,7 @@ def print_info(config: Config): # Message info_message = f"""{logo} -{h} Redis: {rd} \t {h} +{h} Cache Server: {rd} \t {h} {h} Websocket: {ws} \t {h} {h} Monitoring: {mo} \t {h} {h} Log Queries: {lq} \t {h} diff --git a/panther/db/connections.py b/panther/db/connections.py index c852bfa..88d4f35 100644 --- a/panther/db/connections.py +++ b/panther/db/connections.py @@ -17,6 +17,12 @@ # we are going to force him to install it in `panther._load_configs.load_redis` _Redis = type('_Redis', (), {'__new__': lambda x: x}) +try: + from valkey.asyncio import Valkey as _Valkey + from valkey import exceptions as valkey_exceptions +except ImportError: + _Valkey = type('_Valkey', (), {'__new__': lambda x: x}) + if TYPE_CHECKING: from pymongo.database import Database @@ -141,5 +147,61 @@ def create_connection_for_websocket(self) -> _Redis: return self.websocket_connection +class ValkeyConnection(Singleton, _Valkey): + def __init__( + self, + init: bool = False, + host: str = 'localhost', + port: int = 6379, + db: int = 0, + websocket_db: int = 0, + **kwargs + ): + if init: + self.host = host + self.port = port + self.db = db + self.websocket_db = websocket_db + self.kwargs = kwargs + + super().__init__(host=host, port=port, db=db, **kwargs) + self.sync_ping() + + @property + def is_connected(self): + try: + self.sync_ping() + except valkey_exceptions.ConnectionError: + return False + else: + return True + + def sync_ping(self): + from valkey import Valkey + + Valkey(host=self.host, port=self.port, socket_timeout=3, **self.kwargs).ping() + + async def execute_command(self, *args, **options): + if self.is_connected: + return await super().execute_command(*args, **options) + msg = '`VALKEY` is not found in `configs`' + raise ValueError(msg) + + def create_connection_for_websocket(self) -> _Redis: + if not hasattr(self, 'websocket_connection'): + self.websocket_connection = _Redis( + host=self.host, + port=self.port, + db=self.websocket_db, + **self.kwargs + ) + return self.websocket_connection + db: DatabaseConnection = DatabaseConnection() redis: RedisConnection = RedisConnection() +valkey: ValkeyConnection = ValkeyConnection() + +if redis.is_connected: + cache = redis +else: + cache = valkey diff --git a/panther/main.py b/panther/main.py index 7e439b9..3bc54b8 100644 --- a/panther/main.py +++ b/panther/main.py @@ -49,6 +49,7 @@ def load_configs(self) -> None: self._configs_module = load_configs_module(self._configs_module_name) load_redis(self._configs_module) + load_valkey(self._configs_module) load_startup(self._configs_module) load_shutdown(self._configs_module) load_timezone(self._configs_module) diff --git a/panther/panel/apis.py b/panther/panel/apis.py index f3f067a..63e3b60 100644 --- a/panther/panel/apis.py +++ b/panther/panel/apis.py @@ -4,7 +4,7 @@ from panther.app import API from panther.configs import config from panther.db.connections import db -from panther.db.connections import redis +from panther.db.connections import cache from panther.panel.utils import get_model_fields from panther.request import Request from panther.response import Response @@ -76,8 +76,8 @@ async def healthcheck_api(): checks.append(ping) except PyMongoError: checks.append(False) - # Redis - if redis.is_connected: - checks.append(await redis.ping()) + # Cache server + if cache.is_connected: + checks.append(await cache.ping()) return Response(all(checks)) diff --git a/tests/test_caching.py b/tests/test_caching.py index 25efa68..6a43155 100644 --- a/tests/test_caching.py +++ b/tests/test_caching.py @@ -66,7 +66,7 @@ async def test_with_cache_5second_exp_time(self): # Check Logs assert len(captured.records) == 1 - assert captured.records[0].getMessage() == '`cache_exp_time` is not very accurate when `redis` is not connected.' + assert captured.records[0].getMessage() == '`cache_exp_time` is not very accurate when `redis/valkey` is not connected.' # Second Request res2 = await self.client.get('with-expired-cache') diff --git a/tests/test_cli.py b/tests/test_cli.py index 2e7212a..373a65f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -66,6 +66,7 @@ def test_print_info(self): │ \/_/\/__/\/_/\/_/\/_/\/__/ \/_/\/_/\/____/ \/_/ │ │ │ │ Redis: False │ +│ Valkey: False │ │ Websocket: False │ │ Monitoring: True │ │ Log Queries: True │