From c24325c57d8aac3f78e46dd688b8092da5f15124 Mon Sep 17 00:00:00 2001 From: Trim21 Date: Mon, 23 Sep 2024 11:31:04 +0800 Subject: [PATCH 1/6] fix: phpserialize byte types --- chii/compat/phpseralize.py | 6 ++++-- chii/db/tables.py | 4 ++-- poetry.lock | 24 +++++++++++++++++++++++- pyproject.toml | 2 ++ rpc/timeline_service.py | 14 +++++++------- 5 files changed, 38 insertions(+), 12 deletions(-) diff --git a/chii/compat/phpseralize.py b/chii/compat/phpseralize.py index bd2a507..25d95d0 100644 --- a/chii/compat/phpseralize.py +++ b/chii/compat/phpseralize.py @@ -21,6 +21,8 @@ "dumps", ) +import six + def load(fp: BytesIO) -> Any: """Read a string from the open file object `fp` and interpret it as a @@ -108,12 +110,12 @@ def __decode(): return __decode() -def loads(data: bytes) -> Any: +def loads(data: bytes | str) -> Any: """Read a PHP-serialized object hierarchy from a string. Characters in the string past the object's representation are ignored. On Python 3 the string must be a bytestring. """ - with BytesIO(data) as fp: + with BytesIO(six.ensure_binary(data)) as fp: return load(fp) diff --git a/chii/db/tables.py b/chii/db/tables.py index b5cbaf8..2732be8 100644 --- a/chii/db/tables.py +++ b/chii/db/tables.py @@ -60,8 +60,8 @@ class ChiiTimeline: ) }, ) - memo: bytes = field( - default=b"", metadata={"sa": Column("tml_memo", MEDIUMTEXT, nullable=False)} + memo: str = field( + default="", metadata={"sa": Column("tml_memo", MEDIUMTEXT, nullable=False)} ) img: str = field( default="", diff --git a/poetry.lock b/poetry.lock index acd2a51..9847949 100644 --- a/poetry.lock +++ b/poetry.lock @@ -914,6 +914,17 @@ enabler = ["pytest-enabler (>=2.2)"] test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] type = ["importlib-metadata (>=7.0.2)", "jaraco.develop (>=7.21)", "mypy (==1.11.*)", "pytest-mypy"] +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + [[package]] name = "sqlacodegen" version = "3.0.0rc5" @@ -1098,6 +1109,17 @@ files = [ {file = "types_protobuf-5.27.0.20240920-py3-none-any.whl", hash = "sha256:c04140bd3c761a55f4e661372b24a6f508169e0815f2b73da33f34b447ed7a8d"}, ] +[[package]] +name = "types-six" +version = "1.16.21.20240513" +description = "Typing stubs for six" +optional = false +python-versions = ">=3.8" +files = [ + {file = "types-six-1.16.21.20240513.tar.gz", hash = "sha256:cdf445b5161bf17753500713a475ab79a45bd0d87728b8bfcecd86e2fbf66402"}, + {file = "types_six-1.16.21.20240513-py3-none-any.whl", hash = "sha256:af2a105be6d504339bfed81319cc8e8697865f0ee5c6baa63658f127b33b9e63"}, +] + [[package]] name = "typing-extensions" version = "4.12.2" @@ -1132,4 +1154,4 @@ test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "9deae6ff84848edaf08417f7f0fc531743e318d93be4b5e2ecfe7e8f5188874b" +content-hash = "45cd5baf4992692c79a22da3e4bde9fe61fbaca4b987751d3d4f8b2610a985f8" diff --git a/pyproject.toml b/pyproject.toml index 12470c3..8a2184a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,7 @@ pydantic-settings = "2.5.2" typing-extensions = '4.12.2' sslog = "0.0.0a45" protobuf = "5.27.2" +six = "^1.16.0" [tool.poetry.group.dev.dependencies] sqlacodegen = "3.0.0rc5" @@ -33,6 +34,7 @@ pytest-env = "==1.1.5" pre-commit = "==3.8.0" mypy = "==1.11.2" types-protobuf = "^5.27.0.20240920" +types-six = "^1.16.21.20240513" [tool.poetry-plugin-bump] commit_msg = 'bump: v{version}' diff --git a/rpc/timeline_service.py b/rpc/timeline_service.py index 688c94c..f5fae99 100644 --- a/rpc/timeline_service.py +++ b/rpc/timeline_service.py @@ -111,7 +111,7 @@ def merge_previous_timeline( m.collect_rate = req.rate if should_update: - tl.memo = phpseralize.dumps(m.model_dump()) + tl.memo = phpseralize.dumps(m.model_dump()).decode() session.add(tl) return @@ -127,7 +127,7 @@ def merge_previous_timeline( tl.batch = 1 tl.memo = phpseralize.dumps( {key: value.model_dump() for key, value in memo.items()} - ) + ).decode() session.add(tl) @@ -147,7 +147,7 @@ def create_subject_collection_timeline( cat=TimelineCat.Subject, type=type, uid=req.user_id, - memo=phpseralize.dumps(memo.model_dump()), + memo=phpseralize.dumps(memo.model_dump()).decode(), batch=0, related=str(req.subject.id), source=TimelineSource.api, @@ -200,7 +200,7 @@ def __episode_collect(self, req: EpisodeCollectRequest) -> EpisodeCollectRespons and tl.batch == 0 and tl.related == str(req.subject.id) ): - tl.memo = phpseralize.dumps(memo.model_dump()) + tl.memo = phpseralize.dumps(memo.model_dump()).decode() tl.source = TimelineSource.api session.add(tl) return EpisodeCollectResponse(ok=True) @@ -208,7 +208,7 @@ def __episode_collect(self, req: EpisodeCollectRequest) -> EpisodeCollectRespons session.add( ChiiTimeline( uid=req.user_id, - memo=phpseralize.dumps(memo.model_dump()), + memo=phpseralize.dumps(memo.model_dump()).decode(), cat=TimelineCat.Progress, type=tlType, source=TimelineSource.api, @@ -270,7 +270,7 @@ def __subject_progress( and tl.batch == 0 and tl.related == str(req.subject.id) ): - tl.memo = phpseralize.dumps(memo.model_dump()) + tl.memo = phpseralize.dumps(memo.model_dump()).decode() tl.source = TimelineSource.api session.add(tl) session.commit() @@ -279,7 +279,7 @@ def __subject_progress( session.add( ChiiTimeline( uid=req.user_id, - memo=phpseralize.dumps(memo.model_dump()), + memo=phpseralize.dumps(memo.model_dump()).decode(), cat=TimelineCat.Progress, type=tlType, batch=0, From 29f9c942a36164db04f5186c22b1c9515eded47b Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 23 Sep 2024 11:34:48 +0800 Subject: [PATCH 2/6] build(deps): pin dependency six to 1.16.0 (#262) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- poetry.lock | 3 ++- pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/poetry.lock b/poetry.lock index 9847949..cbabc37 100644 --- a/poetry.lock +++ b/poetry.lock @@ -240,6 +240,7 @@ files = [ {file = "greenlet-3.1.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d0028e725ee18175c6e422797c407874da24381ce0690d6b9396c204c7f7276e"}, {file = "greenlet-3.1.1-cp39-cp39-win32.whl", hash = "sha256:5e06afd14cbaf9e00899fae69b24a32f2196c19de08fcb9f4779dd4f004e5e7c"}, {file = "greenlet-3.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:3319aa75e0e0639bc15ff54ca327e8dc7a6fe404003496e3c6925cd3142e0e22"}, + {file = "greenlet-3.1.1.tar.gz", hash = "sha256:4ce3ac6cdb6adf7946475d7ef31777c26d94bccc377e070a7986bd2d5c515467"}, ] [package.extras] @@ -1154,4 +1155,4 @@ test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "45cd5baf4992692c79a22da3e4bde9fe61fbaca4b987751d3d4f8b2610a985f8" +content-hash = "fdff624270946657dbfbeee8e29f18acf01e454a8e4a1f49adb590d5f839a1e7" diff --git a/pyproject.toml b/pyproject.toml index 8a2184a..d446933 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ pydantic-settings = "2.5.2" typing-extensions = '4.12.2' sslog = "0.0.0a45" protobuf = "5.27.2" -six = "^1.16.0" +six = "1.16.0" [tool.poetry.group.dev.dependencies] sqlacodegen = "3.0.0rc5" From ab95c7ae8700b5e14005561dcf3044802a4b667e Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 23 Sep 2024 11:36:27 +0800 Subject: [PATCH 3/6] build(deps): update dependency sslog to v0.0.0a47 (#266) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- poetry.lock | 8 ++++---- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index cbabc37..e5d831f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1037,13 +1037,13 @@ sqlcipher = ["sqlcipher3_binary"] [[package]] name = "sslog" -version = "0.0.0a45" +version = "0.0.0a47" description = "opinionated logger based on structlog" optional = false python-versions = "~=3.8" files = [ - {file = "sslog-0.0.0a45-py3-none-any.whl", hash = "sha256:fe4777e90c9caba57af183db6eed7d0ba797da829e5f4acb920c32a7b9ed54f3"}, - {file = "sslog-0.0.0a45.tar.gz", hash = "sha256:169e17e5fdc7f302fe0e33b94cb22a2cf8f2f296899ed6c239a643de5b3847d0"}, + {file = "sslog-0.0.0a47-py3-none-any.whl", hash = "sha256:f77bb238acaf56116fc1606ffe1c9e7e7f6d9db627f2d716dfe6bd802527982f"}, + {file = "sslog-0.0.0a47.tar.gz", hash = "sha256:2590a7b02d2b7dc6b3db014a89281330a6a7314b6bf54b1a3c64dcbaded79350"}, ] [package.dependencies] @@ -1155,4 +1155,4 @@ test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "fdff624270946657dbfbeee8e29f18acf01e454a8e4a1f49adb590d5f839a1e7" +content-hash = "3c38830fa47e7a1b3988a44d340acbd8b83c879b379a11e91b45d28405fcd732" diff --git a/pyproject.toml b/pyproject.toml index d446933..c8843b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ pymysql = "1.1.1" pydantic = "2.9.2" pydantic-settings = "2.5.2" typing-extensions = '4.12.2' -sslog = "0.0.0a45" +sslog = "0.0.0a47" protobuf = "5.27.2" six = "1.16.0" From da4c054a9e9a2718581157106b07a4e08424f7de Mon Sep 17 00:00:00 2001 From: Trim21 Date: Mon, 23 Sep 2024 11:41:26 +0800 Subject: [PATCH 4/6] log user id --- rpc/timeline_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rpc/timeline_service.py b/rpc/timeline_service.py index f5fae99..11ded9b 100644 --- a/rpc/timeline_service.py +++ b/rpc/timeline_service.py @@ -193,7 +193,7 @@ def __episode_collect(self, req: EpisodeCollectRequest) -> EpisodeCollectRespons ) if tl and tl.dateline >= int(time.time() - 15 * 60): - logger.info("find previous timeline, updating") + logger.info("find previous timeline, updating", user_id=req.user_id) if ( tl.cat == TimelineCat.Progress and tl.type == tlType @@ -263,7 +263,7 @@ def __subject_progress( ) if tl and tl.dateline >= int(time.time() - 15 * 60): - logger.info("find previous timeline, updating") + logger.info("find previous timeline, updating", user_id=req.user_id) if ( tl.cat == TimelineCat.Progress and tl.type == tlType From 3da0679a5c9221a2d970dea9df23f1531bd4b64b Mon Sep 17 00:00:00 2001 From: Trim21 Date: Mon, 23 Sep 2024 11:45:18 +0800 Subject: [PATCH 5/6] improve logging filepath --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 20283c5..83b71bd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,6 +29,6 @@ RUN pip install -U pip && \ WORKDIR /app -ENTRYPOINT [ "python", "./start_grpc_server.py" ] +ENTRYPOINT [ "python", "-m", "start_grpc_server" ] COPY . ./ From 2c4898cc84fa89e5133d6e1ffbb66bbe0ce19ab7 Mon Sep 17 00:00:00 2001 From: Trim21 Date: Mon, 23 Sep 2024 11:53:11 +0800 Subject: [PATCH 6/6] add healthy chck --- scripts/grpc_client.py | 37 ++++++------------------------------- 1 file changed, 6 insertions(+), 31 deletions(-) diff --git a/scripts/grpc_client.py b/scripts/grpc_client.py index 8a05d4f..8b3ad29 100644 --- a/scripts/grpc_client.py +++ b/scripts/grpc_client.py @@ -1,41 +1,16 @@ import grpc +from sslog import logger from api.v1 import timeline_pb2_grpc -from api.v1.timeline_pb2 import ( - Episode, - EpisodeCollectRequest, - EpisodeCollectResponse, - Subject, -) +from api.v1.timeline_pb2 import HelloRequest +from chii.config import config def run() -> None: - print("Will try to greet world ...") - with grpc.insecure_channel("127.0.0.1:5000") as channel: + logger.info("healthy check") + with grpc.insecure_channel(f"127.0.0.1:{config.grpc_port}") as channel: stub = timeline_pb2_grpc.TimeLineServiceStub(channel) - response: EpisodeCollectResponse = stub.EpisodeCollect( - EpisodeCollectRequest( - user_id=100, - last=Episode( - name_cn="22", - type=1, - name="name", - id=1, - sort=1.4, - ), - subject=Subject( - id=88, - name="nn", - image="image image", - type=2, - name_cn="name cn", - series=False, - eps_total=0, - vols_total=0, - ), - ) - ) - print("Greeter client received:", response.ok) + stub.Hello(HelloRequest(name="client")) if __name__ == "__main__":