diff --git a/.gitignore b/.gitignore index 8f94f1e..94e68d9 100644 --- a/.gitignore +++ b/.gitignore @@ -221,10 +221,12 @@ ENV/ .pdm-python .pdm-build/ -data.db +data*.db config.json endpoints.json sessions/ logs/* test/* -file::* \ No newline at end of file +file::* +telegraph.json +conversation_preview.json \ No newline at end of file diff --git a/README.md b/README.md index e67d372..e4bca1a 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ CatGPT is a Telegram bot that integrates with OpenAI's api for people who like t ![new topic](assets/new.png) -* **/topic** `[share | download | title]` +* **/topic** `[share | download | iv | inner | title]` There are three optional operations that you can use to perform some task quickly. @@ -86,9 +86,19 @@ CatGPT is a Telegram bot that integrates with OpenAI's api for people who like t Use `/topic download` to download the topic's content directly without a confirmation operation. The content will be encoded as Markdown file - * **other characters** + * `iv` - Any other characters will be treated as a title, and the topic's title will be updated." + Display the chat history of this conversation in the form of Instant View even if the text length does not overflow. + + ![instant view](assets/iv.png) + + * `inner` + + Display the chat history of this conversation in the form of Telegram Message. + + * **other characters** + + Any other characters will be treated as a title, and the topic's title will be updated." ​ ![operations](assets/dl_share.png) @@ -176,6 +186,8 @@ options: * `respond_group_message`: If true, the bot will respond to group messages even if it is not mentioned. default: `false`, can be changed at runtime using the command `/respond` in groups. +* `topic_preview_type`: **TELEGRAPH** or **INTERNAL** + * `endpoinds`: your endpoints endpint: @@ -202,6 +214,7 @@ options: "access_key": "Specify Access Key to use this bot", "proxy": "http://proxy:port", "respond_group_message": false, + "topic_preview_type": "INTERNAL or TELEGRAPH", "share": [ { "name": "notes", diff --git a/assets/iv.png b/assets/iv.png new file mode 100644 index 0000000..5d0038b Binary files /dev/null and b/assets/iv.png differ diff --git a/config.example.json b/config.example.json index 88b2973..0077609 100644 --- a/config.example.json +++ b/config.example.json @@ -3,6 +3,7 @@ "access_key": "Specify Access Key to use this bot", "proxy": "http://proxy:port", "respond_group_message": false, + "topic_preview_type": "INTERNAL | TELEGRAPH", "share": [ { "name": "share provider name", @@ -10,6 +11,13 @@ "repo": "github repo", "owner": "your github username", "token": "your github personal access token" + }, + { + "name": "telegraph", + "type": "telegraph", + "author": "meiqiu(optional)", + "short_name": "meiqiu(optional)", + "token": "your telegraph token, it will create a new account if you don't provide one" } ], "endpoints": [ diff --git a/pdm.lock b/pdm.lock index 183dbc7..a65c698 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default"] strategy = ["cross_platform", "inherit_metadata"] lock_version = "4.4.1" -content_hash = "sha256:f3498dcf91d10744e13d3cf3dc830dca0630269593488381f965ecccf4005660" +content_hash = "sha256:b95c35ba2633e00812b6a3a1b588ce16d30a0b5e4dcdb34f41475dadfe23fd3a" [[package]] name = "aiohttp" @@ -135,6 +135,33 @@ files = [ {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, ] +[[package]] +name = "beautifulsoup4" +version = "4.12.3" +requires_python = ">=3.6.0" +summary = "Screen-scraping library" +groups = ["default"] +dependencies = [ + "soupsieve>1.2", +] +files = [ + {file = "beautifulsoup4-4.12.3-py3-none-any.whl", hash = "sha256:b80878c9f40111313e55da8ba20bdba06d8fa3969fc68304167741bbf9e082ed"}, + {file = "beautifulsoup4-4.12.3.tar.gz", hash = "sha256:74e3d1928edc070d21748185c46e3fb33490f22f52a3addee9aee0f4f7781051"}, +] + +[[package]] +name = "bs4" +version = "0.0.2" +summary = "Dummy package for Beautiful Soup (beautifulsoup4)" +groups = ["default"] +dependencies = [ + "beautifulsoup4", +] +files = [ + {file = "bs4-0.0.2-py2.py3-none-any.whl", hash = "sha256:abf8742c0805ef7f662dce4b51cca104cffe52b835238afc169142ab9b3fbccc"}, + {file = "bs4-0.0.2.tar.gz", hash = "sha256:a48685c58f50fe127722417bae83fe6badf500d54b55f7e39ffe43b798653925"}, +] + [[package]] name = "cachetools" version = "5.3.3" @@ -576,6 +603,35 @@ files = [ {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"}, ] +[[package]] +name = "importlib-metadata" +version = "8.0.0" +requires_python = ">=3.8" +summary = "Read metadata from Python packages" +groups = ["default"] +marker = "python_version < \"3.10\"" +dependencies = [ + "zipp>=0.5", +] +files = [ + {file = "importlib_metadata-8.0.0-py3-none-any.whl", hash = "sha256:15584cf2b1bf449d98ff8a6ff1abef57bf20f3ac6454f431736cd3e660921b2f"}, + {file = "importlib_metadata-8.0.0.tar.gz", hash = "sha256:188bd24e4c346d3f0a933f275c2fec67050326a856b9a359881d7c2a697e8812"}, +] + +[[package]] +name = "markdown" +version = "3.6" +requires_python = ">=3.8" +summary = "Python implementation of John Gruber's Markdown." +groups = ["default"] +dependencies = [ + "importlib-metadata>=4.4; python_version < \"3.10\"", +] +files = [ + {file = "Markdown-3.6-py3-none-any.whl", hash = "sha256:48f276f4d8cfb8ce6527c8f79e2ee29708508bf4d40aa410fbc3b4ee832c850f"}, + {file = "Markdown-3.6.tar.gz", hash = "sha256:ed4f41f6daecbeeb96e576ce414c41d2d876daa9a16cb35fa8ed8c2ddfad0224"}, +] + [[package]] name = "markdown-it-py" version = "3.0.0" @@ -894,6 +950,17 @@ files = [ {file = "sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"}, ] +[[package]] +name = "soupsieve" +version = "2.5" +requires_python = ">=3.8" +summary = "A modern CSS selector implementation for Beautiful Soup." +groups = ["default"] +files = [ + {file = "soupsieve-2.5-py3-none-any.whl", hash = "sha256:eaa337ff55a1579b6549dc679565eac1e3d000563bcb1c8ab0d0fefbc0c2cdc7"}, + {file = "soupsieve-2.5.tar.gz", hash = "sha256:5663d5a7b3bfaeee0bc4372e7fc48f9cff4940b3eec54a6451cc5299f1097690"}, +] + [[package]] name = "telegramify-markdown" version = "0.1.4" @@ -1016,3 +1083,15 @@ files = [ {file = "yarl-1.9.4-py3-none-any.whl", hash = "sha256:928cecb0ef9d5a7946eb6ff58417ad2fe9375762382f1bf5c55e61645f2c43ad"}, {file = "yarl-1.9.4.tar.gz", hash = "sha256:566db86717cf8080b99b58b083b773a908ae40f06681e87e589a976faf8246bf"}, ] + +[[package]] +name = "zipp" +version = "3.19.2" +requires_python = ">=3.8" +summary = "Backport of pathlib-compatible object wrapper for zip files" +groups = ["default"] +marker = "python_version < \"3.10\"" +files = [ + {file = "zipp-3.19.2-py3-none-any.whl", hash = "sha256:f091755f667055f2d02b32c53771a7a6c8b47e1fdbc4b72a8b9072b3eef8015c"}, + {file = "zipp-3.19.2.tar.gz", hash = "sha256:bf1dcf6450f873a13e952a29504887c89e6de7506209e5b1bcc3460135d4de19"}, +] diff --git a/pyproject.toml b/pyproject.toml index 331f0b7..16c6095 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,9 @@ dependencies = [ "openai==1.6.0", "aiohttp==3.9.1", "telegramify-markdown==0.1.4", - "google-generativeai==0.7.0" + "google-generativeai==0.7.0", + "markdown", + "bs4" ] requires-python = ">=3.9" readme = "README.md" diff --git a/requirements.txt b/requirements.txt index aa175a6..0b19d0b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,6 +7,8 @@ annotated-types==0.7.0 anyio==4.4.0 async-timeout==4.0.3; python_version < "3.11" attrs==23.2.0 +beautifulsoup4==4.12.3 +bs4==0.0.2 cachetools==5.3.3 certifi==2024.6.2 charset-normalizer==3.3.2 @@ -29,6 +31,8 @@ httpcore==1.0.5 httplib2==0.22.0 httpx==0.27.0 idna==3.7 +importlib-metadata==8.0.0; python_version < "3.10" +markdown==3.6 markdown-it-py==3.0.0 mdurl==0.1.2 mistletoe==1.3.0 @@ -45,9 +49,11 @@ pytelegrambotapi==4.18.1 requests==2.32.3 rsa==4.9 sniffio==1.3.1 +soupsieve==2.5 telegramify-markdown==0.1.4 tqdm==4.66.4 typing-extensions==4.12.2 uritemplate==4.1.1 urllib3==2.2.2 yarl==1.9.4 +zipp==3.19.2; python_version < "3.10" diff --git a/src/catgpt/commands/chat.py b/src/catgpt/commands/chat.py index a814bb5..87e53fc 100644 --- a/src/catgpt/commands/chat.py +++ b/src/catgpt/commands/chat.py @@ -2,8 +2,8 @@ from telebot.asyncio_helper import ApiTelegramException from telebot.types import Message -from ..context import profiles, config, get_bot_name, topic, group_config -from ..types import Endpoint, MessageType +from ..context import profiles, config, get_bot_name, topic, group_config, page_preview +from ..types import Endpoint, MessageType, Preview from ..utils.text import get_timeout_from_text, MAX_TEXT_LENGTH from . import create_convo_and_update_profile from ..provider import ask, ask_stream @@ -107,7 +107,7 @@ async def handle_message(message: Message, bot: AsyncTeleBot) -> None: text = "" try: - text = await do_reply(endpoint, model, messages, reply_msg, bot) + text = await do_reply(endpoint, model, messages, reply_msg, bot, convo) reply_msg.text = text await topic.append_messages(convo_id, message, reply_msg) @@ -131,6 +131,7 @@ async def do_reply( messages: list, reply_msg: Message, bot: AsyncTeleBot, + convo: types.Topic, ): text = "" buffered = "" @@ -185,14 +186,24 @@ async def do_reply( else: raise ae - # if len(buffered) > 0: for removing the endpoint info from the message delta = timeout - (time.time() - start) if delta > 0: await asyncio.sleep(int(delta) + 1) - text += buffered - msg_text = escape(text) + msg_text = escape(text + buffered) if text_overflow or len(msg_text) > MAX_TEXT_LENGTH: + if config.topic_preview == Preview.TELEGRAPH: + msg_text = text + buffered + title = f"{convo.title}_{reply_msg.message_id}" + url = await page_preview.preview_chat(convo.label, title, msg_text) + await bot.edit_message_text( + chat_id=reply_msg.chat.id, + message_id=reply_msg.message_id, + text=url, + disable_web_page_preview=False, + ) + return msg_text + text_overflow = True msg_text = escape(text) @@ -211,7 +222,7 @@ async def do_reply( reply_to_message_id=msg.message_id, ) - return text + return text + buffered async def do_generate_title(convo: types.Topic, messages: list, uid: int, text: str): diff --git a/src/catgpt/commands/conversation.py b/src/catgpt/commands/conversation.py index ff267f2..4ab345c 100644 --- a/src/catgpt/commands/conversation.py +++ b/src/catgpt/commands/conversation.py @@ -3,9 +3,15 @@ from ..utils.md2tgmd import escape from ..utils.text import messages_to_segments -from ..context import profiles, topic, get_bot_name +from ..context import profiles, topic, get_bot_name, config, page_preview from . import share, send_file, handle_share from ..storage import types +from ..types import Preview + +preview_mapping = { + "iv": Preview.TELEGRAPH, + "inner": Preview.INTERNAL, +} async def handle_conversation(message: Message, bot: AsyncTeleBot): @@ -19,16 +25,21 @@ async def handle_conversation(message: Message, bot: AsyncTeleBot): return bot_name = await get_bot_name() - instruction = message.text.replace("/topic", "").replace(bot_name, "").strip() - if len(instruction) == 0: + instruction = ( + message.text.replace("/topic", "").replace(bot_name, "").strip().lower() + ) + if len(instruction) == 0 or instruction in ["inner", "iv"]: + preview_type = preview_mapping.get(instruction, config.topic_preview) await show_conversation( chat_id=message.chat.id, msg_id=message.message_id, uid=uid, bot=bot, convo=convo, + profile=profile, reply_msg_id=message.message_id, thread_id=message.message_thread_id, + preview_type=preview_type, ) return @@ -91,8 +102,10 @@ async def show_conversation( uid: int, bot: AsyncTeleBot, convo: types.Topic, + profile: types.Profile, reply_msg_id: int = None, thread_id: int = None, + preview_type: Preview = None, ): messages: list[types.Message] = convo.messages or [] messages = [ @@ -119,17 +132,31 @@ async def show_conversation( InlineKeyboardButton("dismiss", callback_data=f"topic:dismiss:{context}"), ] ] - for content in segments: - reply_msg: Message = await bot.send_message( + + if preview_type == Preview.TELEGRAPH: + md_content = "\n\n".join(segments) + html_url = await page_preview.preview_md_text(profile, convo.title, md_content) + await bot.send_message( chat_id=chat_id, - text=escape(content), - parse_mode="MarkdownV2", - disable_web_page_preview=True, + text=f"[{convo.title}]({html_url})", reply_to_message_id=last_message_id, + disable_web_page_preview=False, reply_markup=InlineKeyboardMarkup(keyboard), message_thread_id=thread_id, + parse_mode="MarkdownV2", ) - last_message_id = reply_msg.message_id + else: + for content in segments: + reply_msg: Message = await bot.send_message( + chat_id=chat_id, + text=escape(content), + parse_mode="MarkdownV2", + disable_web_page_preview=True, + reply_to_message_id=last_message_id, + reply_markup=InlineKeyboardMarkup(keyboard), + message_thread_id=thread_id, + ) + last_message_id = reply_msg.message_id async def handle_download( @@ -174,9 +201,9 @@ async def do_share( await bot.send_message( chat_id=chat_id, - parse_mode="MarkdownV2", - text=escape(f"share link: {html_url}"), + text=html_url, message_thread_id=message.message_thread_id, + disable_web_page_preview=False, ) await bot.delete_messages(chat_id, msg_ids + [message.message_id]) @@ -207,7 +234,6 @@ def register(bot: AsyncTeleBot, decorator, action_provider): action = { "name": "topic", - "description": "current topic: [share|download|title]", - "delete_after_invoke": False, + "description": "current topic: [share|download|inner|title]", "order": 30, } diff --git a/src/catgpt/commands/key.py b/src/catgpt/commands/key.py index e816eb2..d5c6ea6 100644 --- a/src/catgpt/commands/key.py +++ b/src/catgpt/commands/key.py @@ -1,3 +1,4 @@ +import hashlib import hmac from telebot.async_telebot import AsyncTeleBot diff --git a/src/catgpt/commands/list.py b/src/catgpt/commands/list.py index 6dc5111..d287b6a 100644 --- a/src/catgpt/commands/list.py +++ b/src/catgpt/commands/list.py @@ -51,7 +51,6 @@ async def show_conversation_list( ] ) - print(text) if not text: text = "No conversations found." diff --git a/src/catgpt/context.py b/src/catgpt/context.py index 2865f4f..564c3a4 100644 --- a/src/catgpt/context.py +++ b/src/catgpt/context.py @@ -4,10 +4,11 @@ from .user_profile import UserProfile, Users from .group import GroupConfig -from .types import Endpoint, Configuration +from .types import Endpoint, Configuration, Preview from . import share from .topic import Topic from . import storage +from .share.preview import PagePreview import json @@ -20,6 +21,7 @@ topic: Topic | None = None bot: AsyncTeleBot | None = None bot_name = None +page_preview: PagePreview | None = None async def init_configuration(options): @@ -36,6 +38,8 @@ def load_config(): config.proxy_url = c.get("proxy", None) config.share_info = c.get("share", None) config.respond_group_message = c.get("respond_group_message", False) + preview_type = c.get("topic_preview_type", Preview.TELEGRAPH.name) + config.topic_preview = Preview[preview_type.upper()] endpoints = c.get("endpoints", []) assert len(endpoints) > 0, "endpoints is required" @@ -53,7 +57,7 @@ def load_config(): disable_notification=True, ) - share.init_providers(c.get("share", []), config) + await share.init_providers(c.get("share", []), config) async def init_datasource(options): @@ -61,6 +65,7 @@ async def init_datasource(options): global profiles global users global group_config + global page_preview from .storage.sqlite3_session_storage import ( Sqlite3Datasource, Sqlite3TopicStorage, @@ -87,6 +92,8 @@ async def init_datasource(options): group_storage = Sqlite3GroupInfoStorage() group_config = GroupConfig(group_storage, config.respond_group_message) + page_preview = PagePreview(profiles) + async def init(options): assert options.config is not None, "Config file is required" diff --git a/src/catgpt/main.py b/src/catgpt/main.py index 9abef05..de519db 100644 --- a/src/catgpt/main.py +++ b/src/catgpt/main.py @@ -4,6 +4,7 @@ from telebot.async_telebot import AsyncTeleBot from . import context +from . import share async def main(): diff --git a/src/catgpt/share/__init__.py b/src/catgpt/share/__init__.py index 2aeccea..8fd28c3 100644 --- a/src/catgpt/share/__init__.py +++ b/src/catgpt/share/__init__.py @@ -1,12 +1,14 @@ import importlib -from ..types import ShareType, Configuration +from ..types import ShareType, Configuration, Preview from ..storage.types import Topic +from . import telegraph share_providers = {} +default_providers = {} -def init_providers(providers: list[dict], config: Configuration): +async def init_providers(providers: list[dict], config: Configuration): module_info = {} types = {m.name.lower(): m.value for m in ShareType} @@ -26,11 +28,18 @@ def init_providers(providers: list[dict], config: Configuration): module = importlib.import_module(f".{module_name}", __package__) module_info[module_name] = module - instance = module.create(params) + instance = await module.create(params, config) if not instance: raise Exception(f"Got none from module: {module_name}") share_providers[provider["name"]] = instance + default_providers[module_name] = instance + + if Preview.TELEGRAPH.value not in default_providers: + module = importlib.import_module(f".telegraph", __package__) + instance = await module.create({}, config) + share_providers["telegraph"] = instance + default_providers[Preview.TELEGRAPH.value] = instance async def share(name: str, convo: Topic): @@ -39,3 +48,10 @@ async def share(name: str, convo: Topic): raise Exception(f"Unknown share provider: {name}") return await provider.share(convo) + + +def get_provider_by_type(preview_type: Preview): + if preview_type.value in default_providers: + return default_providers[preview_type.value] + + return None diff --git a/src/catgpt/share/github.py b/src/catgpt/share/github.py index b3b8592..f0043ea 100644 --- a/src/catgpt/share/github.py +++ b/src/catgpt/share/github.py @@ -50,7 +50,7 @@ async def create_github_issue(self, title, body, label): url, headers=headers, json=data, proxy=self.proxy ) as response: if not response.ok: - raise Exception(f"Failed to create issue: {response.text}") + raise Exception(f"Failed to create issue: {await response.text()}") return await response.json() @@ -86,8 +86,18 @@ async def share(self, convo: Topic): label=convo.label, ) + async def share_text(self, article_id, title, content): + return await self.create_or_update_issue( + title=title, + body=content, + label=article_id, + ) + + def get_token(self): + return self.token + -def create(params: dict): +async def create(params: dict, config): return GithubProvider( name=params.get("name"), owner=params.get("owner"), diff --git a/src/catgpt/share/preview.py b/src/catgpt/share/preview.py new file mode 100644 index 0000000..84a1477 --- /dev/null +++ b/src/catgpt/share/preview.py @@ -0,0 +1,33 @@ +from . import get_provider_by_type +from ..types import Preview + + +class PagePreview: + def __init__(self, profiles): + self.provider = get_provider_by_type(Preview.TELEGRAPH) + self.profiles = profiles + + async def preview_md_text(self, profile, title, content): + token = self.provider.get_token() + if profile.preview_url and profile.preview_token == token: + url = await self.provider.update_text(profile.preview_url, title, content) + return url + + result = await self.create_preview_page(content) + if "url" in result and "path" in result: + profile.preview_url = result["path"] + profile.preview_token = token + await self.profiles.update( + profile.uid, profile.chat_id, profile.thread_id, profile + ) + + return result["url"] + + async def preview_chat(self, aid, title, content): + return await self.provider.share_text(aid, title, content) + + async def create_preview_page(self, content): + import uuid + + article_id = str(uuid.uuid4()) + return await self.provider.create_page(article_id, content) diff --git a/src/catgpt/share/telegraph.py b/src/catgpt/share/telegraph.py new file mode 100644 index 0000000..a6ccfa5 --- /dev/null +++ b/src/catgpt/share/telegraph.py @@ -0,0 +1,313 @@ +import json +import aiohttp +import os +import markdown + +from bs4 import BeautifulSoup + +from ..utils.text import messages_to_segments +from ..storage.types import Topic + + +TELEGRAPH_API_URL = "https://api.telegra.ph/createAccount" + + +class TelegraphAPI: + def __init__( + self, + access_token=None, + short_name="mq", + author_name="mq", + author_url=None, + proxy_url=None, + ): + self.access_token = access_token + self.base_url = "https://api.telegra.ph" + + self.short_name = short_name + self.author_name = author_name + self.author_url = author_url + self.proxy = proxy_url + + @staticmethod + async def create( + access_token=None, + short_name="meiqiu", + author_name="meiqiu", + author_url=None, + proxy_url=None, + ): + TelegraphAPI.load_token() + + api = TelegraphAPI( + access_token=access_token, + short_name=short_name, + author_name=author_name, + author_url=author_url, + proxy_url=proxy_url, + ) + + if not api.access_token: + token_info = api.load_token() + if token_info is not None: + api.access_token = token_info["access_token"] + api.short_name = token_info["short_name"] + api.author_name = token_info["author_name"] + else: + access_token = await api._create_ph_account( + short_name, author_name, author_url + ) + api.access_token = access_token + api.save_token( + { + "short_name": api.short_name, + "author_name": api.author_name, + "access_token": api.access_token, + } + ) + + return api + + @staticmethod + def save_token(token): + with open("telegraph.json", "w") as f: + json.dump(token, f, indent=4) + + @staticmethod + def load_token(): + if os.path.exists("telegraph.json"): + with open("telegraph.json", "r") as f: + return json.load(f) + + return None + + async def _create_ph_account(self, short_name, author_name, author_url): + # If no existing valid token in TOKEN_FILE, create a new account + data = { + "short_name": short_name, + "author_name": author_name, + "author_url": author_url, + } + + # Make API request + account = await self.post(TELEGRAPH_API_URL, data=data) + if "result" in account and "access_token" in account["result"]: + return account["result"]["access_token"] + + raise Exception("Failed to create telegra.ph account") + + async def create_page( + self, title, content, author_name=None, author_url=None, return_content=False + ): + url = f"{self.base_url}/createPage" + data = { + "access_token": self.access_token, + "title": title, + "content": json.dumps(content, ensure_ascii=False), + "return_content": return_content, + "author_name": author_name if author_name else self.author_name, + "author_url": author_url if author_url else self.author_url, + } + + response = await self.post(url, data=data) + if "result" not in response: + raise Exception("Failed to create page, " + str(response)) + + return response["result"] + + async def get_account_info(self): + url = f'{self.base_url}/getAccountInfo?access_token={self.access_token}&fields=["short_name","author_name","author_url","auth_url"]' + try: + response = await self.get(url) + return response.get("result", None) + except Exception as e: + print(f"Fail getting telegra.ph token info {e}") + + async def edit_page( + self, + path, + title, + content, + author_name=None, + author_url=None, + return_content=False, + ): + url = f"{self.base_url}/editPage" + data = { + "access_token": self.access_token, + "path": path, + "title": title, + "content": json.dumps(content, ensure_ascii=False), + "return_content": return_content, + "author_name": author_name if author_name else self.author_name, + "author_url": author_url if author_url else self.author_url, + } + + response = await self.post(url, data=data) + if "result" not in response or "url" not in response["result"]: + raise Exception("Failed to edit page, " + str(response)) + + return response["result"]["url"] + + async def get_page(self, path): + url = f"{self.base_url}/getPage/{path}?return_content=true" + response = await self.get(url) + return response.get("result") + + async def create_page_md( + self, + title, + markdown_text, + author_name=None, + author_url=None, + return_content=False, + ): + content = self._md_to_dom(markdown_text) + return await self.create_page( + title, content, author_name, author_url, return_content + ) + + async def edit_page_md( + self, + path, + title, + markdown_text, + author_name=None, + author_url=None, + return_content=False, + ): + content = self._md_to_dom(markdown_text) + return await self.edit_page( + path, title, content, author_name, author_url, return_content + ) + + async def authorize_browser(self): + url = f'{self.base_url}/getAccountInfo?access_token={self.access_token}&fields=["auth_url"]' + response = await self.get(url) + return response["result"]["auth_url"] + + def _md_to_dom(self, markdown_text): + html = markdown.markdown( + markdown_text, + extensions=["markdown.extensions.extra", "markdown.extensions.sane_lists"], + ) + + soup = BeautifulSoup(html, "html.parser") + + def parse_element(element): + tag_dict = {"tag": element.name} + if element.name in ["h1", "h2", "h3", "h4", "h5", "h6"]: + if element.name == "h1": + tag_dict["tag"] = "h3" + elif element.name == "h2": + tag_dict["tag"] = "h4" + else: + tag_dict["tag"] = "p" + tag_dict["children"] = [ + {"tag": "strong", "children": element.contents} + ] + + if element.attrs: + tag_dict["attributes"] = element.attrs + if element.contents: + children = [] + for child in element.contents: + if isinstance(child, str): + children.append(child.strip()) + else: + children.append(parse_element(child)) + tag_dict["children"] = children + else: + if element.attrs: + tag_dict["attributes"] = element.attrs + if element.contents: + children = [] + for child in element.contents: + if isinstance(child, str): + children.append(child.strip()) + else: + children.append(parse_element(child)) + if children: + tag_dict["children"] = children + return tag_dict + + new_dom = [] + for element in soup.contents: + if isinstance(element, str) and not element.strip(): + continue + elif isinstance(element, str): + new_dom.append({"tag": "text", "content": element.strip()}) + else: + new_dom.append(parse_element(element)) + + return new_dom + + async def get(self, url): + async with aiohttp.ClientSession() as session: + async with session.get(url, proxy=self.proxy) as response: + if not response.ok: + raise Exception(f"Failed to get: {await response.text()}") + + return await response.json() + + async def post(self, url, data): + async with aiohttp.ClientSession() as session: + async with session.post(url, json=data, proxy=self.proxy) as response: + if not response.ok: + raise Exception(f"Failed to create issue: {await response.text()}") + + return await response.json() + + +class TelegraphProvider: + def __init__(self, api: TelegraphAPI): + self.api = api + + async def share(self, convo: Topic): + md_content = messages_to_segments(convo.messages, 65535)[0] + result = await self.api.create_page_md( + title=convo.title, + markdown_text=md_content, + ) + + if "url" not in result: + raise Exception("Failed to create page") + + return result.get("url") + + async def share_text(self, article_id: str, title: str, content: str): + result = await self.api.create_page_md( + title=title, + markdown_text=content, + ) + if "url" not in result: + raise Exception("Failed to create page") + + return result.get("url") + + async def update_text(self, path, title, content): + return await self.api.edit_page_md( + path=path, + title=title, + markdown_text=content, + ) + + async def create_page(self, title, content): + return await self.api.create_page_md(title=title, markdown_text=content) + + def get_token(self): + return self.api.access_token + + +async def create(params: dict, config): + author = params.get("author", "meiqiu") + short_name = params.get("short_name", author) + proxy_url = config.proxy_url + api = await TelegraphAPI.create( + access_token=params.get("token"), + short_name=short_name, + author_name=author, + proxy_url=proxy_url, + ) + + return TelegraphProvider(api) diff --git a/src/catgpt/storage/sqlite3_session_storage.py b/src/catgpt/storage/sqlite3_session_storage.py index a7650d8..83abb70 100644 --- a/src/catgpt/storage/sqlite3_session_storage.py +++ b/src/catgpt/storage/sqlite3_session_storage.py @@ -10,7 +10,17 @@ CURRENT_VERSION = "0.1.0" VERSION_CODE = 2406252000 -VERSION = [{"version_name": "0.1.0", "version_code": 2406252010, "sql_list": []}] +VERSION = [ + {"version_name": "0.1.0", "version_code": 2406252010, "sql_list": []}, + { + "version_name": "0.1.1", + "version_code": 2406292020, + "sql_list": [ + "alter table profile add preview_url TEXT;", + "alter table profile add preview_token TEXT;", + ], + }, +] def migrate(connection): @@ -21,19 +31,20 @@ def migrate(connection): vi = ("0.0.1", 0) latest_version = None + cursor = connection.cursor() for version in VERSION: if vi[1] >= version["version_code"]: continue sqlite_list = version.get("sql_list", []) for sql in sqlite_list: - connection.execute(sql) + cursor.execute(sql) latest_version = version if latest_version: sql = "insert into version (version_name, version_code) values (?,?)" - connection.execute( + cursor.execute( sql, (latest_version["version_name"], latest_version["version_code"]) ) @@ -328,7 +339,7 @@ class Sqlite3ProfileStorage(types.ProfileStorage, tx.Transactional): @tx.transactional(tx_type="write") async def create_profile(self, profile: types.Profile) -> int: t = await self.retrieve_transaction() - sql = "insert into profile (uid, model, endpoint, prompt, chat_type, chat_id, thread_id, topic_id) values " + fields = "(uid, model, endpoint, prompt, chat_type, chat_id, thread_id, topic_id, preview_url, preview_token)" columns = ( profile.uid, profile.model, @@ -338,8 +349,11 @@ async def create_profile(self, profile: types.Profile) -> int: profile.chat_id, profile.thread_id or 0, profile.topic_id, + profile.preview_url or "", + profile.preview_token or "", ) - sql += "(" + ",".join("?" * len(columns)) + ")" + placeholders = "(" + ",".join("?" * len(columns)) + ")" + sql = f"insert into profile {fields} values {placeholders}" cursor = t.connection.execute(sql, columns) return cursor.lastrowid @@ -374,7 +388,7 @@ async def update( self, uid: int, chat_id: int, thread_id: int, profile: types.Profile ): t = await self.retrieve_transaction() - sql = f"update profile set model = ?, endpoint = ?, prompt = ?, topic_id = ? where uid = ? and chat_id = ? and thread_id = ?" + sql = f"update profile set model = ?, endpoint = ?, prompt = ?, topic_id = ?, preview_url = ?, preview_token = ? where uid = ? and chat_id = ? and thread_id = ?" t.connection.execute( sql, ( @@ -382,6 +396,8 @@ async def update( profile.endpoint, profile.prompt, profile.topic_id, + profile.preview_url, + profile.preview_token, profile.uid, profile.chat_id, profile.thread_id or 0, diff --git a/src/catgpt/storage/types.py b/src/catgpt/storage/types.py index f181ce1..a20d871 100644 --- a/src/catgpt/storage/types.py +++ b/src/catgpt/storage/types.py @@ -79,6 +79,8 @@ def __init__( chat_id: int, thread_id: int, topic_id: int, + preview_url, + preview_token, ): self.uid = uid self.model = model @@ -88,6 +90,8 @@ def __init__( self.chat_id = chat_id self.thread_id = thread_id self.topic_id = topic_id + self.preview_url = preview_url + self.preview_token = preview_token def get_key(self): return f"{self.uid}-{self.chat_id}-{self.thread_id}" @@ -95,7 +99,7 @@ def get_key(self): def __repr__(self): return ( f"Profile(uid={self.uid}, model={self.model}, endpoint={self.endpoint}, prompt={self.prompt}, " - f"chat_type={self.chat_type}, chat_id={self.chat_id}, thread_id={self.thread_id})" + f"chat_type={self.chat_type}, chat_id={self.chat_id}, thread_id={self.thread_id}), preview_url={self.preview_url}" ) diff --git a/src/catgpt/types.py b/src/catgpt/types.py index 078805a..9c2eb45 100644 --- a/src/catgpt/types.py +++ b/src/catgpt/types.py @@ -64,6 +64,11 @@ def __str__(self): )""" +class Preview(enum.Enum): + INTERNAL = "internal" + TELEGRAPH = "telegraph" + + class Configuration: def __init__(self): @@ -72,6 +77,7 @@ def __init__(self): self.share_info = None self.endpoints: [Endpoint] = [] self.respond_group_message = False + self.topic_preview = Preview.INTERNAL def get_endpoints(self) -> [Endpoint]: return self.endpoints @@ -144,3 +150,4 @@ def get(chat_type: str): class ShareType(enum.Enum): GITHUB = "github" + TELEGRAPH = "telegraph" diff --git a/src/catgpt/user_profile.py b/src/catgpt/user_profile.py index 9bb0277..5d214e5 100644 --- a/src/catgpt/user_profile.py +++ b/src/catgpt/user_profile.py @@ -54,6 +54,8 @@ async def create( chat_id=chat_id, thread_id=thread_id, topic_id=topic_id, + preview_url=None, + preview_token=None, ) await self.storage.create_profile(profile)