From 4a63769c590e3350c69091d19662814457bea884 Mon Sep 17 00:00:00 2001 From: cdhigh Date: Tue, 20 Aug 2024 12:07:45 -0300 Subject: [PATCH] get rid of warning utcnow --- application/__init__.py | 2 +- application/back_end/db_models.py | 28 +++++++++---------- application/lib/calibre/web/feeds/news.py | 4 +-- application/lib/calibre/web/fetch/simple.py | 8 +++++- .../lib/ebook_translator/html_translator.py | 26 ++++++++++++----- application/utils.py | 4 +++ application/view/admin.py | 4 +-- application/view/adv.py | 2 +- application/view/library_offical.py | 8 +++--- application/view/login.py | 12 ++++---- application/view/logs.py | 5 ++-- application/view/subscribe.py | 6 ++-- 12 files changed, 66 insertions(+), 43 deletions(-) diff --git a/application/__init__.py b/application/__init__.py index 31c636c7..e0424032 100644 --- a/application/__init__.py +++ b/application/__init__.py @@ -44,7 +44,7 @@ def BeforeRequest(): session.permanent = True app.permanent_session_lifetime = datetime.timedelta(days=31) g.version = appVer - g.now = datetime.datetime.utcnow + g.now = lambda: datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) g.allowSignup = (app.config['ALLOW_SIGNUP'] == 'yes') g.allowReader = app.config['EBOOK_SAVE_DIR'] diff --git a/application/back_end/db_models.py b/application/back_end/db_models.py index 1f2ee731..ff4c8175 100644 --- a/application/back_end/db_models.py +++ b/application/back_end/db_models.py @@ -3,9 +3,9 @@ #数据库结构定义，使用这个文件隔离sql和nosql的差异，尽量向外提供一致的接口 #Visit for the latest version #Author: cdhigh -import os, sys, random, datetime +import os, random, datetime from operator import attrgetter -from ..utils import PasswordManager, ke_encrypt, ke_decrypt, tz_now +from ..utils import PasswordManager, ke_encrypt, ke_decrypt, utcnow if os.getenv('DATABASE_URL', '').startswith(("datastore", "mongodb", "redis", "pickle")): from .db_models_nosql import * @@ -20,7 +20,7 @@ class KeUser(MyBaseModel): # kindleEar User send_time = IntegerField(default=6) expiration_days = IntegerField(default=0) #账号超期设置值，0为永久有效 expires = DateTimeField(null=True) #超过了此日期后账号自动停止推送 - created_time = DateTimeField(default=datetime.datetime.utcnow) + created_time = DateTimeField(default=utcnow) #email,sender,kindle_email,secret_key,enable_send,timezone,inbound_email, #keep_in_email_days,delivery_mode,webshelf_days,reader_params @@ -170,7 +170,7 @@ def local_time(self, fmt=None): class UserBlob(MyBaseModel): name = CharField() user = CharField() - time = DateTimeField(default=datetime.datetime.utcnow) + time = DateTimeField(default=utcnow) data = BlobField(null=True, index=False) #RSS订阅源，包括自定义RSS，上传的recipe，内置在zip里面的builtin_recipe不包括在内 @@ -183,7 +183,7 @@ class Recipe(MyBaseModel): type_ = CharField() #'custom','upload' needs_subscription = BooleanField(default=False) #是否需要登陆网页，只有上传的recipe才有意义 src = TextField(default='', index=False) #保存上传的recipe的unicode字符串表示，已经解码 - time = DateTimeField(default=datetime.datetime.utcnow) #源被加入的时间，用于排序 + time = DateTimeField(default=utcnow) #源被加入的时间，用于排序 user = CharField() #哪个账号创建的，和nosql一致，保存用户名 language = CharField(default='') translator = JSONField(default=JSONField.dict_default) #用于自定义RSS的备份，实际使用的是BookedRecipe @@ -218,7 +218,7 @@ class BookedRecipe(MyBaseModel): encrypted_pwd = CharField(default='') send_days = JSONField(default=JSONField.dict_default) #{'type':'weekday/date','days':[]} send_times = JSONField(default=JSONField.list_default) - time = DateTimeField(default=datetime.datetime.utcnow) #源被订阅的时间，用于排序 + time = DateTimeField(default=utcnow) #源被订阅的时间，用于排序 translator = JSONField(default=JSONField.dict_default) tts = JSONField(default=JSONField.dict_default) custom = JSONField(default=JSONField.dict_default) #留着扩展，避免后续一些小特性还需要升级数据表结构 @@ -239,14 +239,14 @@ class DeliverLog(MyBaseModel): to = CharField() size = IntegerField(default=0) time_str = CharField() #每个用户的时区可能不同，为显示方便，创建记录时就生成推送时间字符串 - datetime = DateTimeField(default=datetime.datetime.utcnow) + datetime = DateTimeField(default=utcnow) book = CharField(default='') status = CharField() class WhiteList(MyBaseModel): mail = CharField() user = CharField() - time = DateTimeField(default=datetime.datetime.utcnow) + time = DateTimeField(default=utcnow) #Shared RSS links from other users [for kindleear.appspot.com only] class SharedRss(MyBaseModel): @@ -259,11 +259,11 @@ class SharedRss(MyBaseModel): src = TextField(default='', index=False) #保存分享的recipe的unicode字符串表示，已经解码 description = CharField(default='') creator = CharField(default='') #保存贡献者的md5 - created_time = DateTimeField(default=datetime.datetime.utcnow) + created_time = DateTimeField(default=utcnow) subscribed = IntegerField(default=0) #for sort - last_subscribed_time = DateTimeField(default=datetime.datetime.utcnow, index=True) + last_subscribed_time = DateTimeField(default=utcnow, index=True) invalid_report_days = IntegerField(default=0) #some one reported it is a invalid link - last_invalid_report_time = DateTimeField(default=datetime.datetime.utcnow) #a rss will be deleted after some days of reported_invalid + last_invalid_report_time = DateTimeField(default=utcnow) #a rss will be deleted after some days of reported_invalid custom = JSONField(default=JSONField.dict_default) #返回数据库中所有的分类 @@ -275,7 +275,7 @@ def categories(self): class SharedRssCategory(MyBaseModel): name = CharField() language = CharField(default='') - last_updated = DateTimeField(default=datetime.datetime.utcnow) + last_updated = DateTimeField(default=utcnow) class LastDelivered(MyBaseModel): user = CharField() @@ -283,7 +283,7 @@ class LastDelivered(MyBaseModel): url = CharField(default='') num = IntegerField(default=0) record = CharField(default='') - datetime = DateTimeField(default=datetime.datetime.utcnow) + datetime = DateTimeField(default=utcnow) class InBox(MyBaseModel): user = CharField() @@ -292,7 +292,7 @@ class InBox(MyBaseModel): subject = CharField() status = CharField() size = IntegerField(default=0) - datetime = DateTimeField(default=datetime.datetime.utcnow) + datetime = DateTimeField(default=utcnow) body = TextField(default='', index=False) attachments = CharField(default='') #存放UserBlob的数据库id，逗号分割 diff --git a/application/lib/calibre/web/feeds/news.py b/application/lib/calibre/web/feeds/news.py index 0afe77dc..f5b0e2c1 100644 --- a/application/lib/calibre/web/feeds/news.py +++ b/application/lib/calibre/web/feeds/news.py @@ -19,7 +19,7 @@ from calibre.ebooks.metadata.toc import TOC from calibre.ptempfile import PersistentTemporaryFile, PersistentTemporaryDirectory from calibre.utils.img import save_cover_data_to -from calibre.utils.date import now as nowf +from calibre.utils.date import now as nowf, utcnow from calibre.utils.localization import canonicalize_lang, ngettext from calibre.utils.threadpool import NoResultsPending, ThreadPool, WorkRequest from calibre.web import Recipe @@ -2210,7 +2210,7 @@ def parse_feeds(self): added = set() #这里oldest_article和其他的recipe不一样，这个参数表示在这个区间内不会重复推送 oldestSeconds = 24 * 3600 * self.oldest_article - now = datetime.datetime.utcnow() + now = utcnow() for obj in main_urls: #type:ignore main_title, main_url = (self.title, obj) if isinstance(obj, str) else obj feed = Feed() diff --git a/application/lib/calibre/web/fetch/simple.py b/application/lib/calibre/web/fetch/simple.py index 1a625e3b..16574347 100644 --- a/application/lib/calibre/web/fetch/simple.py +++ b/application/lib/calibre/web/fetch/simple.py @@ -17,6 +17,7 @@ import sys import time import traceback +from base64 import standard_b64decode from calibre import browser, relpath, unicode_path from calibre.constants import filesystem_encoding, iswindows from calibre.ebooks.BeautifulSoup import BeautifulSoup @@ -252,13 +253,18 @@ def fetch_url(self, url): ans = response(q) ans.newurl = url return ans - self.log.debug('Fetching', url) st = time.monotonic() + is_data_url = url.startswith('data:') + if not is_data_url: + self.log.debug('Fetching', url) # Check for a URL pointing to the local filesystem and special case it # for efficiency and robustness. Bypasses delay checking as it does not # apply to local fetches. Ensures that unicode paths that are not # representable in the filesystem_encoding work. + if is_data_url: + payload = url.partition(',')[2] + return standard_b64decode(payload) is_local = 0 if url.startswith('file://'): is_local = 7 diff --git a/application/lib/ebook_translator/html_translator.py b/application/lib/ebook_translator/html_translator.py index 48daa519..f3360384 100644 --- a/application/lib/ebook_translator/html_translator.py +++ b/application/lib/ebook_translator/html_translator.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding:utf-8 -*- # 调用在线翻译服务，翻译html文件，移植了calibre的 Ebook Translator 插件的在线翻译接口实现 -import time, copy +import re, time, copy from bs4 import BeautifulSoup, NavigableString from ebook_translator.engines import * from application.utils import loc_exc_pos @@ -108,17 +108,29 @@ def _contains_text(tag): return True return False + #过滤掉不需要翻译的tag + def _tag_is_filtered(tag): + return tag.name in ('pre', 'code', 'abbr', 'style', 'script', 'textarea', + 'input', 'select', 'link', 'img', 'option', 'datalist') + #递归函数，用于遍历BeautifulSoup元素的所有子节点并提取文本内容 - def _extract(tag): + #tag: 开始的BeautifulSoup元素 + #position: 翻译后的文本显示的位置 + def _extract(tag, position): for child in tag.find_all(recursive=False): - if _contains_text(child): + if _contains_text(child) and not _tag_is_filtered(child): text = str(child).strip() - if text and child.name not in ('pre', 'code', 'abbr', 'style', 'script', 'textarea', - 'input', 'select', 'link', 'img', 'option', 'datalist'): + if text: + #因为非AI翻译容易误翻译超链接里面的内容，所以这里去掉超链接 + if position != 'replace' and ']*>', '', text) + text = text.replace('', '') elements.append((child, text)) else: - _extract(child) - _extract(soup.body) + _extract(child, position) + + position = self.params.get('position', 'below') + _extract(soup.body, position) return elements #将翻译结果添加到DOM树 diff --git a/application/utils.py b/application/utils.py index eb59a3e5..e28b880e 100644 --- a/application/utils.py +++ b/application/utils.py @@ -75,6 +75,10 @@ def time_str(fmt="%Y-%m-%d %H:%M", tz=0): def tz_now(tz=0): return datetime.datetime.now(tz=datetime.timezone(datetime.timedelta(hours=tz))) +#python 3.12弃用datetime.datetime.utcnow()后使用此函数代替 +def utcnow(): + return datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) + #隐藏真实email地址，使用星号代替部分字符 #输入单个email字符串或列表，返回部分隐藏的字符串 def hide_email(email): diff --git a/application/view/admin.py b/application/view/admin.py index 790143a1..6f9fcef5 100644 --- a/application/view/admin.py +++ b/application/view/admin.py @@ -8,7 +8,7 @@ from flask_babel import gettext as _ from ..base_handler import * from ..back_end.db_models import * -from ..utils import str_to_int +from ..utils import str_to_int, utcnow from .login import CreateAccountIfNotExist bpAdmin = Blueprint('bpAdmin', __name__) @@ -169,7 +169,7 @@ def AdminAccountChangePost(name: str, user: KeUser): dbItem.expiration_days = expiration if expiration: - dbItem.expires = datetime.datetime.utcnow() + datetime.timedelta(days=expiration) + dbItem.expires = utcnow() + datetime.timedelta(days=expiration) else: dbItem.expires = None if smType == 'admin': diff --git a/application/view/adv.py b/application/view/adv.py index 8f15c334..923fd671 100644 --- a/application/view/adv.py +++ b/application/view/adv.py @@ -23,7 +23,7 @@ def adv_render_template(tpl, advCurr, **kwargs): kwargs.setdefault('tab', 'advset') kwargs.setdefault('tips', '') kwargs.setdefault('adminName', app.config['ADMIN_NAME']) - print(kwargs.get('tips')) + #print(kwargs.get('tips')) return render_template(tpl, advCurr=advCurr, **kwargs) #现在推送 diff --git a/application/view/library_offical.py b/application/view/library_offical.py index 9159a241..0853b0b7 100644 --- a/application/view/library_offical.py +++ b/application/view/library_offical.py @@ -7,7 +7,7 @@ from flask import Blueprint, request, Response, current_app as app from flask_babel import gettext as _ from ..base_handler import * -from ..utils import str_to_bool, str_to_int +from ..utils import str_to_bool, str_to_int, utcnow from ..back_end.db_models import * #几个"官方"服务的地址 @@ -109,7 +109,7 @@ def SharedLibraryOfficalAjax(): if category: dbItem = SharedRssCategory.get_or_none(SharedRssCategory.name == category) if dbItem: - dbItem.last_updated = datetime.datetime.utcnow() + dbItem.last_updated = utcnow() dbItem.save() else: SharedRssCategory.create(name=category, language=lang) @@ -122,13 +122,13 @@ def SharedLibraryOfficalAjax(): #更新共享库的最新时间信息 def UpdateLastSharedRssTime(): - now = datetime.datetime.utcnow() + now = utcnow() AppInfo.set_value(AppInfo.lastSharedRssTime, str(int(now.timestamp()))) #共享库的订阅源信息管理 @bpLibraryOffical.post(LIBRARY_MGR + "") def SharedLibraryMgrOffical(mgrType): - now = datetime.datetime.utcnow() + now = utcnow() form = request.form if mgrType == LIBRARY_GETSRC: #获取一个共享recipe的源代码 diff --git a/application/view/login.py b/application/view/login.py index 52b049f8..bab9946f 100644 --- a/application/view/login.py +++ b/application/view/login.py @@ -9,7 +9,7 @@ from ..base_handler import * from ..back_end.db_models import * from ..back_end.send_mail_adpt import send_html_mail -from ..utils import new_secret_key, hide_email, PasswordManager +from ..utils import new_secret_key, hide_email, PasswordManager, utcnow bpLogin = Blueprint('bpLogin', __name__) @@ -54,7 +54,7 @@ def LoginPost(): session['userName'] = name session['role'] = 'admin' if name == adminName else 'user' if user.expires and user.expiration_days > 0: #用户登陆后自动续期 - user.expires = datetime.datetime.utcnow() + datetime.timedelta(days=user.expiration_days) + user.expires = utcnow() + datetime.timedelta(days=user.expiration_days) user.save() if 'resetpwd' in user.custom: #成功登录后清除复位密码的设置 user.set_custom('resetpwd', None) @@ -109,7 +109,7 @@ def CreateAccountIfNotExist(name, password=None, email='', sender=None, sm_servi user = KeUser(name=name, passwd_hash=pwdHash, expires=None, expiration_days=expiration, share_links={'key': shareKey}, base_config=base_config, send_mail_service=sm_service) if expiration: - user.expires = datetime.datetime.utcnow() + datetime.timedelta(days=expiration) + user.expires = utcnow() + datetime.timedelta(days=expiration) user.save() return True @@ -139,7 +139,7 @@ def ResetPasswordGet(): user = KeUser.get_or_none(KeUser.name == name) if name else None if user and token: #重设密码的最后一步 pre_set = user.custom.get('resetpwd', {}) - now = datetime.datetime.utcnow() + now = utcnow() pre_time = pre_set.get('expires') or (now - datetime.timedelta(hours=1)).timestamp() if (token == pre_set.get('token')) and (now.timestamp() < pre_time): return render_template('reset_password.html', tips='', userName=name, firstStep=False, @@ -178,7 +178,7 @@ def ResetPasswordPost(): return render_template('reset_password.html', tips=tips, userName=name, firstStep=True) else: token = new_secret_key(length=24) - expires = (datetime.datetime.utcnow() + datetime.timedelta(days=1)).timestamp() + expires = (utcnow() + datetime.timedelta(days=1)).timestamp() user.set_custom('resetpwd', {'token': token, 'expires': expires}) user.save() tips = send_resetpwd_email(user, token) @@ -261,7 +261,7 @@ def send_resetpwd_email(user, token): #重置密码的最后一步，校验密码，写入数据库 def reset_pwd_final_step(user, token, new_p1, new_p2): pre_set = user.custom.get('resetpwd', {}) - now = datetime.datetime.utcnow() + now = utcnow() pre_time = pre_set.get('expires') or (now - datetime.timedelta(hours=1)).timestamp() if (token == pre_set.get('token')) and (now.timestamp() < pre_time): if new_p1 == new_p2: diff --git a/application/view/logs.py b/application/view/logs.py index 1cf17f82..a1d75bc5 100644 --- a/application/view/logs.py +++ b/application/view/logs.py @@ -8,6 +8,7 @@ from flask_babel import gettext as _ from ..base_handler import * from ..back_end.db_models import * +from ..utils import utcnow bpLogs = Blueprint('bpLogs', __name__) @@ -35,7 +36,7 @@ def RemoveLogsRoute(): def RemoveLogs(): ret = [] #停止过期用户的推送 - now = datetime.datetime.utcnow() + now = utcnow() cnt = 0 for user in KeUser.select(): if user.cfg('enable_send') and user.expires and (user.expires < now): @@ -68,7 +69,7 @@ def GetOrderedDeliverLog(userName, limit): #删除过期的收件箱内容 def RemoveOldEmail(user: KeUser): cnt = 0 - now = datetime.datetime.utcnow() + now = utcnow() inbound_email = user.cfg('inbound_email') keep_in_email_days = user.cfg('keep_in_email_days') if ('save' in inbound_email) and keep_in_email_days: diff --git a/application/view/subscribe.py b/application/view/subscribe.py index c38250b7..410944bd 100644 --- a/application/view/subscribe.py +++ b/application/view/subscribe.py @@ -9,7 +9,7 @@ from ..base_handler import * from ..back_end.db_models import * from ..back_end.task_queue_adpt import create_notifynewsubs_task -from ..utils import str_to_bool, xml_escape +from ..utils import str_to_bool, xml_escape, utcnow from ..lib.urlopener import UrlOpener from ..lib.recipe_helper import GetBuiltinRecipeInfo, GetBuiltinRecipeSource from .library import LIBRARY_MGR, SUBSCRIBED_FROM_LIBRARY, LIBRARY_GETSRC, buildKeUrl @@ -205,7 +205,7 @@ def UpdateBookedCustomRss(user: KeUser): if user.cfg('enable_send') == 'all': #添加自定义RSS的订阅 for rss in custom_rss: BookedRecipe.get_or_create(recipe_id=rss.recipe_id, defaults={'user': userName, - 'title': rss.title, 'description': rss.description, 'time': datetime.datetime.utcnow(), + 'title': rss.title, 'description': rss.description, 'time': utcnow(), 'translator': rss.translator, 'tts': rss.tts, 'custom': rss.custom, 'separated': rss.custom.get('separated', False)}) elif custom_rss: #删除订阅 @@ -351,7 +351,7 @@ def SaveRecipeIfCorrect(user: KeUser, src: str): raise Exception(_('The recipe is already in the library.')) params = {"title": recipe.title, "description": recipe.description, "type_": 'upload', - "needs_subscription": recipe.needs_subscription, "src": src, "time": datetime.datetime.utcnow(), + "needs_subscription": recipe.needs_subscription, "src": src, "time": utcnow(), "user": user.name, "language": recipe.language} dbInst = Recipe.create(**params) params.pop('src')