From 86badd758d87a67965769632bdb3bbbf330256b8 Mon Sep 17 00:00:00 2001 From: cdhigh Date: Sat, 23 Mar 2024 23:24:17 -0300 Subject: [PATCH] porting to python3 --- application/back_end/db_models.py | 16 +- application/back_end/db_models_sql.py | 5 +- application/lib/calibre/web/feeds/__init__.py | 90 +++- application/lib/calibre/web/feeds/news.py | 9 +- application/routes.py | 25 +- application/templates/adv_base.html | 10 +- .../templates/adv_calibre_options.html | 2 +- application/templates/adv_delivernow.html | 4 +- application/templates/adv_whitelist.html | 2 +- application/view/adv.py | 36 +- application/view/email_filter.py | 23 + application/view/inbound_email.py | 435 ++++++++++-------- application/view/library_offical.py | 27 +- docs/Chinese/2.config.md | 9 - docs/Chinese/3.deployment.md | 9 +- docs/English/2.config.md | 8 - docs/English/3.deployment.md | 10 +- requirements.txt | 15 +- tests/runtests.py | 13 +- tests/test_admin.py | 4 +- tests/test_adv.py | 14 +- tests/test_base.py | 4 +- tests/test_inbound_email.py | 79 ++-- tests/test_login.py | 2 +- tests/test_logs.py | 2 +- tests/test_subscribe.py | 3 +- tools/update_req.py | 3 +- 27 files changed, 485 insertions(+), 374 deletions(-) create mode 100644 application/view/email_filter.py diff --git a/application/back_end/db_models.py b/application/back_end/db_models.py index dd84194b..7ec1fb89 100644 --- a/application/back_end/db_models.py +++ b/application/back_end/db_models.py @@ -215,7 +215,7 @@ def categories(self): #Buffer for category of shared rss [for kindleear.appspot.com only] class SharedRssCategory(MyBaseModel): name = CharField() - language = CharField() + language = CharField(default='') last_updated = DateTimeField(default=datetime.datetime.utcnow) class LastDelivered(MyBaseModel): @@ -225,6 +225,16 @@ class LastDelivered(MyBaseModel): record = CharField(default='') datetime = DateTimeField(default=datetime.datetime.utcnow) +class InBox(MyBaseModel): + user = CharField() + sender = CharField() + to = CharField() + subject = CharField() + size = IntegerField(default=0) + datetime = DateTimeField(default=datetime.datetime.utcnow) + body = TextField(default='', index=False) + attachments = CharField(default='') #存放UserBlob的数据库id，逗号分割 + class AppInfo(MyBaseModel): name = CharField(unique=True) value = CharField(default='') @@ -251,9 +261,9 @@ def create_database_tables(): #with dbInstance.connection_context(): #connect_database() dbInstance.create_tables([KeUser, UserBlob, Recipe, BookedRecipe, DeliverLog, WhiteList, - SharedRss, SharedRssCategory, LastDelivered, AppInfo], safe=True) + SharedRss, SharedRssCategory, LastDelivered, InBox, AppInfo], safe=True) if not AppInfo.get_value(AppInfo.dbSchemaVersion): AppInfo.set_value(AppInfo.dbSchemaVersion, appVer) #close_database() - + return 'Created database tables successfully' diff --git a/application/back_end/db_models_sql.py b/application/back_end/db_models_sql.py index 2c017c98..da1b9f4c 100644 --- a/application/back_end/db_models_sql.py +++ b/application/back_end/db_models_sql.py @@ -16,7 +16,10 @@ if not fileName.startswith('/'): dbName = 'sqlite:///{}'.format(os.path.join(appDir, fileName)) -dbInstance = connect(dbName) +if dbName == 'sqlite://:memory:': + dbInstance = SqliteDatabase(':memory:') +else: + dbInstance = connect(dbName) #调用此函数正式连接到数据库（打开数据库） def connect_database(): diff --git a/application/lib/calibre/web/feeds/__init__.py b/application/lib/calibre/web/feeds/__init__.py index e46bfa39..55604448 100644 --- a/application/lib/calibre/web/feeds/__init__.py +++ b/application/lib/calibre/web/feeds/__init__.py @@ -10,6 +10,8 @@ import re import time import traceback +import json +import datetime from calibre import entity_to_unicode, force_unicode, strftime from calibre.utils.cleantext import clean_ascii_chars, clean_xml_chars @@ -56,7 +58,7 @@ def __init__(self, id, title, url, author, summary, published, content): self.text_summary = clean_ascii_chars(summary) self.author = author self.content = content - self.date = published + self.date = published #time.struct_time self.utctime = dt_factory(self.date, assume_utc=True, as_utc=True) self.localtime = self.utctime.astimezone(local_tz) self._formatted_date = None @@ -143,6 +145,27 @@ def populate_from_feed(self, feed, title=None, oldest_article=7, break self.parse_article(item) + #added by cdhigh + def populate_from_json(self, feed, title=None, oldest_article=7, max_articles_per_feed=100): + self.title = feed.get('title', _('Unknown section')) if not title else title + self.description = feed.get('description', '') + self.image_url = feed.get('icon', None) or feed.get('favicon', None) + self.image_width = 88 + self.image_height = 31 + self.image_alt = '' + + self.articles = [] + self.id_counter = 0 + self.added_articles = [] + + self.oldest_article = oldest_article + + entries = feed.get('items') + for item in entries: + if len(self.articles) >= max_articles_per_feed: + break + self.parse_article_json(item) + def populate_from_preparsed_feed(self, title, articles, oldest_article=7, max_articles_per_feed=100): self.title = str(title if title else _('Unknown feed')) @@ -240,6 +263,50 @@ def parse_article(self, item): title = title.decode('utf-8', 'replace') self.logger.debug('Skipping article %s as it is too old'%title) + #added by cdhigh + def parse_article_json(self, item): + self.id_counter += 1 + id_ = item.get('id', None) or f'internal id#{self.id_counter}' + if id_ in self.added_articles: + return + + self.added_articles.append(id_) + published = item.get('date_modified', None) or item.get('date_published', None) + if published: + import dateutil + try: + published = dateutil.parser.parse(published).timetuple() + except: + published = time.gmtime() + else: + published = time.gmtime() + + title = item.get('title', _('Untitled article')) + if title.startswith('<'): + title = re.sub(r'<.+?>', '', title) + link = item.get('url', None) or item.get('external_url', None) + description = item.get('summary', None) + authors = item.get('authors', []) + author = ' '.join([aut.get('name', '') for aut in authors]) + if not author: + author = item.get('author', {}).get('name', '') + content = item.get('content_html', None) or item.get('content_text', None) + if not link and not content: + return + + article = Article(id_, title, link, author, description, published, content) + delta = utcnow() - article.utctime + if (self.oldest_article == 0) or (delta.days*24*3600 + delta.seconds <= 24*3600*self.oldest_article): + self.articles.append(article) + else: + try: + self.logger.debug('Skipping article %s (%s) from feed %s as it is too old.'% + (title, article.localtime.strftime('%a, %d %b, %Y %H:%M'), self.title)) + except UnicodeDecodeError: + if not isinstance(title, str): + title = title.decode('utf-8', 'replace') + self.logger.debug('Skipping article %s as it is too old'%title) + def reverse(self): self.articles.reverse() @@ -343,11 +410,12 @@ def feed_from_xml(raw_xml, title=None, oldest_article=7, max_articles_per_feed=100, get_article_url=lambda item: item.get('link', None), log=default_log): + from feedparser import parse # Handle unclosed escaped entities. They trip up feedparser and HBR for one # generates them - raw_xml = re.sub(br'(&#\d+)([^0-9;])', br'\1;\2', raw_xml) + raw_xml = re.sub(r'(&#\d+)([^0-9;])', r'\1;\2', raw_xml) feed = parse(raw_xml) pfeed = Feed(get_article_url=get_article_url, log=log) pfeed.populate_from_feed(feed, title=title, @@ -355,6 +423,24 @@ def feed_from_xml(raw_xml, title=None, oldest_article=7, max_articles_per_feed=max_articles_per_feed) return pfeed +#added by cdhigh +def feed_from_json(raw_json, title=None, oldest_article=7, + max_articles_per_feed=100, + get_article_url=lambda item: item.get('link', None), + log=default_log): + + pfeed = Feed(get_article_url=get_article_url, log=log) + + try: + feed = json.loads(raw_json) + except Exception as e: + log.warning('Parse json feed failed {}: {}'.format(title, str(e))) + return pfeed + + pfeed.populate_from_json(feed, title=title, + oldest_article=oldest_article, + max_articles_per_feed=max_articles_per_feed) + return pfeed def feeds_from_index(index, oldest_article=7, max_articles_per_feed=100, log=default_log): diff --git a/application/lib/calibre/web/feeds/news.py b/application/lib/calibre/web/feeds/news.py index 59eaec12..b2dea0b7 100644 --- a/application/lib/calibre/web/feeds/news.py +++ b/application/lib/calibre/web/feeds/news.py @@ -29,7 +29,7 @@ from calibre.utils.logging import ThreadSafeWrapper from calibre.utils.threadpool import NoResultsPending, ThreadPool, WorkRequest from calibre.web import Recipe -from calibre.web.feeds import Feed, Article, feed_from_xml, feeds_from_index, templates +from calibre.web.feeds import Feed, Article, feed_from_xml, feeds_from_index, templates, feed_from_json from calibre.web.fetch.simple import AbortArticle, RecursiveFetcher from calibre.web.fetch.utils import prepare_masthead_image from polyglot.builtins import string_or_bytes @@ -1853,9 +1853,10 @@ def parse_feeds(self): # br.add_password(url, purl.username, purl.password) resp = br.open(url, timeout=self.timeout) if resp.status_code == 200: - raw = resp.content - feed = feed_from_xml(raw, title=title, log=self.log, oldest_article=self.oldest_article, - max_articles_per_feed=self.max_articles_per_feed, get_article_url=self.get_article_url) + raw = resp.text.lstrip() + pFunc = feed_from_json if raw and raw[0] == '{' else feed_from_xml + feed = pFunc(raw, title=title, log=self.log, oldest_article=self.oldest_article, + max_articles_per_feed=self.max_articles_per_feed, get_article_url=self.get_article_url) parsed_feeds.append(feed) else: raise Exception(f'Cannot fetch {url}:{resp.status_code}') diff --git a/application/routes.py b/application/routes.py index cbeb394e..c7d638c3 100644 --- a/application/routes.py +++ b/application/routes.py @@ -3,18 +3,9 @@ #主页和其他路由 import os from flask import Blueprint, render_template, send_from_directory, current_app -from .view import login -from .view import admin -from .view import adv -from .view import deliver -from .view import library -from .view import library_offical -from .view import logs -from .view import setting -from .view import share -from .view import subscribe -from .work import worker -from .work import url2book +from .view import (login, admin, adv, deliver, library, library_offical, logs, setting, share, + subscribe, inbound_email) +from .work import worker, url2book bpHome = Blueprint('bpHome', __name__) @@ -62,10 +53,10 @@ def register_routes(app): app.register_blueprint(worker.bpWorker) app.register_blueprint(url2book.bpUrl2Book) app.register_blueprint(library_offical.bpLibraryOffical) + app.register_blueprint(inbound_email.bpInBoundEmail) - #使用GAE来接收邮件 - if app.config['INBOUND_EMAIL_SERVICE'] == 'gae': + #启用GAE邮件服务如果部署在GAE平台 + if app.config['DATABASE_URL'] == 'datastore': from google.appengine.api import wrap_wsgi_app - from application.view.inbound_email import bpInBoundEmail - app.wsgi_app = wrap_wsgi_app(app.wsgi_app) #启用GAE邮件服务 - app.register_blueprint(bpInBoundEmail) + app.wsgi_app = wrap_wsgi_app(app.wsgi_app) + diff --git a/application/templates/adv_base.html b/application/templates/adv_base.html index d1e626b4..bfca7277 100644 --- a/application/templates/adv_base.html +++ b/application/templates/adv_base.html @@ -36,14 +36,13 @@ diff --git a/application/templates/adv_calibre_options.html b/application/templates/adv_calibre_options.html index 1b3ea29b..e211bb4f 100644 --- a/application/templates/adv_calibre_options.html +++ b/application/templates/adv_calibre_options.html @@ -9,7 +9,7 @@ {% if tips -%}
{{tips|safe}}
{% endif -%} -

{{_("Calibre options")}}

 +

{{_("Calibre Options")}}

{{_("Set the parameters for Calibre, in JSON dictionary format.")}}

diff --git a/application/templates/adv_delivernow.html b/application/templates/adv_delivernow.html index a421c851..4e018ac6 100644 --- a/application/templates/adv_delivernow.html +++ b/application/templates/adv_delivernow.html @@ -5,8 +5,8 @@ {% block advcontent -%}
-

{{ _("Deliver now") }}

 -

{{ _("Deliver selected recipes now") }}

+

{{ _("Deliver Now") }}

 +

{{ _("Deliver selected recipes now.") }}

{% if recipes|length == 0 -%}
{{ _("There are no recipes subscribed") }}
diff --git a/application/templates/adv_whitelist.html b/application/templates/adv_whitelist.html index 18e09ce1..919d918e 100644 --- a/application/templates/adv_whitelist.html +++ b/application/templates/adv_whitelist.html @@ -12,7 +12,7 @@

{{_("White List")}}

- {{_("Emails sent to %(name)sxxx@appid.appspotmail.com will be transferred to your email.", name='' if (user.name == adminName) else user.name + '__')}} + {{_("Emails sent to %(name)sxxx@%(mailHost)s will be transferred to your kindle email.", name='' if (user.name == adminName) else user.name + '__', mailHost=mailHost)}}

{%- for wl in user.white_lists() -%} diff --git a/application/view/adv.py b/application/view/adv.py index f6a34316..b0877e5a 100644 --- a/application/view/adv.py +++ b/application/view/adv.py @@ -3,7 +3,7 @@ #一些高级设置功能页面 import datetime, hashlib, io -from urllib.parse import quote, unquote, urljoin +from urllib.parse import quote, unquote, urljoin, urlparse from flask import Blueprint, url_for, render_template, redirect, session, send_file, abort, current_app as app from flask_babel import gettext as _ from PIL import Image @@ -19,7 +19,6 @@ def adv_render_template(tpl, advCurr, **kwargs): kwargs.setdefault('tab', 'advset') kwargs.setdefault('tips', '') kwargs.setdefault('adminName', app.config['ADMIN_NAME']) - kwargs.setdefault('in_email_service', app.config['INBOUND_EMAIL_SERVICE']) return render_template(tpl, advCurr=advCurr, **kwargs) #现在推送 @@ -35,28 +34,27 @@ def AdvDeliverNow(): @bpAdv.route("/adv/whitelist", endpoint='AdvWhiteList') @login_required() def AdvWhiteList(): - if app.config['INBOUND_EMAIL_SERVICE'] == 'gae': - user = get_login_user() - return adv_render_template('adv_whitelist.html', 'whitelist', user=user) + user = get_login_user() + if app.config['DATABASE_URL'] == 'datastore': + mailHost = 'appid.appspotmail.com' else: - abort(404) - + mailHost = urlparse(app.config['APP_DOMAIN']).netloc.split(':')[0] + + return adv_render_template('adv_whitelist.html', 'whitelist', user=user, mailHost=mailHost) + @bpAdv.post("/adv/whitelist", endpoint='AdvWhiteListPost') @login_required() def AdvWhiteListPost(): - if app.config['INBOUND_EMAIL_SERVICE'] == 'gae': - user = get_login_user() - wlist = request.form.get('wlist') + user = get_login_user() + wlist = request.form.get('wlist') + if wlist: + wlist = wlist.replace('"', "").replace("'", "").strip() + if wlist.startswith('*@'): #输入*@xx.xx则修改为@xx.xx + wlist = wlist[1:] if wlist: - wlist = wlist.replace('"', "").replace("'", "").strip() - if wlist.startswith('*@'): #输入*@xx.xx则修改为@xx.xx - wlist = wlist[1:] - if wlist: - WhiteList.get_or_create(mail=wlist, user=user.name) - return redirect(url_for('bpAdv.AdvWhiteList')) - else: - abort(404) - + WhiteList.get_or_create(mail=wlist, user=user.name) + return redirect(url_for('bpAdv.AdvWhiteList')) + #删除白名单项目 @bpAdv.route("/advdel", endpoint='AdvDel') @login_required() diff --git a/application/view/email_filter.py b/application/view/email_filter.py new file mode 100644 index 00000000..e4cbac94 --- /dev/null +++ b/application/view/email_filter.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +"""linux平台的postfix的content filter，将其注册为postfix的content filter即可在VPS上实现类似部署在GAE的邮件中转功能。 +此文件是受 启发而编写的。 +核心 /etc/postfix/master.cf 配置行 +mycustomfilter unix - n n - - pipe + flags=F user=your_user argv=/path/to/email_filter.py +""" +import sys, requests + +def process_email(): + data = sys.stdin.buffer.read() + headers = {'Content-Type': 'application/octet-stream'} + res = requests.post(url='http://172.0.0.1/mail', data=data, headers=headers) + + #错误码参考： + #EX_UNAVAILABLE: 69, postfix会将邮件退回 + #EX_TEMPFAIL: 75, postfix之后会尝试再次投递 + #0: 成功 + return 1 #让postfix丢弃邮件 + +if __name__ == "__main__": + sys.exit(process_email()) diff --git a/application/view/inbound_email.py b/application/view/inbound_email.py index 8ef7a805..28b3003c 100644 --- a/application/view/inbound_email.py +++ b/application/view/inbound_email.py @@ -3,192 +3,285 @@ #Author: cdhigh #将发到string@appid.appspotmail.com的邮件正文转成附件发往kindle邮箱。 -import re, io +import os, sys, re, email from urllib.parse import urljoin -from email.header import decode_header -from email.utils import parseaddr, collapse_rfc2231_value from bs4 import BeautifulSoup -from flask import Blueprint, request, current_app as app -from google.appengine.api import mail +from flask import Blueprint, request from calibre import guess_type from ..back_end.task_queue_adpt import create_delivery_task, create_url2book_task -from ..back_end.db_models import KeUser, WhiteList +from ..back_end.db_models import KeUser from ..back_end.send_mail_adpt import send_to_kindle from ..base_handler import * from ..utils import local_time from build_ebook import html_to_book -bpInBoundEmail = Blueprint('bpInBoundEmail', __name__) - -#subject of email will be truncated based limit of word count -SUBJECT_WORDCNT = 30 - -#if word count more than the number, the email received by appspotmail will -#be transfered to kindle directly, otherwise, will fetch the webpage for links in email. -WORDCNT_THRESHOLD_APMAIL = 100 - -#clean css in dealing with content from string@appid.appspotmail.com or not -DELETE_CSS_APMAIL = True - -#解码邮件主题 -def decode_subject(subject): - if subject.startswith('=?') and subject.endswith('?='): - subject = ''.join(str(s, c or 'us-ascii') for s, c in decode_header(subject)) - else: - subject = str(collapse_rfc2231_value(subject)) - return subject - -#判断一个字符串是否是超链接，返回链接本身，否则空串 -def IsHyperLink(txt): - expr = r"""^(?i)\b((?:https?://|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:'".,<>???“”‘’]))""" - match = re.match(expr, txt) - return match.group() if match else '' - -#从接收地址提取账号名和真实地址 -#如果有多个收件人的话，只解释第一个收件人 -def extractUsernameFromEmail(to): - to = parseaddr(to)[1] - to = to.split('@')[0] if to and '@' in to else 'xxx' - if '__' in to: - userNameParts = to.split('__') - userName = userNameParts[0] if userNameParts[0] else app.config['ADMIN_NAME'] - return userName, userNameParts[1] - else: - return app.config['ADMIN_NAME'], to +try: + from google.appengine.api import mail as gae_mail +except ImportError: + gae_mail = None -#判断是否是垃圾邮件 -#sender: 发件人地址 -#user: 用户账号数据库行实例 -def IsSpamMail(sender, user): - if not sender or '@' not in sender: - return True +bpInBoundEmail = Blueprint('bpInBoundEmail', __name__) - mailHost = sender.split('@')[1] - whitelist = [item.mail.lower() for item in user.white_lists()] +#主题最长字数 +SUBJECT_WORDCNT = 50 - return not (('*' in whitelist) or (sender.lower() in whitelist) or (f'@{mailHost}' in whitelist)) +#除非强制提取链接，否则超过这个字数的邮件直接转发内容 +WORDCNT_THRESHOLD_APMAIL = 300 #GAE的退信通知 @bpInBoundEmail.post("/_ah/bounce") -def ReceiveBounce(): - msg = mail.BounceNotification(dict(request.form.lists())) +def ReceiveGaeBounce(): + msg = gae_mail.BounceNotification(dict(request.form.lists())) default_log.warning("Bounce original: {}, notification: {}".format(msg.original, msg.notification)) - return "OK", 200 + return "OK" -#有新的邮件到达, _ah=apphosting +#有新的GAE邮件到达, _ah=apphosting #每个邮件限额: 31.5 MB @bpInBoundEmail.post("/_ah/mail/") -def ReceiveMail(path): - global default_log - log = default_log +def ReceiveGaeMail(path): + message = gae_mail.InboundEmailMessage(request.get_data()) + + subject = message.subject if hasattr(message, 'subject') else 'NoSubject' + + try: + txtBodies = [body.decode() for cType, body in message.bodies('text/plain')] + except: + txtBodies = [] + + try: + htmlBodies = [body.decode() for cType, body in message.bodies('text/html')] + except: + htmlBodies = [] + + attachments = message.attachments if hasattr(message, 'attachments') else [] + + return ReceiveMailImpl(sender=message.sender, to=message.to, subject=subject, txtBodies=txtBodies, + htmlBodies=htmlBodies, attachments=attachments) + +#postfix的content filter转发的邮件，适用与除GAE之外的平台 +@bpInBoundEmail.post("/mail") +def ReceiveMail(): + msg = email.message_from_bytes(request.get_data()) + + txtBodies = [] + htmlBodies = [] + attachments = [] + for part in msg.walk(): + cType = part.get_content_type() + body = part.get_payload(decode=True) + if part.get('Content-Disposition') == 'attachment': + attachments.append((part.get_filename(), body)) + elif cType == 'text/plain': + txtBodies.append(body.decode(part.get_content_charset('us-ascii'))) + elif cType == 'text/html': + htmlBodies.append(body.decode(part.get_content_charset('us-ascii'))) - message = mail.InboundEmailMessage(request.get_data()) - userName, to = extractUsernameFromEmail(message.to) #从接收地址提取账号名和真实地址 - adminName = app.config['ADMIN_NAME'] + return ReceiveMailImpl(sender=msg.get('From'), to=msg.get_all('To'), subject=msg.get('Subject'), txtBodies=txtBodies, + htmlBodies=htmlBodies, attachments=attachments) #实际实现接收邮件功能 -def ReceiveMailImpl(): - user = KeUser.get_or_none(KeUser.name == (userName or adminName)) +#sender/to/subject: 发件人，收件人，主题 +#txtBodies: text/plain的邮件内容列表 +#htmlBodies: text/html的邮件内容列表 +#attachements: 附件列表，格式为[(fileName, content),...] +def ReceiveMailImpl(sender, to, subject, txtBodies, htmlBodies, attachments): + adminName = os.getenv('ADMIN_NAME') + userName, to = ExtractUsernameFromEmail(to) #从接收地址提取账号名和真实地址, 格式：user__to + userName = userName if userName else adminName + + user = KeUser.get_or_none(KeUser.name == userName) if not user and (userName != adminName): user = KeUser.get_or_none(KeUser.name == adminName) if not user or not user.kindle_email: - return "OK" + return "The user does not exists" #阻挡垃圾邮件 - sender = parseaddr(message.sender)[1] - if IsSpamMail(sender, user): - log.warning('Spam mail from : {}'.format(sender)) - return "Spam mail!" + sender = email.utils.parseaddr(sender)[1] + if IsSpamMail(user, sender): + default_log.warning(f'Spam mail from : {sender}') + return "Spam mail" - if hasattr(message, 'subject'): - subject = decode_subject(message.subject).strip() - else: - subject = "NoSubject" + subject = DecodeSubject(subject) + + #通过邮件触发一次“现在投递” + if to.lower() == 'trigger': + create_delivery_task({'userName': userName, 'recipeId': subject}) + return f'A delivery task for "{userName}" is triggered' forceToLinks = False forceToArticle = False - #邮件主题中如果在最后添加一个 !links，则强制提取邮件中的链接然后生成电子书 + #邮件主题中如果存在 !links，则强制提取邮件中的链接然后生成电子书 if subject.endswith('!links') or ' !links ' in subject: - subject = subject.replace('!links', '').replace(' !links ', '').strip() + subject = subject.replace(' !links ', '').replace('!links', '').strip() forceToLinks = True - # 如果邮件主题在最后添加一个 !article，则强制转换邮件内容为电子书，忽略其中的链接 + # 如果主题存在 !article，则强制转换邮件内容为电子书，忽略其中的链接 elif subject.endswith('!article') or ' !article ' in subject: - subject = subject.replace('!article', '').replace(' !article ', '').strip() + subject = subject.replace(' !article ', '').replace('!article', '').strip() forceToArticle = True + + soup = CreateMailSoup(subject, txtBodies, htmlBodies) + if not soup: + return "There is no html body neither text body." + + #提取文章的超链接 + links = [] if forceToArticle else CollectSoupLinks(soup, forceToLinks) - #通过邮件触发一次“现在投递” - if to.lower() == 'trigger': - create_delivery_task({'userName': userName, 'recipeId': subject}) - return f'A delivery task for "{userName}" is triggered' + if links: + #判断是下载文件还是要转发邮件内容 + isBook = ((to.lower() in ('book', 'file', 'download')) or + links[0].lower().endswith(('.mobi', '.epub', '.docx', '.pdf', '.txt', '.doc', '.rtf'))) + + if to.lower() == 'debug': + action = 'debug' + elif isBook: + action = 'download' + else: + action = '' + + params = {'userName': userName, + 'urls': '|'.join(links), + 'action': action, + 'key': user.share_links.get('key', ''), + 'title': subject[:SUBJECT_WORDCNT]} + create_url2book_task(params) + else: #直接转发邮件正文 + #只处理图像，忽略其他类型的附件 + #guess_type返回元祖 (type, encoding) + imgs = [(f, c) for f, c in (attachments or []) if (guess_type(f)[0] or '').startswith('image/')] + + #有图像的话，生成MOBI或EPUB，没有图像则直接推送HTML文件 + if imgs: + book = html_to_book(str(soup), subject[:SUBJECT_WORDCNT], user, imgs) + else: + book = (f'KindleEar_{local_time("%Y-%m-%d_%H-%M", user.timezone)}.html', str(soup).encode('utf-8')) + + send_to_kindle(user, subject[:SUBJECT_WORDCNT], book, fileWithTime=False) - #获取和解码邮件内容 - txtBodies = message.bodies('text/plain') - try: - allBodies = [body.decode() for cType, body in message.bodies('text/html')] - except: - log.warning('Decode html bodies of mail failed.') - allBodies = [] + return 'OK' + + +#解码邮件主题 +def DecodeSubject(subject): + if not subject: + subject = 'NoSubject' + elif subject.startswith('=?') and subject.endswith('?='): + subject = ''.join(str(s, c or 'us-ascii') for s, c in email.header.decode_header(subject)) + else: + subject = str(email.utils.collapse_rfc2231_value(subject)) + return subject.strip() + +#判断一个字符串是否是超链接 +def IsHyperLink(txt): + expr = r"""^\b((?:https?://|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:'".,<>???“”‘’]))""" + return re.match(expr, txt, re.IGNORECASE) + +#从接收地址提取账号名和真实地址 +#如果有多个收件人的话，只解释第一个收件人 +#返回 (userName, to的@前面的部分) +def ExtractUsernameFromEmail(to): + to = email.utils.parseaddr(to)[1] + to = (to or 'xxx').split('@')[0] - #此邮件为纯文本邮件，将文本信息转换为HTML格式 - if not allBodies: - log.info('There is no html body, use text body instead.') - try: - allBodies = [body.decode() for cType, body in txtBodies] - except: - log.warning('Decode text bodies of mail failed.') - allBodies = [] - bodies = ''.join(allBodies) - if not bodies: - return "There is no html body neither text body." - - bodyUrls = [] - for line in bodies.split('\n'): - line = line.strip() - if not line: - continue - link = IsHyperLink(line) - if link: - bodyUrls.append('{}
'.format(link, link)) - else: #有非链接行则终止，因为一般的邮件最后都有推广链接 - break - - bodies = """ - {}{}""".format(subject, - ''.join(bodyUrls) if bodyUrls else bodies) - allBodies = [bodies] + return to.split('__', 1) if '__' in to else ('', to) - #开始处理邮件内容 - soup = BeautifulSoup(allBodies[0], 'lxml') +#判断是否是垃圾邮件 +#user: 用户账号数据库行实例 +#sender: 发件人地址 +def IsSpamMail(user, sender): + if not sender or '@' not in sender: + return True + + host = sender.split('@')[-1] + whitelist = [item.mail.lower() for item in user.white_lists()] + + return not (('*' in whitelist) or (sender.lower() in whitelist) or (f'@{host}' in whitelist)) + + +#将邮件里面的纯文本内容转换为一个合法的html字符串 +def ConvertTextToHtml(subject, text): + if not text: + return '' + + #转换纯文本到html时需要将里面的文本链接变成tag a + bodyUrls = [] + for line in text.split('\n'): + line = line.strip() + if not line: + continue + if IsHyperLink(line): + bodyUrls.append(f'{line}
') + else: #有非链接行则终止，因为一般的邮件最后都有推广链接 + break + + text = ''.join(bodyUrls) if bodyUrls else text + + return f""" + {subject}{text}""" + +#根据邮件内容，创建一个 BeautifulSoup 实例 +def CreateMailSoup(subject, txtBodies, htmlBodies): + if not htmlBodies: #此邮件为纯文本邮件，将文本信息转换为HTML格式 + html = ConvertTextToHtml(subject, '\n'.join(txtBodies or [])) + htmlBodies = [html] if html else [] + + if not htmlBodies: + return None + + soup = BeautifulSoup(htmlBodies[0], 'lxml') + for other in htmlBodies[1:]: #合并多个邮件HTML内容 + tag = BeautifulSoup(other, 'lxml').find('body') + soup.body.extend(tag.contents if tag else []) + + #修正不规范的HTML邮件 + h = soup.find('head') + if not h: + h = soup.new_tag('head') + soup.html.insert(0, h) + t = soup.head.find('title') + if not t: + t = soup.new_tag('title') + t.string = subject + soup.head.insert(0, t) + + #删除CSS/JS + for tag in list(soup.find_all(['link', 'meta', 'style', 'script'])): + tag.extract() + for tag in soup.find_all(attrs={'style': True}): + del tag['style'] - #合并多个邮件文本段 - for otherBody in allBodies[1:]: - bodyOther = BeautifulSoup(otherBody, 'lxml').find('body') - soup.body.extend(bodyOther.contents if bodyOther else []) + #将图片的src的文件名修改正确，因为邮件中的图像可能会以cid:开头 + for tag in soup.find_all('img', attrs={'src': True}): + if tag['src'].lower().startswith('cid:'): + tag['src'] = tag['src'][4:] + + m = soup.new_tag('meta', attrs={"content": "text/html; charset=utf-8", "http-equiv": "Content-Type"}) + soup.html.head.insert(0, m) + return soup - #判断邮件内容是文本还是链接（包括多个链接的情况） +#提取Soup的超链接，返回一个列表 +#判断邮件内容是文本还是链接（包括多个链接的情况） +#forceToLinks: 不管文章内容如何，强制提取链接 +def CollectSoupLinks(soup, forceToLinks): links = [] - body = soup.body if soup.find('body') else soup - if not forceToArticle: #如果强制转正文就不分析链接了，否则先分析和提取链接 - for s in body.stripped_strings: - link = IsHyperLink(s) - if link: - if link not in links: - links.append(link) - #如果是多个链接，则必须一行一个，不能留空，除非强制提取链接 - #这个处理是为了去除部分邮件客户端在邮件末尾添加的一个广告链接 - elif not forceToLinks: - break + body = soup.body + for s in body.stripped_strings: + if IsHyperLink(s): + if s not in links: + links.append(s) + #如果是多个链接，则必须一行一个，不能留空，除非强制提取链接 + #这个处理是为了去除部分邮件客户端在邮件末尾添加的一个广告链接 + elif not forceToLinks: + break - if not links and not forceToArticle: #如果通过正常字符（显示出来的）判断没有链接，则看html的a标签 + if not links: #如果通过正常字符（显示出来的）判断没有链接，则看html的a标签 links = [link['href'] for link in soup.find_all('a', attrs={'href': True})] - text = ' '.join([s for s in body.stripped_strings]) - #如果有相对路径，则在里面找一个绝对路径，然后转换其他 hasRelativePath = False fullPath = '' + text = ' '.join([s for s in body.stripped_strings]) for link in links: text = text.replace(link, '') if not link.startswith('http'): @@ -204,72 +297,6 @@ def ReceiveMailImpl(): #如果字数太多，则认为直接推送正文内容 if not forceToLinks and (len(links) != 1 or len(text) > WORDCNT_THRESHOLD_APMAIL): links = [] - - if links: - #判断是下载文件还是转发内容 - isBook = bool(to.lower() in ('book', 'file', 'download')) - if not isBook: - isBook = bool(link[-5:].lower() in ('.mobi','.epub','.docx')) - if not isBook: - isBook = bool(link[-4:].lower() in ('.pdf','.txt','.doc','.rtf')) - isDebug = bool(to.lower() == 'debug') - - if isDebug: - action = 'debug' - elif isBook: - action = 'download' - else: - action = '' - - params = {'userName': userName, - 'urls': '|'.join(links), - 'action': action, - 'key': user.share_links.get('key', ''), - 'title': subject[:SUBJECT_WORDCNT]} - create_url2book_task(params) - else: #直接转发邮件正文 - imageContents = [] - if hasattr(message, 'attachments'): #先判断是否有图片 - imageContents = [(f, c) for f, c in message.attachments if (guess_type(f)[0] or '').startswith('image/')] - - #先修正不规范的HTML邮件 - h = soup.find('head') - if not h: - h = soup.new_tag('head') - soup.html.insert(0, h) - t = soup.head.find('title') - if not t: - t = soup.new_tag('title') - t.string = subject - soup.head.insert(0, t) - - #有图片的话，要生成MOBI或EPUB才行 - #而且多看邮箱不支持html推送，也先转换epub再推送 - if imageContents: - #仿照Amazon的转换服务器的处理，去掉CSS - if DELETE_CSS_APMAIL: - tag = soup.find('style', attrs={'type': 'text/css'}) - if tag: - tag.decompose() - for tag in soup.find_all(attrs={'style': True}): - del tag['style'] - - #将图片的src的文件名调整好 - for img in soup.find_all('img', attrs={'src': True}): - if img['src'].lower().startswith('cid:'): - img['src'] = img['src'][4:] - - book = html_to_book(str(soup), subject[:SUBJECT_WORDCNT], user, imageContents) - else: #没有图片则直接推送HTML文件，阅读体验更佳 - m = soup.find('meta', attrs={"http-equiv": "Content-Type"}) - if not m: - m = soup.new_tag('meta', content="text/html; charset=utf-8") - m["http-equiv"] = "Content-Type" - soup.html.head.insert(0, m) - else: - m['content'] = "text/html; charset=utf-8" - book = (f'KindleEar_{local_time("%Y-%m-%d_%H-%M", user.timezone)}.html', str(soup).encode('utf-8')) - send_to_kindle(user, subject[:SUBJECT_WORDCNT], book, fileWithTime=False) - - return 'OK' + return links + diff --git a/application/view/library_offical.py b/application/view/library_offical.py index a810c74c..492a7526 100644 --- a/application/view/library_offical.py +++ b/application/view/library_offical.py @@ -232,17 +232,18 @@ def SharedLibraryCategoryOffical(): cats = sorted(SharedRssCategory.get_all(), key=attrgetter('last_updated'), reverse=True) return [item.name for item in cats] -#@bpLibraryOffical.route('/translib') -#def TransferLib(): -# fileName = os.path.join(os.path.dirname(__file__), 'ke_final.json') -# with open(fileName, 'r', encoding='utf-8') as f: -# ke_final = json.loads(f.read()) -# SharedRss.delete().execute() -# cats = set() -# for t in ke_final: -# cats.add(t['c']) -# SharedRss.create(title=t['t'], url=t['u'], isfulltext=t['f'], language=t['l'], category=t['c'], +# @bpLibraryOffical.route('/translib') +# def TransferLib(): +# fileName = os.path.join(os.path.dirname(__file__), 'ke_final.json') +# with open(fileName, 'r', encoding='utf-8') as f: +# ke_final = json.loads(f.read()) +# SharedRss.delete().execute() +# SharedRssCategory.delete().execute() +# cats = set() +# for t in ke_final: +# cats.add(t['c']) +# SharedRss.create(title=t['t'], url=t['u'], isfulltext=t['f'], language=t['l'], category=t['c'], # subscribed=t['s'], created_time=t['d']) -# for cat in cats: -# SharedRssCategory.create(name=cat) -# return f'Finished, data count={len(ke_final)}, category count={len(cats)}' +# for cat in cats: +# SharedRssCategory.create(name=cat, language='') +# return f'Finished, data count={len(ke_final)}, category count={len(cats)}' diff --git a/docs/Chinese/2.config.md b/docs/Chinese/2.config.md index feca7022..20a82f27 100644 --- a/docs/Chinese/2.config.md +++ b/docs/Chinese/2.config.md @@ -193,15 +193,6 @@ TASK_QUEUE_BROKER_URL = "redis://127.0.0.1:6379/" - -## 邮件接收服务 -gae平台有一个比较独特的服务，应用部署完成后除了可以发送邮件，还可以接收邮件，这个功能有时候还是比较有用的，具体的用法可以参考 [FAQ](3.faq.html#appspotmail)。 -如果要启用此服务，配置如下（注意：仅gae平台支持，其他平台设置了也没用）： -```python -INBOUND_EMAIL_SERVICE = 'gae' -``` - - ## WSGI协议容器 diff --git a/docs/Chinese/3.deployment.md b/docs/Chinese/3.deployment.md index 5fd30246..f6dc4342 100644 --- a/docs/Chinese/3.deployment.md +++ b/docs/Chinese/3.deployment.md @@ -14,7 +14,6 @@ KindleEar支持多种平台部署，我只在这里列出一些我测试通过 ```python SERVER_LOCATION = "us-central1" DATABASE_URL = "datastore" -INBOUND_EMAIL_SERVICE = "gae" TASK_QUEUE_SERVICE = "gae" TASK_QUEUE_BROKER_URL = "" ``` @@ -51,7 +50,6 @@ gcloud beta app deploy --version=1 app.yaml 1. config.py关键参数样例 ```python DATABASE_URL = "sqlite:////home/ubuntu/site/kindleear/database.db" -INBOUND_EMAIL_SERVICE = "" TASK_QUEUE_SERVICE = "apscheduler" TASK_QUEUE_BROKER_URL = "redis://127.0.0.1:6379/" TEMP_DIR = "/tmp" @@ -140,9 +138,9 @@ sudo systemctl status gunicorn.service #确认running 8. 出现错误后，查询后台log的命令 ```bash -cat /var/log/nginx/error.log -cat /home/ubuntu/log/gunicorn.error.log -cat /home/ubuntu/log/gunicorn.access.log +cat /var/log/nginx/error.log | tail -n 100 +cat /home/ubuntu/log/gunicorn.error.log | tail -n 100 +cat /home/ubuntu/log/gunicorn.access.log | tail -n 100 ``` 9. 后语，如果部署在Oracle cloud，建议开启其"OCI Email Delivery"服务，然后使用SMTP发送邮件，单邮件最大支持60MB，我还没有发现有哪家服务商能支持那么大的邮件。 @@ -154,7 +152,6 @@ cat /home/ubuntu/log/gunicorn.access.log 1. config.py关键参数样例 ```python DATABASE_URL = "mysql://name:pass@name.mysql.pythonanywhere-services.com/name$default" -INBOUND_EMAIL_SERVICE = "" TASK_QUEUE_SERVICE = "" TASK_QUEUE_BROKER_URL = "" ``` diff --git a/docs/English/2.config.md b/docs/English/2.config.md index 7fddc654..973f86d6 100644 --- a/docs/English/2.config.md +++ b/docs/English/2.config.md @@ -204,14 +204,6 @@ In addition to using existing services available in the market, platforms like U - -## Email Receiving Service -The GAE platform has a unique service. After the application is deployed, besides sending emails, it can also receive emails, which can be quite useful sometimes. For specific usage, refer to the [FAQ](3.faq.html#appspotmail). -If you want to enable this service, configure as follows (Note: only supported on GAE platform, setting on other platforms won't work): -```python -INBOUND_EMAIL_SERVICE = 'gae' -``` - diff --git a/docs/English/3.deployment.md b/docs/English/3.deployment.md index 335cd15f..97db2d66 100644 --- a/docs/English/3.deployment.md +++ b/docs/English/3.deployment.md @@ -16,7 +16,6 @@ Note: If you have previously deployed the Python 2 version of KindleEar, you can ```python SERVER_LOCATION = "us-central1" DATABASE_URL = "datastore" -INBOUND_EMAIL_SERVICE = "gae" TASK_QUEUE_SERVICE = "gae" TASK_QUEUE_BROKER_URL = "" ``` @@ -54,7 +53,6 @@ gcloud beta app deploy --version=1 app.yaml 1. config.py Key Parameter Example ```python DATABASE_URL = "sqlite:////home/ubuntu/site/kindleear/database.db" -INBOUND_EMAIL_SERVICE = "" TASK_QUEUE_SERVICE = "apscheduler" TASK_QUEUE_BROKER_URL = "redis://127.0.0.1:6379/" TEMP_DIR = "/tmp" @@ -100,6 +98,7 @@ cd ~/site #fetch code from github, or you can upload code files by xftp/scp git clone https://github.com/cdhigh/kindleear.git +chmod -R 775 ~ #nginx user www-data read static resource cd kindleear virtualenv --python=python3 venv #create virtual environ vim ./config.py #start to modify some config items @@ -144,9 +143,9 @@ If you already have a domain name, you can bind it to your instance. If not, you 8. To check for errors, use the following commands to query the backend logs: ```bash -cat /var/log/nginx/error.log -cat /home/ubuntu/log/gunicorn.error.log -cat /home/ubuntu/log/gunicorn.access.log +cat /var/log/nginx/error.log | tail -n 100 +cat /home/ubuntu/log/gunicorn.error.log | tail -n 100 +cat /home/ubuntu/log/gunicorn.access.log | tail -n 100 ``` 9. Epilogue: If you choose Oracle Cloud, it is recommended to enable their "OCI Email Delivery" service and utilize SMTP for sending emails. This service supports single email up to 60MB, which I have yet to find supported by any other service provider. @@ -160,7 +159,6 @@ cat /home/ubuntu/log/gunicorn.access.log 1. config.py Key Parameter Example ```python DATABASE_URL = "mysql://name:pass@name.mysql.pythonanywhere-services.com/name$default" -INBOUND_EMAIL_SERVICE = "" TASK_QUEUE_SERVICE = "" TASK_QUEUE_BROKER_URL = "" ``` diff --git a/requirements.txt b/requirements.txt index 748eb5f9..7e8ac5cc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,18 +15,17 @@ Flask>=3.0.0,<4.0.0 flask-babel>=4.0.0,<5.0.0 six>=1.16.0,<2.0.0 feedparser>=6.0.11,<7.0.0 -peewee>=3.1.7,<4.0.0 -flask-apscheduler>=1.13.1,<2.0.0 -redis>=4.5.0,<6.0.0 +weedata>=0.2.1,<1.0.0 +google-cloud-datastore>=2.19.0,<3.0.0 +google-cloud-tasks>=2.15.0,<3.0.0 +appengine-python-standard>=1.1.6,<2.0.0 +#peewee>=3.1.7,<4.0.0 #pymysql>=1.1.0,<2.0.0 #psycopg2>=2.9.9,<3.0.0 -#weedata>=0.2.1,<1.0.0 -#google-cloud-datastore>=2.19.0,<3.0.0 #pymongo>=3.7.2,<4.0.0 -#google-cloud-tasks>=2.15.0,<3.0.0 +#redis>=4.5.0,<6.0.0 +#flask-apscheduler>=1.13.1,<2.0.0 #celery>=5.3.6,<6.0.0 #eventlet>=0.35.1,<1.0.0 #flask-rq2>=18.3,<19.0 -#appengine-python-standard>=1.1.6,<2.0.0 -#sqlalchemy>=2.0.28,<3.0.0 diff --git a/tests/runtests.py b/tests/runtests.py index d3554f4c..7727707b 100644 --- a/tests/runtests.py +++ b/tests/runtests.py @@ -11,6 +11,7 @@ log.setLevel(logging.INFO) builtins.__dict__['default_log'] = log builtins.__dict__['appDir'] = appDir +builtins.__dict__['appVer'] = '3.0' from config import * @@ -24,8 +25,8 @@ def set_env(): os.environ['TEMP_DIR'] = os.path.join(appDir, TEMP_DIR) os.environ['DOWNLOAD_THREAD_NUM'] = str(int(DOWNLOAD_THREAD_NUM)) os.environ['DATABASE_URL'] = 'sqlite://:memory:' - os.environ['TASK_QUEUE_SERVICE'] = TASK_QUEUE_SERVICE - os.environ['TASK_QUEUE_BROKER_URL'] = TASK_QUEUE_BROKER_URL + os.environ['TASK_QUEUE_SERVICE'] = '' + os.environ['TASK_QUEUE_BROKER_URL'] = '' os.environ['APP_DOMAIN'] = APP_DOMAIN os.environ['ADMIN_NAME'] = ADMIN_NAME os.environ['HIDE_MAIL_TO_LOCAL'] = '1' if HIDE_MAIL_TO_LOCAL else '' @@ -56,7 +57,7 @@ def start_test(verbosity=1, failfast=0, testonly='', report=''): cov = coverage.coverage() cov.start() - runtests(collect_tests(), verbosity, failfast) + runtests(collect_tests(testonly), verbosity, failfast) if report: cov.stop() @@ -69,15 +70,13 @@ def start_test(verbosity=1, failfast=0, testonly='', report=''): return 0 TEST_MODULES = ['test_login', 'test_setting', 'test_admin', 'test_subscribe', 'test_adv', - 'test_logs'] #'test_share', -if INBOUND_EMAIL_SERVICE == 'gae': - TEST_MODULES.append('test_inbound_email') + 'test_logs', 'test_inbound_email'] #'test_share', if __name__ == '__main__': verbosity = 1 #Verbosity of output, 0 | 1 | 4 failfast = 0 #Exit on first failure/error report = '' # '' | 'html' | 'console' - testonly = '' #module name, empty for testing all + testonly = 'test_inbound_email' #module name, empty for testing all os.environ['KE_TEST_VERBOSITY'] = str(verbosity) os.environ['KE_SLOW_TESTS'] = '1' #Run tests that may be slow diff --git a/tests/test_admin.py b/tests/test_admin.py index ac14334c..ba45c818 100644 --- a/tests/test_admin.py +++ b/tests/test_admin.py @@ -79,14 +79,14 @@ def test_change_admin_password(self): data['password2'] = '1' resp = self.client.post('/account/change/admin', data=data) self.assertEqual(resp.status_code, 200) - self.assertIn("Change password success.", resp.text) + self.assertIn("Changes saved successfully.", resp.text) data['orgpwd'] = '1' data['password1'] = 'admin' data['password2'] = 'admin' resp = self.client.post('/account/change/admin', data=data) self.assertEqual(resp.status_code, 200) - self.assertIn("Change password success.", resp.text) + self.assertIn("Changes saved successfully.", resp.text) def test_change_user_password(self): data = {'username': '2', 'password1': '2', 'password2': '2', 'email': '1@1.com', diff --git a/tests/test_adv.py b/tests/test_adv.py index ce99ae90..ec64ff95 100644 --- a/tests/test_adv.py +++ b/tests/test_adv.py @@ -17,13 +17,9 @@ def test_advdeliver(self): def test_whitelist(self): resp = self.client.get('/adv/whitelist') - if INBOUND_EMAIL_SERVICE == 'gae': - self.assertEqual(resp.status_code, 200) - self.assertTrue('White List' in resp.text) - else: - self.assertEqual(resp.status_code, 404) - return - + self.assertEqual(resp.status_code, 200) + self.assertTrue('White List' in resp.text) + resp = self.client.post('/adv/whitelist', data={'wlist': ''}) self.assertEqual(resp.status_code, 302) @@ -63,7 +59,7 @@ def test_advimport(self): xml = """KindleEar.opml Wed, 14 Feb 2024 02:18:32 GMT+0000Wed, 14 Feb 2024 02:18:32 GMT+0000 KindleEar - + """ @@ -81,7 +77,7 @@ def test_advimport(self): self.assertEqual(resp.status_code, 200) content = resp.data self.assertIn(b'KindleEar', content) - self.assertIn(b'https%3A%2F%2Fwww.news.com%2Fnews.xml', content) + self.assertIn(b'https%3A//www.news.com/news.xml', content) def test_advuploadcover(self): resp = self.client.get('/adv/cover') diff --git a/tests/test_base.py b/tests/test_base.py index 1a15b244..c7732f62 100644 --- a/tests/test_base.py +++ b/tests/test_base.py @@ -29,8 +29,8 @@ def setUp(self): self.temp_files = [] def tearDown(self): - if self.login_required: - self.client.post('/logout') + #if self.login_required: + # self.client.post('/logout') if self.temp_files: for f in self.temp_files: try: diff --git a/tests/test_inbound_email.py b/tests/test_inbound_email.py index 86559067..6216ed54 100644 --- a/tests/test_inbound_email.py +++ b/tests/test_inbound_email.py @@ -21,46 +21,49 @@ def test_ah_bounce(self): self.assertEqual(resp.status_code, 200) def test_ah_mail(self): - data = {'sender': 'Bill ', 'subject': 'teardown', 'text': 'is text body', 'files': None} - resp = self.send('dl', data) - self.assertEqual(resp.status_code, 200) - self.assertIn('Spam mail!', resp.text) - - WhiteList.create(mail='*', user='admin') - resp = self.send('dl', data) - self.assertEqual(resp.status_code, 200) - - data['text'] = "www.google.com" - resp = self.send('dl', data) - self.assertEqual(resp.status_code, 200) - - resp = self.send('trigger', data) - self.assertEqual(resp.status_code, 200) - self.assertIn('is triggered', resp.text) - - resp = self.send('book', data) - self.assertEqual(resp.status_code, 200) - - resp = self.send('download', data) - self.assertEqual(resp.status_code, 200) - - data['subject'] = 'Teardown!links' - resp = self.send('download', data) - self.assertEqual(resp.status_code, 200) - - data['subject'] = 'Teardown!article' - resp = self.send('download', data) - self.assertEqual(resp.status_code, 200) - - imgDir = os.path.join(appDir, 'application', 'images') - data['files'] = [os.path.join(imgDir, 'cover0.jpg'), os.path.join(imgDir, 'cover1.jpg')] - resp = self.send('d', data) - self.assertEqual(resp.status_code, 200) - - def send(self, to, data): + for mType in ('gae', 'general'): + WhiteList.delete().execute() + data = {'sender': 'Bill ', 'subject': 'teardown', 'text': 'is text body', 'files': None} + resp = self.send('dl', data, mType) + self.assertEqual(resp.status_code, 200) + self.assertIn('Spam mail', resp.text) + + WhiteList.create(mail='*', user='admin') + resp = self.send('dl', data, mType) + self.assertEqual(resp.status_code, 200) + + data['text'] = "www.google.com" + resp = self.send('dl', data, mType) + self.assertEqual(resp.status_code, 200) + + resp = self.send('trigger', data, mType) + self.assertEqual(resp.status_code, 200) + self.assertIn('is triggered', resp.text) + + resp = self.send('book', data, mType) + self.assertEqual(resp.status_code, 200) + + resp = self.send('download', data, mType) + self.assertEqual(resp.status_code, 200) + + data['subject'] = 'Teardown!links' + resp = self.send('download', data, mType) + self.assertEqual(resp.status_code, 200) + + data['subject'] = 'Teardown!article' + resp = self.send('download', data, mType) + self.assertEqual(resp.status_code, 200) + + imgDir = os.path.join(appDir, 'application', 'images') + data['files'] = [os.path.join(imgDir, 'cover0.jpg'), os.path.join(imgDir, 'cover1.jpg')] + resp = self.send('d', data, mType) + self.assertEqual(resp.status_code, 200) + + def send(self, to, data, mType): to = f'{to}@kindleear.appspotmail.com' data['to'] = to - return self.client.post(f'/_ah/mail/{quote(to)}', data=self.build_mail(**data), content_type='multipart/alternative') + url = f'/_ah/mail/{quote(to)}' if mType == 'gae' else '/mail' + return self.client.post(url, data=self.build_mail(**data).encode('utf-8'), content_type='multipart/alternative') def build_mail(self, sender, to, subject, text, files=None): msg = MIMEMultipart() diff --git a/tests/test_login.py b/tests/test_login.py index c2e9c69c..54766b0a 100644 --- a/tests/test_login.py +++ b/tests/test_login.py @@ -48,5 +48,5 @@ def test_resetpwd(self): resp = self.client.get('/resetpwd?name=admin') self.assertEqual(resp.status_code, 200) - self.assertIn('The email of account', resp.text) + self.assertIn('Reset password', resp.text) diff --git a/tests/test_logs.py b/tests/test_logs.py index 54170e95..4efbc5ac 100644 --- a/tests/test_logs.py +++ b/tests/test_logs.py @@ -35,7 +35,7 @@ def test_remove_logs(self): self.assertEqual(resp.status_code, 200) self.assertIn('lines delivery log removed.', resp.text) - KeUser.create(name='other', passwd='pwd', email='1@1', , sender='1@1', enable_send=True, expiration_days=7, + KeUser.create(name='other', passwd='pwd', email='1@1', sender='1@1', enable_send=True, expiration_days=7, expires=datetime.datetime.utcnow() - datetime.timedelta(days=30)) resp = self.client.get('/removelogs') self.assertEqual(resp.status_code, 200) diff --git a/tests/test_subscribe.py b/tests/test_subscribe.py index 529788cc..20dd7dfd 100644 --- a/tests/test_subscribe.py +++ b/tests/test_subscribe.py @@ -130,8 +130,7 @@ def test_upload_recipe(self): resp = self.client.get('/viewsrc/{}'.format(upload_id.replace(':', '__'))) self.assertEqual(resp.status_code, 200) - self.assertIn('class BasicUserRecipe', resp.text) - self.assertIn('auto_cleanup = True', resp.text) + self.assertIn('class UserRecipe', resp.text) resp = self.client.post('/recipe/delete', data={'id': upload_id}) self.assertEqual(resp.json['status'], 'ok') diff --git a/tools/update_req.py b/tools/update_req.py index 70369e28..af79c4fb 100644 --- a/tools/update_req.py +++ b/tools/update_req.py @@ -91,8 +91,7 @@ def config_to_dict(cfgFile): db = cfg['DATABASE_URL'].split('://')[0] task = cfg['TASK_QUEUE_SERVICE'] broker = cfg['TASK_QUEUE_BROKER_URL'] - if (cfg['DATABASE_URL'].startswith('datastore') or cfg['INBOUND_EMAIL_SERVICE'] == 'gae' or - cfg['TASK_QUEUE_SERVICE'] == 'gae'): + if (cfg['DATABASE_URL'].startswith('datastore') or cfg['TASK_QUEUE_SERVICE'] == 'gae'): plat = 'gae' else: plat = ''