diff --git a/application/back_end/db_models.py b/application/back_end/db_models.py index dd84194b..7ec1fb89 100644 --- a/application/back_end/db_models.py +++ b/application/back_end/db_models.py @@ -215,7 +215,7 @@ def categories(self): #Buffer for category of shared rss [for kindleear.appspot.com only] class SharedRssCategory(MyBaseModel): name = CharField() - language = CharField() + language = CharField(default='') last_updated = DateTimeField(default=datetime.datetime.utcnow) class LastDelivered(MyBaseModel): @@ -225,6 +225,16 @@ class LastDelivered(MyBaseModel): record = CharField(default='') datetime = DateTimeField(default=datetime.datetime.utcnow) +class InBox(MyBaseModel): + user = CharField() + sender = CharField() + to = CharField() + subject = CharField() + size = IntegerField(default=0) + datetime = DateTimeField(default=datetime.datetime.utcnow) + body = TextField(default='', index=False) + attachments = CharField(default='') #存放UserBlob的数据库id,逗号分割 + class AppInfo(MyBaseModel): name = CharField(unique=True) value = CharField(default='') @@ -251,9 +261,9 @@ def create_database_tables(): #with dbInstance.connection_context(): #connect_database() dbInstance.create_tables([KeUser, UserBlob, Recipe, BookedRecipe, DeliverLog, WhiteList, - SharedRss, SharedRssCategory, LastDelivered, AppInfo], safe=True) + SharedRss, SharedRssCategory, LastDelivered, InBox, AppInfo], safe=True) if not AppInfo.get_value(AppInfo.dbSchemaVersion): AppInfo.set_value(AppInfo.dbSchemaVersion, appVer) #close_database() - + return 'Created database tables successfully' diff --git a/application/back_end/db_models_sql.py b/application/back_end/db_models_sql.py index 2c017c98..da1b9f4c 100644 --- a/application/back_end/db_models_sql.py +++ b/application/back_end/db_models_sql.py @@ -16,7 +16,10 @@ if not fileName.startswith('/'): dbName = 'sqlite:///{}'.format(os.path.join(appDir, fileName)) -dbInstance = connect(dbName) +if dbName == 'sqlite://:memory:': + dbInstance = SqliteDatabase(':memory:') +else: + dbInstance = connect(dbName) #调用此函数正式连接到数据库(打开数据库) def connect_database(): diff --git a/application/lib/calibre/web/feeds/__init__.py b/application/lib/calibre/web/feeds/__init__.py index e46bfa39..55604448 100644 --- a/application/lib/calibre/web/feeds/__init__.py +++ b/application/lib/calibre/web/feeds/__init__.py @@ -10,6 +10,8 @@ import re import time import traceback +import json +import datetime from calibre import entity_to_unicode, force_unicode, strftime from calibre.utils.cleantext import clean_ascii_chars, clean_xml_chars @@ -56,7 +58,7 @@ def __init__(self, id, title, url, author, summary, published, content): self.text_summary = clean_ascii_chars(summary) self.author = author self.content = content - self.date = published + self.date = published #time.struct_time self.utctime = dt_factory(self.date, assume_utc=True, as_utc=True) self.localtime = self.utctime.astimezone(local_tz) self._formatted_date = None @@ -143,6 +145,27 @@ def populate_from_feed(self, feed, title=None, oldest_article=7, break self.parse_article(item) + #added by cdhigh + def populate_from_json(self, feed, title=None, oldest_article=7, max_articles_per_feed=100): + self.title = feed.get('title', _('Unknown section')) if not title else title + self.description = feed.get('description', '') + self.image_url = feed.get('icon', None) or feed.get('favicon', None) + self.image_width = 88 + self.image_height = 31 + self.image_alt = '' + + self.articles = [] + self.id_counter = 0 + self.added_articles = [] + + self.oldest_article = oldest_article + + entries = feed.get('items') + for item in entries: + if len(self.articles) >= max_articles_per_feed: + break + self.parse_article_json(item) + def populate_from_preparsed_feed(self, title, articles, oldest_article=7, max_articles_per_feed=100): self.title = str(title if title else _('Unknown feed')) @@ -240,6 +263,50 @@ def parse_article(self, item): title = title.decode('utf-8', 'replace') self.logger.debug('Skipping article %s as it is too old'%title) + #added by cdhigh + def parse_article_json(self, item): + self.id_counter += 1 + id_ = item.get('id', None) or f'internal id#{self.id_counter}' + if id_ in self.added_articles: + return + + self.added_articles.append(id_) + published = item.get('date_modified', None) or item.get('date_published', None) + if published: + import dateutil + try: + published = dateutil.parser.parse(published).timetuple() + except: + published = time.gmtime() + else: + published = time.gmtime() + + title = item.get('title', _('Untitled article')) + if title.startswith('<'): + title = re.sub(r'<.+?>', '', title) + link = item.get('url', None) or item.get('external_url', None) + description = item.get('summary', None) + authors = item.get('authors', []) + author = ' '.join([aut.get('name', '') for aut in authors]) + if not author: + author = item.get('author', {}).get('name', '') + content = item.get('content_html', None) or item.get('content_text', None) + if not link and not content: + return + + article = Article(id_, title, link, author, description, published, content) + delta = utcnow() - article.utctime + if (self.oldest_article == 0) or (delta.days*24*3600 + delta.seconds <= 24*3600*self.oldest_article): + self.articles.append(article) + else: + try: + self.logger.debug('Skipping article %s (%s) from feed %s as it is too old.'% + (title, article.localtime.strftime('%a, %d %b, %Y %H:%M'), self.title)) + except UnicodeDecodeError: + if not isinstance(title, str): + title = title.decode('utf-8', 'replace') + self.logger.debug('Skipping article %s as it is too old'%title) + def reverse(self): self.articles.reverse() @@ -343,11 +410,12 @@ def feed_from_xml(raw_xml, title=None, oldest_article=7, max_articles_per_feed=100, get_article_url=lambda item: item.get('link', None), log=default_log): + from feedparser import parse # Handle unclosed escaped entities. They trip up feedparser and HBR for one # generates them - raw_xml = re.sub(br'(&#\d+)([^0-9;])', br'\1;\2', raw_xml) + raw_xml = re.sub(r'(&#\d+)([^0-9;])', r'\1;\2', raw_xml) feed = parse(raw_xml) pfeed = Feed(get_article_url=get_article_url, log=log) pfeed.populate_from_feed(feed, title=title, @@ -355,6 +423,24 @@ def feed_from_xml(raw_xml, title=None, oldest_article=7, max_articles_per_feed=max_articles_per_feed) return pfeed +#added by cdhigh +def feed_from_json(raw_json, title=None, oldest_article=7, + max_articles_per_feed=100, + get_article_url=lambda item: item.get('link', None), + log=default_log): + + pfeed = Feed(get_article_url=get_article_url, log=log) + + try: + feed = json.loads(raw_json) + except Exception as e: + log.warning('Parse json feed failed {}: {}'.format(title, str(e))) + return pfeed + + pfeed.populate_from_json(feed, title=title, + oldest_article=oldest_article, + max_articles_per_feed=max_articles_per_feed) + return pfeed def feeds_from_index(index, oldest_article=7, max_articles_per_feed=100, log=default_log): diff --git a/application/lib/calibre/web/feeds/news.py b/application/lib/calibre/web/feeds/news.py index 59eaec12..b2dea0b7 100644 --- a/application/lib/calibre/web/feeds/news.py +++ b/application/lib/calibre/web/feeds/news.py @@ -29,7 +29,7 @@ from calibre.utils.logging import ThreadSafeWrapper from calibre.utils.threadpool import NoResultsPending, ThreadPool, WorkRequest from calibre.web import Recipe -from calibre.web.feeds import Feed, Article, feed_from_xml, feeds_from_index, templates +from calibre.web.feeds import Feed, Article, feed_from_xml, feeds_from_index, templates, feed_from_json from calibre.web.fetch.simple import AbortArticle, RecursiveFetcher from calibre.web.fetch.utils import prepare_masthead_image from polyglot.builtins import string_or_bytes @@ -1853,9 +1853,10 @@ def parse_feeds(self): # br.add_password(url, purl.username, purl.password) resp = br.open(url, timeout=self.timeout) if resp.status_code == 200: - raw = resp.content - feed = feed_from_xml(raw, title=title, log=self.log, oldest_article=self.oldest_article, - max_articles_per_feed=self.max_articles_per_feed, get_article_url=self.get_article_url) + raw = resp.text.lstrip() + pFunc = feed_from_json if raw and raw[0] == '{' else feed_from_xml + feed = pFunc(raw, title=title, log=self.log, oldest_article=self.oldest_article, + max_articles_per_feed=self.max_articles_per_feed, get_article_url=self.get_article_url) parsed_feeds.append(feed) else: raise Exception(f'Cannot fetch {url}:{resp.status_code}') diff --git a/application/routes.py b/application/routes.py index cbeb394e..c7d638c3 100644 --- a/application/routes.py +++ b/application/routes.py @@ -3,18 +3,9 @@ #主页和其他路由 import os from flask import Blueprint, render_template, send_from_directory, current_app -from .view import login -from .view import admin -from .view import adv -from .view import deliver -from .view import library -from .view import library_offical -from .view import logs -from .view import setting -from .view import share -from .view import subscribe -from .work import worker -from .work import url2book +from .view import (login, admin, adv, deliver, library, library_offical, logs, setting, share, + subscribe, inbound_email) +from .work import worker, url2book bpHome = Blueprint('bpHome', __name__) @@ -62,10 +53,10 @@ def register_routes(app): app.register_blueprint(worker.bpWorker) app.register_blueprint(url2book.bpUrl2Book) app.register_blueprint(library_offical.bpLibraryOffical) + app.register_blueprint(inbound_email.bpInBoundEmail) - #使用GAE来接收邮件 - if app.config['INBOUND_EMAIL_SERVICE'] == 'gae': + #启用GAE邮件服务如果部署在GAE平台 + if app.config['DATABASE_URL'] == 'datastore': from google.appengine.api import wrap_wsgi_app - from application.view.inbound_email import bpInBoundEmail - app.wsgi_app = wrap_wsgi_app(app.wsgi_app) #启用GAE邮件服务 - app.register_blueprint(bpInBoundEmail) + app.wsgi_app = wrap_wsgi_app(app.wsgi_app) + diff --git a/application/templates/adv_base.html b/application/templates/adv_base.html index d1e626b4..bfca7277 100644 --- a/application/templates/adv_base.html +++ b/application/templates/adv_base.html @@ -36,14 +36,13 @@
diff --git a/application/templates/adv_calibre_options.html b/application/templates/adv_calibre_options.html index 1b3ea29b..e211bb4f 100644 --- a/application/templates/adv_calibre_options.html +++ b/application/templates/adv_calibre_options.html @@ -9,7 +9,7 @@ {% if tips -%}{{_("Set the parameters for Calibre, in JSON dictionary format.")}}