diff --git a/application/__init__.py b/application/__init__.py index 62b37ccc..1501fbd9 100644 --- a/application/__init__.py +++ b/application/__init__.py @@ -10,7 +10,7 @@ builtins.__dict__['_'] = gettext #创建并初始化Flask wsgi对象 -def init_app(name, debug=False): +def init_app(name, cfgMap, debug=False): thisDir = os.path.dirname(os.path.abspath(__file__)) rootDir = os.path.abspath(os.path.join(thisDir, '..')) template_folder = os.path.join(thisDir, 'templates') @@ -18,7 +18,7 @@ def init_app(name, debug=False): i18n_folder = os.path.join(thisDir, 'translations') app = Flask(name, template_folder=template_folder, static_folder=static_folder) - app.config.from_pyfile(os.path.join(rootDir, 'config.py')) + app.config.from_mapping(cfgMap) app.config['MAX_CONTENT_LENGTH'] = 32 * 1024 * 1024 #32MB from .view import setting @@ -31,13 +31,14 @@ def init_app(name, debug=False): from .back_end.task_queue_adpt import init_task_queue_service init_task_queue_service(app) - from .back_end.db_models import connect_database, close_database + from .back_end.db_models import create_database_tables, connect_database, close_database + create_database_tables() @app.before_request def BeforeRequest(): g.version = appVer g.now = datetime.datetime.utcnow - g.allowSignup = app.config['ALLOW_SIGNUP'] + g.allowSignup = app.config['ALLOW_SIGNUP'] == 'yes' connect_database() @app.teardown_request diff --git a/application/back_end/db_models.py b/application/back_end/db_models.py index e8d462b6..fe2e2b70 100644 --- a/application/back_end/db_models.py +++ b/application/back_end/db_models.py @@ -259,12 +259,9 @@ def set_value(cls, name, value): #创建数据库表格,一个数据库只需要创建一次 def create_database_tables(): - #with dbInstance.connection_context(): - #connect_database() dbInstance.create_tables([KeUser, UserBlob, Recipe, BookedRecipe, DeliverLog, WhiteList, SharedRss, SharedRssCategory, LastDelivered, InBox, AppInfo], safe=True) if not AppInfo.get_value(AppInfo.dbSchemaVersion): AppInfo.set_value(AppInfo.dbSchemaVersion, appVer) - #close_database() return 'Created database tables successfully' diff --git a/application/back_end/send_mail_adpt.py b/application/back_end/send_mail_adpt.py index 9ee5580f..6d62a754 100644 --- a/application/back_end/send_mail_adpt.py +++ b/application/back_end/send_mail_adpt.py @@ -6,12 +6,12 @@ #https://cloud.google.com/appengine/docs/standard/python3/reference/services/bundled/google/appengine/api/mail #https://cloud.google.com/appengine/docs/standard/python3/services/mail import os, datetime, zipfile, base64 -from ..utils import local_time, ke_decrypt +from ..utils import local_time, ke_decrypt, str_to_bool from ..base_handler import save_delivery_log from .db_models import KeUser #google.appengine will apply patch for os.env module -hideMailLocal = os.getenv('HIDE_MAIL_TO_LOCAL') +hideMailLocal = str_to_bool(os.getenv('HIDE_MAIL_TO_LOCAL')) #判断是否是部署在gae平台 if os.getenv('DATABASE_URL') == 'datastore': @@ -105,7 +105,7 @@ def send_mail(user, to, subject, body, attachments=None, html=None): elif srv_type == 'smtp': data['host'] = sm_service.get('host', '') data['port'] = sm_service.get('port', 587) - data['username'] = user.sender + data['username'] = sm_service.get('username', '') data['password'] = ke_decrypt(sm_service.get('password', ''), user.secret_key) smtp_send_mail(**data) elif srv_type == 'local': diff --git a/application/base_handler.py b/application/base_handler.py index c077ccdd..6fa1484b 100644 --- a/application/base_handler.py +++ b/application/base_handler.py @@ -17,7 +17,7 @@ def login_required(forAjax=False): def decorator(func): @wraps(func) def wrapper(*args, **kwargs): - if (session.get('login', '') == 1) and session.get('userName', ''): + if (session.get('login', '') == 1) and get_login_user(): return func(*args, **kwargs) else: return redirect(url_for("bpLogin.NeedLoginAjax") if forAjax else url_for("bpLogin.Login")) diff --git a/application/lib/build_ebook.py b/application/lib/build_ebook.py index 941b7d33..33d87cd3 100644 --- a/application/lib/build_ebook.py +++ b/application/lib/build_ebook.py @@ -79,7 +79,7 @@ def ke_opts(user, options=None): opt.setdefault('dont_split_on_page_breaks', True) opt['user'] = user - #opt.setdefault('debug_pipeline', os.getenv('TEMP_DIR')) + #opt.setdefault('debug_pipeline', os.getenv('KE_TEMP_DIR')) #opt.setdefault('verbose', 1) #opt.setdefault('test', 1) return opt diff --git a/application/lib/calibre/ebooks/conversion/plumber.py b/application/lib/calibre/ebooks/conversion/plumber.py index 2affffb0..5a821c42 100644 --- a/application/lib/calibre/ebooks/conversion/plumber.py +++ b/application/lib/calibre/ebooks/conversion/plumber.py @@ -308,8 +308,13 @@ def set_profile(profiles, which): self.opts.no_inline_navbars = self.opts.output_profile.supports_mobi_indexing \ and self.output_fmt == 'mobi' - if self.opts.verbose: - self.log.filter_level = self.log.DEBUG + + levelNames = {'CRITICAL': self.log.ERROR, 'FATAL': self.log.ERROR, 'ERROR': self.log.ERROR, + 'WARN': self.log.WARN, 'WARNING': self.log.WARN, 'INFO': self.log.INFO, + 'DEBUG': self.log.DEBUG} + level = levelNames.get(os.getenv('LOG_LEVEL', '').upper(), self.log.WARN) + self.log.filter_level = self.log.DEBUG if self.opts.verbose else level + if self.opts.verbose > 1: self.log.debug('Resolved conversion options') try: @@ -358,8 +363,6 @@ def run(self): ''' # Setup baseline option values self.setup_options() - if self.opts.verbose: - self.log.filter_level = self.log.DEBUG css_parser.log.setLevel(logging.WARN) #get_types_map() # Ensure the mimetypes module is initialized @@ -378,7 +381,7 @@ def run(self): self.output_plugin.specialize_options(self.log, self.opts, self.input_fmt) #根据需要,创建临时目录或创建内存缓存 - system_temp_dir = os.environ.get('TEMP_DIR') + system_temp_dir = os.environ.get('KE_TEMP_DIR') if system_temp_dir and self.input_fmt != 'html': tdir = PersistentTemporaryDirectory(prefix='plumber_', dir=system_temp_dir) fs = FsDictStub(tdir) diff --git a/application/lib/calibre/ebooks/oeb/base.py b/application/lib/calibre/ebooks/oeb/base.py index c7a6d705..62a3730a 100644 --- a/application/lib/calibre/ebooks/oeb/base.py +++ b/application/lib/calibre/ebooks/oeb/base.py @@ -1084,7 +1084,7 @@ def unload_data_from_memory(self, memory=None): if isinstance(self._data, bytes): if memory is None: from calibre.ptempfile import PersistentTemporaryFile - temp_dir = os.environ.get('TEMP_DIR') + temp_dir = os.environ.get('KE_TEMP_DIR') pt = PersistentTemporaryFile(suffix='_oeb_base_mem_unloader.img', dir=temp_dir) with pt: pt.write(self._data) diff --git a/application/lib/calibre/utils/img.py b/application/lib/calibre/utils/img.py index 6a0c00d0..e7b1ae16 100644 --- a/application/lib/calibre/utils/img.py +++ b/application/lib/calibre/utils/img.py @@ -141,6 +141,8 @@ def image_to_data(img, compression_quality=95, fmt='JPEG', png_compression_level if fmt == 'GIF': return png_data_to_gif_data(img) else: + if img.mode != 'RGB': + img = img.convert('RGB') data = BytesIO() img.save(data, fmt) return data.getvalue() diff --git a/application/lib/calibre/web/feeds/news.py b/application/lib/calibre/web/feeds/news.py index b2dea0b7..fbaf4cda 100644 --- a/application/lib/calibre/web/feeds/news.py +++ b/application/lib/calibre/web/feeds/news.py @@ -397,7 +397,7 @@ class BasicNewsRecipe(Recipe): #: parameters are left at their default values, JPEG images will be scaled to fit #: in the screen dimensions set by the output profile and compressed to size at #: most (w * h)/16 where w x h are the scaled image dimensions. - compress_news_images = False + compress_news_images = True #: The factor used when auto compressing JPEG images. If set to None, #: auto compression is disabled. Otherwise, the images will be reduced in size to @@ -410,7 +410,7 @@ class BasicNewsRecipe(Recipe): #: first be scaled and then its quality lowered until its size is less than #: (w * h)/factor where w and h are now the *scaled* image dimensions. In #: other words, this compression happens after scaling. - compress_news_images_auto_size = 16 + compress_news_images_auto_size = None #16 #: Set JPEG quality so images do not exceed the size given (in KBytes). #: If set, this parameter overrides auto compression via compress_news_images_auto_size. diff --git a/application/lib/calibre/web/fetch/simple.py b/application/lib/calibre/web/fetch/simple.py index 66ff58e9..38b1a99a 100644 --- a/application/lib/calibre/web/fetch/simple.py +++ b/application/lib/calibre/web/fetch/simple.py @@ -458,11 +458,11 @@ def process_images(self, soup, baseurl): if itype not in {'png', 'jpg', 'jpeg'}: itype = 'png' if itype == 'gif' else 'jpeg' data = image_to_data(img, fmt=itype) - if self.compress_news_images and itype in {'jpg','jpeg'}: + if self.compress_news_images: # and itype in {'jpg','jpeg'}: try: data = self.rescale_image(data) except Exception: - self.log.exception('failed to compress image '+iurl) + self.log.warning('failed to compress image ' + iurl) # Moon+ apparently cannot handle .jpeg files if itype == 'jpeg': itype = 'jpg' diff --git a/application/lib/ebook_translator/engines/base.py b/application/lib/ebook_translator/engines/base.py index d4ec16d1..513686f6 100644 --- a/application/lib/ebook_translator/engines/base.py +++ b/application/lib/ebook_translator/engines/base.py @@ -184,20 +184,29 @@ def _is_auto_lang(self): def _get_api_key(self): if self.need_api_key and self.api_keys: - return self.api_keys.pop(0) + return self.api_keys.pop(0).strip() return None - def get_result(self, url, data=None, headers={}, method='GET', + def get_result(self, url, data=None, headers=None, method='GET', stream=False, silence=False, callback=None): result = '' br = UrlOpener(headers=headers, timeout=self.request_timeout) resp = br.open(url, data=data, method=method, stream=stream) if resp.status_code == 200: - text = resp.text + text = [] + if stream: + for line in resp.iter_content(chunk_size=None, decode_unicode=True): + text.append(line) + text = ''.join(text) + else: + text = resp.text + try: return callback(text) if callback else text except: raise Exception('Can not parse translation. Raw data: {text}') + finally: + resp.close() elif silence: return None else: diff --git a/application/lib/smtp_mail.py b/application/lib/smtp_mail.py index 8caa88d8..6caafe88 100644 --- a/application/lib/smtp_mail.py +++ b/application/lib/smtp_mail.py @@ -14,7 +14,7 @@ def smtp_send_mail(sender, to, subject, body, host, username, password, port=Non host, port = host.split(':', 2) port = int(port) else: - port = 587 #587-TLS, 465-SSL + port = 587 #587-TLS, 465-SSL, 25-Nocrpt to = to if isinstance(to, list) else [to] message = MIMEMultipart('alternative') if html else MIMEMultipart() @@ -33,11 +33,13 @@ def smtp_send_mail(sender, to, subject, body, host, username, password, port=Non message.attach(part) klass = smtplib.SMTP if port != 465 else smtplib.SMTP_SSL - with klass(host=host, port=port) as smtp_server: - smtp_server.connect(host, port) - smtp_server.ehlo() - smtp_server.starttls() - smtp_server.ehlo() - smtp_server.login(user=username, password=password) - smtp_server.sendmail(sender, to, message.as_string()) + with klass(host=host, port=port) as server: + server.set_debuglevel(0) #0-no debug info, 1-base, 2- verbose + server.connect(host, port) + server.ehlo() + if port == 587: + server.starttls() + server.ehlo() + server.login(user=username, password=password) + server.sendmail(sender, to, message.as_string()) diff --git a/application/templates/base.html b/application/templates/base.html index fef70fe3..dab87994 100644 --- a/application/templates/base.html +++ b/application/templates/base.html @@ -92,7 +92,7 @@ abbrSep: '{{_("Sep")|safe}}', abbrLog: '{{_("Log")|safe}}', abbrEmb: '{{_("Emb")|safe}}', - testEmailOk: '{{_("The test email has been successfully sent to [{0}]. Please check your inbox or spam folder to confirm its delivery. Depending on your email server, there may be a slight delay.")|safe}}' + testEmailOk: '{{_("The test email has been successfully sent to the following addresses. Please check your inbox or spam folder to confirm its delivery. Depending on your email server, there may be a slight delay.")|safe}}' }; {% block javascript_inhead %} diff --git a/application/templates/setting.html b/application/templates/setting.html index 2f4d4c0d..83733549 100644 --- a/application/templates/setting.html +++ b/application/templates/setting.html @@ -216,7 +216,7 @@
- +
@@ -317,7 +317,7 @@ function SendTestEmail() { $.post("/send_test_email", {url: window.location.href}, function (data) { if (data.status == "ok") { - ShowSimpleModalDialog('

{0}

'.format(i18n.testEmailOk.format(data.email))); + ShowSimpleModalDialog('

{0}


{1}

'.format(i18n.testEmailOk, data.emails.join('
'))); } else { alert(data.status); } diff --git a/application/utils.py b/application/utils.py index 1f8a91d8..46142d71 100644 --- a/application/utils.py +++ b/application/utils.py @@ -129,6 +129,24 @@ def filesizeformat(value, binary=False, suffix='B'): value /= base return f"{value:.1f} {unit}{suffix}" + +#将字符串安全转义到xml格式,有标准库函数xml.sax.saxutils.escape(),但是简单的功能就简单的函数就好 +def xml_escape(txt): + txt = txt.replace("&", "&") + txt = txt.replace("<", "<") + txt = txt.replace(">", ">") + txt = txt.replace('"', """) + txt = txt.replace("'", "'") + return txt + +def xml_unescape(txt): + txt = txt.replace("&", "&") + txt = txt.replace("<", "<") + txt = txt.replace(">", ">") + txt = txt.replace(""", '"') + txt = txt.replace("'", "'") + return txt + #-----------以下几个函数为安全相关的 def new_secret_key(length=12): allchars = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXZYabcdefghijklmnopqrstuvwxyz' diff --git a/application/view/adv.py b/application/view/adv.py index b0877e5a..18d1dc6c 100644 --- a/application/view/adv.py +++ b/application/view/adv.py @@ -2,14 +2,14 @@ # -*- coding:utf-8 -*- #一些高级设置功能页面 -import datetime, hashlib, io +import datetime, hashlib, io, textwrap from urllib.parse import quote, unquote, urljoin, urlparse from flask import Blueprint, url_for, render_template, redirect, session, send_file, abort, current_app as app from flask_babel import gettext as _ from PIL import Image from ..base_handler import * from ..back_end.db_models import * -from ..utils import local_time, ke_encrypt, ke_decrypt, str_to_bool, safe_eval +from ..utils import local_time, ke_encrypt, ke_decrypt, str_to_bool, safe_eval, xml_escape, xml_unescape from ..lib.pocket import Pocket from ..lib.urlopener import UrlOpener @@ -148,7 +148,7 @@ def AdvImportPost(): return adv_render_template('adv_import.html', 'import', user=user, tips=str(e)) for o in walkOpmlOutline(rssList): - title, url, isfulltext = o.text, unquote(o.xmlUrl), o.isFulltext #isFulltext为非标准属性 + title, url, isfulltext = xml_unescape(o.text), xml_unescape(o.xmlUrl), o.isFulltext #isFulltext为非标准属性 if isfulltext: isfulltext = str_to_bool(isfulltext) else: @@ -190,7 +190,8 @@ def AdvExport(): user = get_login_user() #为了简单起见,就不用其他库生成xml,而直接使用字符串格式化生成 - opmlTpl = """ + opmlTpl = textwrap.dedent("""\ + KindleEar.opml @@ -201,7 +202,7 @@ def AdvExport(): {outLines} - """ + """) date = local_time('%a, %d %b %Y %H:%M:%S GMT', user.timezone) #添加时区信息 @@ -210,14 +211,10 @@ def AdvExport(): outLines = [] for feed in user.all_custom_rss(): outLines.append(''.format( - feed.title, quote(feed.url), feed.isfulltext)) + xml_escape(feed.title), xml_escape(feed.url), feed.isfulltext)) outLines = '\n'.join(outLines) - opmlFile = opmlTpl.format(date=date, outLines=outLines) - outLines = [] - for line in opmlFile.split('\n'): - outLines.append(line[4:] if line.startswith(' ') else line) - opmlFile = '\n'.join(outLines).encode('utf-8') + opmlFile = opmlTpl.format(date=date, outLines=outLines).encode('utf-8') return send_file(io.BytesIO(opmlFile), mimetype="text/xml", as_attachment=True, download_name="KindleEar_subscription.xml") #在本地选择一个图片上传做为自定义RSS书籍的封面 diff --git a/application/view/inbound_email.py b/application/view/inbound_email.py index 28b3003c..a02b2229 100644 --- a/application/view/inbound_email.py +++ b/application/view/inbound_email.py @@ -62,7 +62,7 @@ def ReceiveGaeMail(path): @bpInBoundEmail.post("/mail") def ReceiveMail(): msg = email.message_from_bytes(request.get_data()) - + txtBodies = [] htmlBodies = [] attachments = [] @@ -79,6 +79,29 @@ def ReceiveMail(): return ReceiveMailImpl(sender=msg.get('From'), to=msg.get_all('To'), subject=msg.get('Subject'), txtBodies=txtBodies, htmlBodies=htmlBodies, attachments=attachments) +#postfix的一个内容过滤器 mailglove 转发的邮件 +# +@bpInBoundEmail.post("/mailglove") +def ReceiveMailGlove(): + msg = request.get_json(silent=True) + if not msg or not isinstance(msg, dict): + return "The content is invalid" + + sender = msg.get('envelopeSender', '') + to = msg.get('envelopeRecipient', '') + if not sender: + sender = msg.get('from', {}).get('text', '') + if not to: + to = msg.get('to', {}).get('text', '') + + subject = msg.get('subject', '') + txtBodies = msg.get('text', '') + htmlBodies = msg.get('html', '') + attachments = msg.get('attachments', '') + + return ReceiveMailImpl(sender=sender, to=to, subject=subject, txtBodies=txtBodies, + htmlBodies=htmlBodies, attachments=attachments) + #实际实现接收邮件功能 #sender/to/subject: 发件人,收件人,主题 #txtBodies: text/plain的邮件内容列表 @@ -102,7 +125,7 @@ def ReceiveMailImpl(sender, to, subject, txtBodies, htmlBodies, attachments): default_log.warning(f'Spam mail from : {sender}') return "Spam mail" - subject = DecodeSubject(subject) + subject = DecodeSubject(subject or 'NoSubject') #通过邮件触发一次“现在投递” if to.lower() == 'trigger': @@ -222,6 +245,11 @@ def ConvertTextToHtml(subject, text): #根据邮件内容,创建一个 BeautifulSoup 实例 def CreateMailSoup(subject, txtBodies, htmlBodies): + if txtBodies and not isinstance(txtBodies, list): + txtBodies = [str(txtBodies)] + if htmlBodies and not isinstance(htmlBodies, list): + htmlBodies = [str(htmlBodies)] + if not htmlBodies: #此邮件为纯文本邮件,将文本信息转换为HTML格式 html = ConvertTextToHtml(subject, '\n'.join(txtBodies or [])) htmlBodies = [html] if html else [] diff --git a/application/view/library.py b/application/view/library.py index c9bbfa4c..197b144c 100644 --- a/application/view/library.py +++ b/application/view/library.py @@ -13,7 +13,7 @@ bpLibrary = Blueprint('bpLibrary', __name__) -GITHUB_SHARED_RSS = 'https://github.com/cdhigh/KindleEar/blob/master/application/recipes/shared_rss.json' +GITHUB_SHARED_RSS = 'https://raw.githubusercontent.com/cdhigh/KindleEar/master/application/recipes/shared_rss.json' g_ke_url = KINDLEEAR_SITE #给网友提供共享的订阅源数据,初始只返回一个空白页,然后在页面内使用ajax获取数据,参加 SharedLibraryMgrPost() diff --git a/application/view/library_offical.py b/application/view/library_offical.py index 492a7526..9738a341 100644 --- a/application/view/library_offical.py +++ b/application/view/library_offical.py @@ -3,7 +3,7 @@ #网友共享的订阅源数据,仅为 KINDLEEAR_SITE 使用 import datetime, json, hashlib from operator import attrgetter -from flask import Blueprint, render_template, request, Response +from flask import Blueprint, render_template, request, Response, current_app as app from flask_babel import gettext as _ from ..base_handler import * from ..utils import str_to_bool, str_to_int @@ -232,18 +232,63 @@ def SharedLibraryCategoryOffical(): cats = sorted(SharedRssCategory.get_all(), key=attrgetter('last_updated'), reverse=True) return [item.name for item in cats] -# @bpLibraryOffical.route('/translib') -# def TransferLib(): -# fileName = os.path.join(os.path.dirname(__file__), 'ke_final.json') -# with open(fileName, 'r', encoding='utf-8') as f: -# ke_final = json.loads(f.read()) -# SharedRss.delete().execute() -# SharedRssCategory.delete().execute() -# cats = set() -# for t in ke_final: -# cats.add(t['c']) -# SharedRss.create(title=t['t'], url=t['u'], isfulltext=t['f'], language=t['l'], category=t['c'], -# subscribed=t['s'], created_time=t['d']) -# for cat in cats: -# SharedRssCategory.create(name=cat, language='') -# return f'Finished, data count={len(ke_final)}, category count={len(cats)}' +#一次性导入共享库数据 +@bpLibraryOffical.post('/translib', endpoint='TransferLibPost') +@login_required() +def TransferLibPost(): + user = get_login_user() + key = request.form.get('key') + if user.name != app.config['ADMIN_NAME'] or key != user.share_links.get('key', ''): + return {} + + act = request.form.get('act') + src = request.form.get('src') + try: + if src == 'json': + sharedData = json.loads(request.form.get('data')) + else: + upload = request.files.get('data_file') + sharedData = json.loads(upload.read()) + except Exception as e: + return {'status': str(e)} + + if act == 'init': + SharedRss.delete().execute() + SharedRssCategory.delete().execute() + + cats = set() + for t in sharedData: + title = t.get('t', '') + url = t.get('u', '') + if not title or not url: + continue + + isfulltext = t.get('f', False) + language = t.get('l', '') + category = t.get('c', '') + subscribed = t.get('s', 1) + created_time = t.get('d', 0) + cats.add(category) + try: + created_time = datetime.datetime.utcfromtimestamp(created_time) + except: + created_time = None + + dbItem = SharedRss.get_or_none(SharedRss.title == title) or SharedRss(title=title, url=url) + dbItem.url = url + dbItem.isfulltext = isfulltext + if language: + dbItem.language = language + if category: + dbItem.category = category + if subscribed: + dbItem.subscribed = subscribed + if created_time: + dbItem.created_time = created_time + dbItem.save() + + for cat in cats: + if not SharedRssCategory.get_or_none(SharedRssCategory.name == cat): + SharedRssCategory.create(name=cat, language='') + UpdateLastSharedRssTime() + return f'Finished, data count={len(sharedData)}, category count={len(cats)}' diff --git a/application/view/login.py b/application/view/login.py index ab392d1b..7e25f1b2 100644 --- a/application/view/login.py +++ b/application/view/login.py @@ -189,7 +189,7 @@ def ResetPasswordPost(): #注册表单 @bpLogin.route("/signup") def Signup(): - if app.config['ALLOW_SIGNUP']: + if app.config['ALLOW_SIGNUP'] == 'yes': inviteNeed = AppInfo.get_value(AppInfo.signupType, 'oneTimeCode') != 'public' return render_template('signup.html', tips='', inviteNeed=inviteNeed) else: @@ -199,7 +199,7 @@ def Signup(): #注册验证 @bpLogin.post("/signup") def SignupPost(): - if not app.config['ALLOW_SIGNUP']: + if not app.config['ALLOW_SIGNUP'] == 'yes': tips = _("The website does not allow registration. You can ask the owner for an account.") return render_template('tipsback.html', title='not allow', urltoback='/', tips=tips) diff --git a/application/view/setting.py b/application/view/setting.py index 49d97882..05ab623c 100644 --- a/application/view/setting.py +++ b/application/view/setting.py @@ -40,11 +40,11 @@ def SettingPost(): sm_secret_key = form.get('sm_secret_key', '') sm_host = form.get('sm_host', '') sm_port = str_to_int(form.get('sm_port')) - #sm_username = form.get('sm_username', '') #replace by sender field + sm_username = form.get('sm_username', '') sm_password = form.get('sm_password', '') sm_save_path = form.get('sm_save_path', '') send_mail_service = {'service': sm_srv_type, 'apikey': sm_apikey, 'secret_key': sm_secret_key, - 'host': sm_host, 'port': sm_port, 'username': '', 'password': '', + 'host': sm_host, 'port': sm_port, 'username': sm_username, 'password': '', 'save_path': sm_save_path} #只有处于smtp模式并且密码存在才更新,空或几个星号则不更新 if sm_srv_type == 'smtp': @@ -110,7 +110,7 @@ def SendTestEmailPost(): Dear {user.name}, This is a test email from KindleEar, sent to verify the accuracy of the email sending configuration. - Please ignore it and do not reply. + Please do not reply it. Receiving this email confirms that your KindleEar web application can send emails successfully. Thank you for your attention to this matter. @@ -120,16 +120,20 @@ def SendTestEmailPost(): [From {srcUrl}] """) - if user.email: + emails = user.kindle_email.split(',') if user.kindle_email else [] + if user.email and user.email not in emails: + emails.append(user.email) + + if emails: status = 'ok' try: - send_mail(user, user.email, 'Test email from KindleEar', body) + send_mail(user, emails, 'Test email from KindleEar', body, attachments=[('test.txt', body.encode('utf-8'))]) except Exception as e: status = str(e) else: status = _("You have not yet set up your email address. Please go to the 'Admin' page to add your email address firstly.") - return {'status': status, 'email': user.email} + return {'status': status, 'emails': emails} #设置国际化语种 @bpSetting.route("/setlocale/") diff --git a/application/view/subscribe.py b/application/view/subscribe.py index 5695bdd4..6ee74fa1 100644 --- a/application/view/subscribe.py +++ b/application/view/subscribe.py @@ -9,7 +9,7 @@ from flask_babel import gettext as _ from ..base_handler import * from ..back_end.db_models import * -from ..utils import str_to_bool +from ..utils import str_to_bool, xml_escape from ..lib.urlopener import UrlOpener from ..lib.recipe_helper import GetBuiltinRecipeInfo, GetBuiltinRecipeSource from .library import LIBRARY_MGR, SUBSCRIBED_FROM_LIBRARY, LIBRARY_GETSRC, buildKeUrl @@ -444,15 +444,13 @@ def ViewRecipeSourceCode(id_): if recipeType == 'upload': recipe = Recipe.get_by_id_or_none(dbId) if recipe and recipe.src: - src = recipe.src.replace('<', '<').replace('>', '>') - return htmlTpl.format(title=recipe.title, body=src) + return htmlTpl.format(title=recipe.title, body=xml_escape(recipe.src)) else: return htmlTpl.format(title="Error", body=_('The recipe does not exist.')) else: #内置recipe recipe = GetBuiltinRecipeInfo(recipeId) src = GetBuiltinRecipeSource(recipeId) if recipe and src: - src = src.replace('<', '<').replace('>', '>') - return htmlTpl.format(title=recipe.get('title'), body=src) + return htmlTpl.format(title=recipe.get('title'), body=xml_escape(src)) else: return htmlTpl.format(title="Error", body=_('The recipe does not exist.')) diff --git a/config.py b/config.py index 6a044ec5..8d31b44b 100644 --- a/config.py +++ b/config.py @@ -1,13 +1,15 @@ #!/usr/bin/env python3 # -*- coding:utf-8 -*- -"""For KindleEar configuration, the first several variables need to be modified. -If some configurations need to be obtained through environment variables, -you can use the form os.environ['name'] -""" -import os +# KindleEar configuration. +# All configuration variables in this file can be set via environment variables too. +# Values from this file are only used if the environment variables are not present. +# +# 这个文件里面的所有配置变量都可以通过环境变量来设置。 +# 程序会优先使用环境变量,只有环境变量不存在时才使用此文件中的数值。 +# -APP_ID = os.getenv("APP_ID") or "kindleear" -APP_DOMAIN = os.getenv("APP_DOMAIN") or "https://kindleear.appspot.com" +APP_ID = "kindleear" +APP_DOMAIN = "https://kindleear.appspot.com" #Need for google taskqueue only, Refers to #Find it at Upper right corner of @@ -20,43 +22,39 @@ #DATABASE_URL = "mongodb://127.0.0.1:27017/" #DATABASE_URL = 'sqlite:////home/ubuntu/site/kindleear/database.db' #DATABASE_URL = 'sqlite:///database.db' -DATABASE_URL = os.getenv('DATABASE_URL') or 'datastore' - -#Email receiving service, "gae", "" -INBOUND_EMAIL_SERVICE = "" +DATABASE_URL = 'datastore' #Select the type of task queue, "gae", "apscheduler", "celery", "rq", "" -TASK_QUEUE_SERVICE = os.getenv('TASK_QUEUE_SERVICE') or "apscheduler" +TASK_QUEUE_SERVICE = "apscheduler" #If task queue service is apscheduler, celery, rq #Options: 'redis://', 'mongodb://', 'sqlite://', 'mysql://', 'postgresql://' -#For apscheduler, it can be a empty str '' if a memory store is used +#For apscheduler, it can be 'memory'. (Only if gunicorn have one worker) #For rq, only 'redis://' is supported -TASK_QUEUE_BROKER_URL = os.getenv('TASK_QUEUE_BROKER_URL') or "redis://127.0.0.1:6379/" -#TASK_QUEUE_BROKER_URL = '' +#TASK_QUEUE_BROKER_URL = "redis://127.0.0.1:6379/" +TASK_QUEUE_BROKER_URL = "memory" #If this option is empty, temporary files will be stored in memory #Setting this option can reduce memory consumption, supports both relative and absolute paths -TEMP_DIR = "/tmp" if os.getenv("TEMP_DIR") is None else os.getenv("TEMP_DIR") +KE_TEMP_DIR = "" #If the depolyment plataform supports multi-threads, set this option will boost the download speed -DOWNLOAD_THREAD_NUM = 1 +DOWNLOAD_THREAD_NUM = "3" -#If the website allow visitors to signup or not -ALLOW_SIGNUP = False +#If the website allow visitors to signup or not, "yes"|"no" +ALLOW_SIGNUP = "no" #For security reasons, it's suggested to change the secret key. SECRET_KEY = "n7ro8QJI1qfe" -#------------------------------------------------------------------------------------ -#Configurations below this line generally do not need to be modified -#------------------------------------------------------------------------------------ - #The administrator's login name ADMIN_NAME = "admin" #You can use this public key or apply for your own key POCKET_CONSUMER_KEY = "50188-e221424f1c9ed0c010058aef" -#Hide the option 'local (debug)' of 'Send Mail Service' setting or not -HIDE_MAIL_TO_LOCAL = False +#Hide the option 'local (debug)' of 'Send Mail Service' setting or not, "yes"|"no" +HIDE_MAIL_TO_LOCAL = "yes" + +#'debug', 'info', 'warning', 'error', 'critical' +LOG_LEVEL = "warning" diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..13328bff --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,32 @@ +#step 1 +#FROM python:3.10.14-slim AS req_builder +FROM python:3.9.19-alpine AS req_builder +ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONUNBUFFERED 1 + +WORKDIR /usr/site/ +COPY ./config.py . +COPY ./tools/update_req.py . +RUN python update_req.py docker + +#step 2 +#FROM python:3.10.14-slim +FROM python:3.9.19-alpine +ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONUNBUFFERED 1 + +USER root +RUN mkdir -p /usr/site /data +WORKDIR /usr/site +RUN pip install --upgrade pip +COPY --from=req_builder /usr/site/requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY ./main.py . +COPY --from=req_builder /usr/site/config.py . +COPY ./application/ ./application/ +COPY ./docker/gunicorn.conf.py . + +EXPOSE 8000 + +CMD ["/usr/local/bin/gunicorn", "-c", "/usr/site/gunicorn.conf.py", "main:app"] diff --git a/docker/compose_up.sh b/docker/compose_up.sh new file mode 100644 index 00000000..a1a58701 --- /dev/null +++ b/docker/compose_up.sh @@ -0,0 +1,2 @@ +sudo docker-compose rm -fs +sudo docker-compose up -d diff --git a/docker/default.conf b/docker/default.conf new file mode 100644 index 00000000..92fe5cf2 --- /dev/null +++ b/docker/default.conf @@ -0,0 +1,48 @@ +server { + listen 80 default_server; + listen [::]:80 default_server; + charset utf-8; + client_max_body_size 32M; + + # SSL configuration + # + # listen 443 ssl default_server; + # listen [::]:443 ssl default_server; + # + # Note: You should disable gzip for SSL traffic. + # See: https://bugs.debian.org/773332 + # + # Read up on ssl_ciphers to ensure a secure configuration. + # See: https://bugs.debian.org/765782 + # + # Self signed certs generated by the ssl-cert package + # Don't use them in a production server! + # + # include snippets/snakeoil.conf; + + root /var/www/html; + + # Add index.php to the list if you are using PHP + index index.html index.htm index.nginx-debian.html; + + server_name localhost; + + location /static/ { + proxy_pass http://kindleear:8000/static/; + } + location /images/ { + proxy_pass http://kindleear:8000/images/; + } + location = /favicon.ico { + proxy_pass http://kindleear:8000/static/favicon.ico; + } + location = /robots.txt { + proxy_pass http://kindleear:8000/static/robots.txt; + } + location / { + proxy_pass http://kindleear:8000; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } +} diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 00000000..e748e05e --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,49 @@ + +services: + kindleear: + container_name: kindleear + image: kindleear/kindleear + restart: always + volumes: + - ./data/:/data/ + expose: + - "8000" + networks: + - web_network + environment: + APP_ID: kindleear + APP_DOMAIN: https://kindleear.appspot.com + LOG_LEVEL: warning + + nginx: + container_name: nginx + image: nginx:stable-alpine3.17-slim + restart: always + ports: + - "80:80" + volumes: + - ./default.conf:/etc/nginx/conf.d/default.conf:ro + depends_on: + - kindleear + networks: + - web_network + + # mailglove: + # container_name: mailglove + # image: thingless/mailglove + # restart: unless-stopped + # ports: + # - "25:25" + # depends_on: + # - kindleear + # - nginx + # environment: + # #change DOMAIN to your email domain + # DOMAIN: local + # URL: http://kindleear:8000/mailglove + # networks: + # - web_network + +networks: + web_network: + driver: bridge diff --git a/docker/gunicorn.conf.py b/docker/gunicorn.conf.py new file mode 100644 index 00000000..d2e26779 --- /dev/null +++ b/docker/gunicorn.conf.py @@ -0,0 +1,13 @@ +# gunicorn.conf.py +pythonpath = "/usr/local/lib/python3.10/site-packages" +bind = "0.0.0.0:8000" +workers = 1 +threads = 3 +accesslog = "/data/gunicorn.access.log" +errorlog = "/data/gunicorn.error.log" +capture_output = True +enable_stdio_inheritance = True +loglevel = "info" +#preload_app = True +#certfile = 'cert.pem' +#keyfile = 'key.pem' diff --git a/docker/run_docker.sh b/docker/run_docker.sh new file mode 100644 index 00000000..ec55f443 --- /dev/null +++ b/docker/run_docker.sh @@ -0,0 +1 @@ +sudo docker run -d -p 80:8000 -v /data:/data --restart always kindleear/kindleear diff --git a/docker/ubuntu_docker.sh b/docker/ubuntu_docker.sh new file mode 100644 index 00000000..3ffccd89 --- /dev/null +++ b/docker/ubuntu_docker.sh @@ -0,0 +1,10 @@ +#!/bin/bash +sudo apt install -y apt-transport-https ca-certificates curl software-properties-common gnupg lsb-release +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg +echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null +sudo apt update +sudo apt install -y docker-ce +sudo pip install docker-compose +sudo groupadd docker +sudo usermod -aG docker $USER +sudo systemctl enable docker diff --git a/docs/Chinese/2.config.md b/docs/Chinese/2.config.md index ba0f66eb..aed9f29d 100644 --- a/docs/Chinese/2.config.md +++ b/docs/Chinese/2.config.md @@ -11,9 +11,9 @@ sort: 2 | ------------------- | -------------------------------------------------------- | | APP_ID | 应用标识符,gae平台为应用ID,其他平台用于标识数据库等资源 | | APP_DOMAIN | 应用部署后的域名 | -| TEMP_DIR | 制作电子书时的临时目录,为空则使用内存保存临时文件 | +| KE_TEMP_DIR | 制作电子书时的临时目录,为空则使用内存保存临时文件 | | DOWNLOAD_THREAD_NUM | 下载网页的线程数量,需要目标平台支持多线程,最大值为5 | -| ALLOW_SIGNUP | 是否允许用户注册,False为需要管理员创建账号 | +| ALLOW_SIGNUP | 是否允许用户注册,"yes"-用户可以自主注册(可以通过邀请码限制),"no"- 只有管理员能创建其他账号。| | SECRET_KEY | 浏览器session的加密密钥,建议修改,任意字符串即可 | | ADMIN_NAME | 管理员的账号名 | | POCKET_CONSUMER_KEY | 用于稍后阅读服务Pocket,可以用你自己的Key或就直接使用这个 | diff --git a/docs/Chinese/3.deployment.md b/docs/Chinese/3.deployment.md index 5746e4f0..93932e73 100644 --- a/docs/Chinese/3.deployment.md +++ b/docs/Chinese/3.deployment.md @@ -12,6 +12,8 @@ KindleEar支持多种平台部署,我只在这里列出一些我测试通过 1. config.py关键参数样例 ```python +APP_ID = "kindleear" +APP_DOMAIN = "https://kindleear.appspot.com" SERVER_LOCATION = "us-central1" DATABASE_URL = "datastore" TASK_QUEUE_SERVICE = "gae" @@ -41,19 +43,74 @@ gcloud beta app deploy --version=1 app.yaml 6. 部署成功后先到 [GAE后台](https://console.cloud.google.com/appengine/settings/emailsenders) 将你的发件地址添加到 "Mail API Authorized Senders",否则投递会出现 "Unauthorized sender" 错误。 -7. 如果你之前已经部署过Python2版本的KindleEar,建议新建一个项目来部署Python3版本,因为Python3版本消耗资源更多,而且第二代运行时更贵,因GAE不再支持Python2部署,所以覆盖后无法恢复原先的版本。 +7. 如果你之前已经部署过Python2版本的KindleEar,建议新建一个项目来部署Python3版本,因GAE不再支持Python2部署,所以覆盖后无法恢复原先的版本。 + + + + + +## Docker (VPS) +Docker是什么?如果你不了解,就把它类比为Windows平台的绿色软件的增强版。 +Docker不限平台,只要目标平台支持Docker,资源足够就可以部署。 +发布在docker hub的KindleEar映像采用最简原则,如果你需要其他配置,可以自己修改Dockerfile重新创建。 +- 使用sqlite数据库和apscheduler的内存队列。 +- 数据库文件和log文件保存到同一目录 /data 。 + +1. [安装Docker](https://docs.docker.com/engine/install/) (已安装则跳过) +每个平台的安装方法不一样,KindleEar提供了一个ubuntu的脚本。 +```bash +wget -O - https://raw.githubusercontent.com/cdhigh/KindleEar/master/docker/ubuntu_docker.sh | bash +``` + +2. 安装完Docker后,执行一条命令就可以让服务运行起来(yourid/yourdomain修改为你自己的值)。 +命令执行后就使用浏览器 http://ip 确认服务是否正常运行。 +因为使用了 restart 参数,所以系统重启后会自动重启此服务。 +```bash +mkdir data #for database and logs, you can use any folder (change ./data to your folder) +sudo docker run -d -p 80:8000 -v ./data:/data --restart always -e APP_ID=yourid -e APP_DOMAIN=yourdomain -e LOG_LEVEL=warning kindleear/kindleear +``` + +如果连不上,请确认80端口是否已经开放,不同的平台开放80端口的方法不一样,可能为iptables或ufw。 +比如: +```bash +sudo iptables -I INPUT 6 -m state --state NEW -p tcp --dport 80 -j ACCEPT +sudo iptables -I INPUT 6 -m state --state NEW -p tcp --dport 443 -j ACCEPT +sudo netfilter-persistent save +``` + +3. 上面的命令直接使用gunicorn为web服务器,尽管对于我们的应用来说,绰绰有余,而且还能省点资源。 +但是你感觉有些不专业,希望使用更强大的nginx,并且也想启用邮件中转功能(需要开放25端口并且申请域名): +```bash +mkdir data #for database and logs +wget https://raw.githubusercontent.com/cdhigh/KindleEar/master/docker/docker-compose.yml +wget https://raw.githubusercontent.com/cdhigh/KindleEar/master/docker/default.conf + +#Change the environ variables APP_ID/APP_DOMAIN +#If the email feature is needed, uncomment the section mailglove and change the DOMAIN. +vim ./docker-compose.yml + +sudo docker compose up -d +``` + +4. 需要查询日志文件 +```bash +tail -n 100 ./data/gunicorn.error.log +tail -n 100 ./data/gunicorn.access.log +``` ## Oracle cloud (VPS) +这是手动在一个VPS上部署的步骤,比较复杂,一般不建议,如果没有特殊要求,推荐使用docker映像。 + 1. config.py关键参数样例 ```python DATABASE_URL = "sqlite:////home/ubuntu/site/kindleear/database.db" TASK_QUEUE_SERVICE = "apscheduler" TASK_QUEUE_BROKER_URL = "redis://127.0.0.1:6379/" -TEMP_DIR = "/tmp" -DOWNLOAD_THREAD_NUM = 1 +KE_TEMP_DIR = "/tmp" +DOWNLOAD_THREAD_NUM = "3" ``` 2. 创建一个计算实例,选择的配置建议"符合始终免费条件",映像选择自己熟悉的,我选择的是ubuntu minimal。 @@ -72,7 +129,7 @@ passwd ``` -5. 然后就是命令时间 +5. 然后就是命令行时间 ```bash sudo apt update sudo apt upgrade @@ -88,8 +145,7 @@ sudo systemctl enable redis-server curl localhost #test if nginx works well sudo apt install vim-common -mkdir ~/site -mkdir ~/log +mkdir -p ~/site cd ~/site #fetch code from github, or you can upload code files by xftp/scp @@ -110,9 +166,13 @@ sudo iptables -I INPUT 6 -m state --state NEW -p tcp --dport 80 -j ACCEPT sudo iptables -I INPUT 6 -m state --state NEW -p tcp --dport 443 -j ACCEPT sudo netfilter-persistent save +mkdir -p /var/log/gunicorn/ +chown ubuntu:ubuntu /var/log/gunicorn/ #current user have right to write log +sudo cp ./tools/nginx/gunicorn_logrotate /etc/logrotate.d/gunicorn #auto split log file + #modify nginx configuration -vim ./tools/nginx/default #optional, change server_name if you want -sudo cp -rf ./tools/nginx/default /etc/nginx/sites-enabled/default +vim ./tools/nginx/nginx_default #optional, change server_name if you want +sudo cp -rf ./tools/nginx/nginx_default /etc/nginx/sites-enabled/default sudo nginx -t #test if nginx config file is correct #set gunicorn auto start @@ -138,9 +198,9 @@ sudo systemctl status gunicorn #确认running 8. 出现错误后,查询后台log的命令 ```bash -cat /var/log/nginx/error.log | tail -n 100 -cat /home/ubuntu/log/gunicorn.error.log | tail -n 100 -cat /home/ubuntu/log/gunicorn.access.log | tail -n 100 +tail -n 100 /var/log/nginx/error.log +tail -n 100 /var/log/gunicorn/error.log +tail -n 100 /var/log/gunicorn/access.log ``` 9. 后语,如果部署在Oracle cloud,建议开启其"OCI Email Delivery"服务,然后使用SMTP发送邮件,单邮件最大支持60MB,我还没有发现有哪家服务商能支持那么大的邮件。 diff --git a/docs/Chinese/README.md b/docs/Chinese/README.md index ae98ef10..7d459f17 100644 --- a/docs/Chinese/README.md +++ b/docs/Chinese/README.md @@ -7,6 +7,7 @@ sort: 2 - [配置项描述](https://cdhigh.github.io/Chinese/2.config.html) - [部署方法](https://cdhigh.github.io/Chinese/3.deployment.html) - [Google cloud (PaaS)](https://cdhigh.github.io/Chinese/3.deployment.html#gae) + - [Docker (VPS)](https://cdhigh.github.io/English/3.deployment.html#docker) - [Oracle cloud (VPS)](https://cdhigh.github.io/Chinese/3.deployment.html#oracle-cloud) - [Pythonanywhere (PaaS)](https://cdhigh.github.io/Chinese/3.deployment.html#python-anywhere) diff --git a/docs/English/2.config.md b/docs/English/2.config.md index 59c252aa..98da58b7 100644 --- a/docs/English/2.config.md +++ b/docs/English/2.config.md @@ -12,9 +12,9 @@ This section describes several simple configuration items, with more detailed de | ------------------ | ------------------------------------------------------ | | APP_ID | Application identifier; for GAE platform, it's the app ID, while for other platforms, it's used to identify database and other resources | | APP_DOMAIN | Domain name of the deployed application | -| TEMP_DIR | Temporary directory for creating eBooks; if empty, temporary files are stored in memory | +| KE_TEMP_DIR | Temporary directory for creating eBooks; if empty, temporary files are stored in memory | | DOWNLOAD_THREAD_NUM| Number of threads for downloading web pages; the target platform needs to support multithreading, with a maximum value of 5 | -| ALLOW_SIGNUP | Whether to allow user registration; False means account creation is done by administrators | +| ALLOW_SIGNUP | Whether to allow user registration; "yes" - Users can register autonomously (with the option to restrict via invitation codes), "no" - means account creation is done by administrators | | SECRET_KEY | Encryption key for browser session, recommended to change, any string is acceptable | | ADMIN_NAME | Administrator's username | | POCKET_CONSUMER_KEY| Used for Pocket's read-later service; you can use your own key or use this one directly | diff --git a/docs/English/3.deployment.md b/docs/English/3.deployment.md index 2146e111..82af19e6 100644 --- a/docs/English/3.deployment.md +++ b/docs/English/3.deployment.md @@ -14,6 +14,8 @@ Note: If you have previously deployed the Python 2 version of KindleEar, you can 1. config.py Key Parameter Example ```python +APP_ID = "kindleear" +APP_DOMAIN = "https://kindleear.appspot.com" SERVER_LOCATION = "us-central1" DATABASE_URL = "datastore" TASK_QUEUE_SERVICE = "gae" @@ -43,20 +45,74 @@ gcloud beta app deploy --version=1 app.yaml 6. After successful deployment, go to the [GAE console](https://console.cloud.google.com/appengine/settings/emailsenders) and add your sender address to "Mail API Authorized Senders" to prevent "Unauthorized sender" errors during delivery. -7. If you have previously deployed Python2 version of KindleEar, it is advisable to create a new project to deploy the Python3 version. This is because the Python 3 version consumes more resources, and the second-generation runtime is more expensive. Additionally, since GAE no longer supports Python 2 deployment, reverting to the original version after overwriting is not possible. +7. If you have previously deployed Python2 version of KindleEar, it is advisable to create a new project to deploy the Python3 version. Since GAE no longer supports Python 2 deployment, reverting to the original version after overwriting is not possible. + +## Docker (VPS) +What is Docker? just think of it as an enhanced version of portable software. + +The KindleEar image published on Docker Hub follows the principle of simplicity. If you need other configurations, you can modify the Dockerfile and rebuild it yourself. +- Utilizes SQLite database and APScheduler's memory job store. +- Database and log files are stored in the same directory `/data`. + +1. [Install Docker](https://docs.docker.com/engine/install/) (Skip if already installed) +Installation methods vary for each platform. KindleEar provides a script for Ubuntu. +```bash +wget -O - https://raw.githubusercontent.com/cdhigh/KindleEar/master/docker/ubuntu_docker.sh | bash +``` + +2. Execute a command to start the service (replace `yourid/yourdomain` with your own values). +Confirm if the service is running properly by visiting http://ip in a browser. +The service will automatically restart after system reboot due to the `restart` parameter. +```bash +mkdir data #for database and logs, you can use any folder (change ./data to your folder) +sudo docker run -d -p 80:8000 -v ./data:/data --restart always -e APP_ID=yourid -e APP_DOMAIN=yourdomain kindleear/kindleear +``` + +If unable to connect, ensure port 80 is open. Methods to open port 80 vary across platforms, such as iptables or ufw. +For example: +```bash +sudo iptables -I INPUT 6 -m state --state NEW -p tcp --dport 80 -j ACCEPT +sudo iptables -I INPUT 6 -m state --state NEW -p tcp --dport 443 -j ACCEPT +sudo netfilter-persistent save +``` + +3. The above command directly uses Gunicorn as the web server, which is more than sufficient for our application and saves resources. +However, if you feel it's unprofessional and want to use the more powerful Nginx, and also enable email forwarding (requires opening port 25 and registering a domain): +```bash +mkdir data #for database and logs +wget https://raw.githubusercontent.com/cdhigh/KindleEar/master/docker/docker-compose.yml +wget https://raw.githubusercontent.com/cdhigh/KindleEar/master/docker/default.conf + +# Change the environment variables APP_ID/APP_DOMAIN +# If the email feature is needed, uncomment the section mailglove and change the DOMAIN. +vim ./docker-compose.yml + +sudo docker compose up -d +``` + +4. To check log files: +```bash +tail -n 100 ./data/gunicorn.error.log +tail -n 100 ./data/gunicorn.access.log +``` + + -## Oracle cloud (VPS) +## Oracle cloud (VPS) +These are manual deployment steps on a VPS, which can be quite complex. Generally, it's not recommended. +If there are no specific requirements, it's advisable to use Docker images instead. + 1. config.py Key Parameter Example ```python DATABASE_URL = "sqlite:////home/ubuntu/site/kindleear/database.db" TASK_QUEUE_SERVICE = "apscheduler" TASK_QUEUE_BROKER_URL = "redis://127.0.0.1:6379/" -TEMP_DIR = "/tmp" -DOWNLOAD_THREAD_NUM = 1 +KE_TEMP_DIR = "/tmp" +DOWNLOAD_THREAD_NUM = "3" ``` 2. Create a compute instance, with the recommended configuration being "Always Free". @@ -116,8 +172,8 @@ sudo netfilter-persistent save #1. change the first line to 'user loginname;', for example 'user ubuntu;' #2. add a line 'client_max_body_size 16M;' to http section sudo vim /etc/nginx/nginx.conf -vim ./tools/nginx/default #change server_name if you want -sudo cp -rf ./tools/nginx/default /etc/nginx/sites-enabled/default +vim ./tools/nginx/nginx_default #change server_name if you want +sudo cp -rf ./tools/nginx/nginx_default /etc/nginx/sites-enabled/default sudo nginx -t #test if nginx config file is correct #set gunicorn auto start diff --git a/docs/English/README.md b/docs/English/README.md index 06a3abc1..9ff41f29 100644 --- a/docs/English/README.md +++ b/docs/English/README.md @@ -7,6 +7,7 @@ sort: 1 - [Configuration](https://cdhigh.github.io/English/2.config.html) - [Deployment](https://cdhigh.github.io/English/3.deployment.html) - [Google cloud (PaaS)](https://cdhigh.github.io/English/3.deployment.html#gae) + - [Docker (VPS)](https://cdhigh.github.io/English/3.deployment.html#docker) - [Oracle cloud (VPS)](https://cdhigh.github.io/English/3.deployment.html#oracle-cloud) - [Pythonanywhere (PaaS)](https://cdhigh.github.io/English/3.deployment.html#python-anywhere) diff --git a/main.py b/main.py index 637bff08..5dc57442 100644 --- a/main.py +++ b/main.py @@ -7,7 +7,7 @@ __Version__ = '3.0.0b' import os, sys, builtins, logging -from config import * +import config appDir = os.path.dirname(os.path.abspath(__file__)) #logName = None if (DATABASE_URL == 'datastore') else 'gunicorn.error' @@ -18,39 +18,32 @@ builtins.__dict__['appVer'] = __Version__ sys.path.insert(0, os.path.join(appDir, 'application', 'lib')) -#将config.py里面的部分配置信息写到 os.environ ,因为有些部分可能不依赖flask运行 +#合并config.py配置信息到os.environ,如果对应环境变量存在,则不会覆盖 def set_env(): - if not TEMP_DIR: - os.environ['TEMP_DIR'] = '' - elif os.path.isabs(TEMP_DIR): - os.environ['TEMP_DIR'] = TEMP_DIR - else: - os.environ['TEMP_DIR'] = os.path.join(appDir, TEMP_DIR) - os.environ['DOWNLOAD_THREAD_NUM'] = str(int(DOWNLOAD_THREAD_NUM)) - os.environ['DATABASE_URL'] = DATABASE_URL - os.environ['TASK_QUEUE_SERVICE'] = TASK_QUEUE_SERVICE - os.environ['TASK_QUEUE_BROKER_URL'] = TASK_QUEUE_BROKER_URL - os.environ['APP_ID'] = APP_ID - os.environ['APP_DOMAIN'] = APP_DOMAIN - os.environ['SERVER_LOCATION'] = SERVER_LOCATION - os.environ['ADMIN_NAME'] = ADMIN_NAME - os.environ['HIDE_MAIL_TO_LOCAL'] = '1' if HIDE_MAIL_TO_LOCAL else '' + cfgMap = {} + keys = ['APP_ID', 'APP_DOMAIN', 'SERVER_LOCATION', 'DATABASE_URL', 'TASK_QUEUE_SERVICE', + 'TASK_QUEUE_BROKER_URL', 'KE_TEMP_DIR', 'DOWNLOAD_THREAD_NUM', 'ALLOW_SIGNUP', + 'SECRET_KEY', 'ADMIN_NAME', 'POCKET_CONSUMER_KEY', 'HIDE_MAIL_TO_LOCAL', 'LOG_LEVEL'] + for key in keys: + cfgMap[key] = os.getenv(key) if key in os.environ else getattr(config, key) + os.environ[key] = cfgMap[key] + return cfgMap -set_env() +#设置logging的level +def set_log_level(level): + level = logging._nameToLevel.get(level.upper(), logging.WARNING) if isinstance(level, str) else level + for handler in logging.root.handlers: + handler.setLevel(level) + +cfgMap = set_env() from application import init_app -app = init_app(__name__, debug=False) +app = init_app(__name__, cfgMap, debug=False) celery_app = app.extensions.get("celery", None) -log.setLevel(logging.INFO) #logging.DEBUG, WARNING +set_log_level(cfgMap.get('LOG_LEVEL')) def main(): if len(sys.argv) == 2 and sys.argv[1] == 'debug': - #os.environ['DATASTORE_DATASET'] = app.config['APP_ID'] - #os.environ['DATASTORE_EMULATOR_HOST'] = 'localhost:8081' - #os.environ['DATASTORE_EMULATOR_HOST_PATH'] = 'localhost:8081/datastore' - #os.environ['DATASTORE_HOST'] = 'http://localhost:8081' - #os.environ['DATASTORE_PROJECT_ID'] = app.config['APP_ID'] - #cmd to start datastore emulator: gcloud beta emulators datastore start default_log.setLevel(logging.DEBUG) app.run(host='0.0.0.0', debug=False) return 0 @@ -65,10 +58,6 @@ def main(): from application.work.worker import WorkerAllNow print(WorkerAllNow()) return 0 - elif (act == 'db') and (param == 'create'): - from application.back_end.db_models import create_database_tables - print(create_database_tables()) - return 0 elif (act == 'log') and (param == 'purge'): from application.view.logs import RemoveLogs print(RemoveLogs()) @@ -79,7 +68,6 @@ def main(): print('\nUsage: main.py commands') print('\ncommands:') print(' debug \t Run the application in debug mode') - print(' db create \t Create database tables') print(' deliver check\t Start delivery if time set is matched') print(' deliver now \t Force start delivery task') print(' log purge \t remove logs older than one month') diff --git a/readme.md b/readme.md index 0e6ce1a0..b2eb3a17 100644 --- a/readme.md +++ b/readme.md @@ -5,7 +5,7 @@ __English__ · [简体中文](readme_zh.md) # Announcement March 10, 2024 Official release of 3.0 beta, KindleEar is ready for deployment now. -The author has successfully deployed it to GAE/Oracle Cloud/PythonAnywhere. +The author has successfully deployed it to GAE/Docker/Oracle Cloud/PythonAnywhere. **Significant Updates:** * Supports Python 3 @@ -13,6 +13,7 @@ The author has successfully deployed it to GAE/Oracle Cloud/PythonAnywhere. * Support for multiple platforms, no longer limited to the GAE platform * Support for Calibre's recipe format directly * Preset with over 1,000 Calibre builtin recipe files +* Built-in bilingual translation feature @@ -22,8 +23,8 @@ It automatically aggregates various web content into epub/mobi and delivers it t ## The features included: -* Unlimited RSS/ATOM or web content collection with support for Calibre's recipe format -* Unlimited custom RSS, directly enter RSS/ATOM link and title for automatic push +* Unlimited RSS/ATOM/JSON or web content collection with support for Calibre's recipe format +* Unlimited custom feeds, directly enter RSS/ATOM/JSON link and title for automatic push * Multiple account management, supporting multiple users and multiple Kindles * Generate epub/mobi with images and table of contents * Automatic daily scheduled push @@ -33,11 +34,13 @@ It automatically aggregates various web content into epub/mobi and delivers it t -![Screenshot](https://raw.githubusercontent.com/cdhigh/KindleEar/master/docs/scrshot.gif) +For other details, please refer to the **[project documentation](https://cdhigh.github.io/KindleEar)**. + +![Screenshot](https://raw.githubusercontent.com/cdhigh/KindleEar/master/docs/scrshot.gif) + -For other details, please refer to the [project documentation](https://cdhigh.github.io/KindleEar). # License diff --git a/readme_zh.md b/readme_zh.md index ee484cde..f5855290 100644 --- a/readme_zh.md +++ b/readme_zh.md @@ -4,7 +4,7 @@ # 公告 2024-03-10 -正式发布3.0 beta版,代码库已经可以用来部署,作者已经将其成功部署到GAE/Oracle Cloud/pythonAnywhere。 +正式发布3.0 beta版,代码库已经可以用来部署,作者已经将其成功部署到GAE/Docker/Oracle Cloud/pythonAnywhere。 **主要新特性:** * 支持Python 3 @@ -12,6 +12,7 @@ * 多平台支持,不再受限于gae平台 * 支持不用修改的Calibre的recipe格式 * 预置Calibre的一千多个recipe文件 +* 内置双语对照翻译功能,打破语言障碍,获取信息和学习外语同步进行 @@ -22,8 +23,8 @@ 此应用目前的主要功能有: -* 支持Calibre的recipe格式的不限量RSS/ATOM或网页内容收集 -* 不限量自定义RSS,直接输入RSS/ATOM链接和标题即可自动推送 +* 支持Calibre的recipe格式的不限量RSS/ATOM/JSON或网页内容收集 +* 不限量自定义RSS,直接输入RSS/ATOM/JSON链接和标题即可自动推送 * 多账号管理,支持多用户和多Kindle * 生成带图像有目录的epub/mobi * 自动每天定时推送 @@ -32,12 +33,14 @@ * 和Evernote/Pocket/Instapaper等系统的集成 +其他细节请参考 **[项目文档](https://cdhigh.github.io/KindleEar)** + + ![Screenshot](https://raw.githubusercontent.com/cdhigh/KindleEar/master/docs/scrshot.gif) -其他细节请参考 [项目文档](https://cdhigh.github.io/KindleEar) diff --git a/requirements.txt b/requirements.txt index 7e8ac5cc..e3f1e657 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,17 +15,17 @@ Flask>=3.0.0,<4.0.0 flask-babel>=4.0.0,<5.0.0 six>=1.16.0,<2.0.0 feedparser>=6.0.11,<7.0.0 -weedata>=0.2.1,<1.0.0 -google-cloud-datastore>=2.19.0,<3.0.0 -google-cloud-tasks>=2.15.0,<3.0.0 -appengine-python-standard>=1.1.6,<2.0.0 +peewee>=3.1.7,<4.0.0 +flask-apscheduler>=1.13.1,<2.0.0 -#peewee>=3.1.7,<4.0.0 #pymysql>=1.1.0,<2.0.0 #psycopg2>=2.9.9,<3.0.0 +#weedata>=0.2.1,<1.0.0 +#google-cloud-datastore>=2.19.0,<3.0.0 #pymongo>=3.7.2,<4.0.0 #redis>=4.5.0,<6.0.0 -#flask-apscheduler>=1.13.1,<2.0.0 +#google-cloud-tasks>=2.15.0,<3.0.0 #celery>=5.3.6,<6.0.0 #eventlet>=0.35.1,<1.0.0 #flask-rq2>=18.3,<19.0 +#appengine-python-standard>=1.1.6,<2.0.0 diff --git a/tests/readme.developer.md b/tests/readme.developer.md index 8e5f5a27..be09dbfd 100644 --- a/tests/readme.developer.md +++ b/tests/readme.developer.md @@ -1,4 +1,4 @@ -# KindleEar面向开发者备忘录 +# KindleEar开发者备忘录 # 本地环境构建和调试 1. 安装标准环境google cloud SDK/gloud CLI,并且执行 gcloud init @@ -64,6 +64,28 @@ gcloud app deploy queue.yaml * javascript的翻译没有采用其他复杂或引入其他依赖的方案,而是简单粗暴的在base.html里面将要翻译的字段预先翻译, 然后保存到一个全局字典 + +# Docker +## 构建镜像 +1. 将docker/Dockerfile文件拷贝到根目录 +2. 执行 +```bash +sudo docker build -t kindleear/kindleear . +#or +sudo docker build --no-cache -t kindleear/kindleear . +``` + +## 常用Docker命令 +```bash +sudo docker images +sudo docker rmi id +sudo docker stop name +sudo docker rm name +sudo docker ps -a +sudo docker compose up -d +sudo docker run -d +``` + # Python托管平台的一些了解 * [appengine](https://cloud.google.com):必须绑定信用卡,但有免费额度,有收发邮件服务,任务队列,后台进程 * [Heroku](https://www.heroku.com): 没有免费额度,入门套餐也需要付费 diff --git a/tests/runtests.py b/tests/runtests.py index 7727707b..f4eab30c 100644 --- a/tests/runtests.py +++ b/tests/runtests.py @@ -15,21 +15,15 @@ from config import * -#将config.py里面的部分配置信息写到 os.environ,因为有些部分可能不依赖flask运行 def set_env(): - if not TEMP_DIR: - os.environ['TEMP_DIR'] = '' - elif os.path.isabs(TEMP_DIR): - os.environ['TEMP_DIR'] = TEMP_DIR - else: - os.environ['TEMP_DIR'] = os.path.join(appDir, TEMP_DIR) - os.environ['DOWNLOAD_THREAD_NUM'] = str(int(DOWNLOAD_THREAD_NUM)) - os.environ['DATABASE_URL'] = 'sqlite://:memory:' - os.environ['TASK_QUEUE_SERVICE'] = '' - os.environ['TASK_QUEUE_BROKER_URL'] = '' - os.environ['APP_DOMAIN'] = APP_DOMAIN - os.environ['ADMIN_NAME'] = ADMIN_NAME - os.environ['HIDE_MAIL_TO_LOCAL'] = '1' if HIDE_MAIL_TO_LOCAL else '' + cfgMap = {} + keys = ['APP_ID', 'APP_DOMAIN', 'SERVER_LOCATION', 'DATABASE_URL', 'TASK_QUEUE_SERVICE', + 'TASK_QUEUE_BROKER_URL', 'KE_TEMP_DIR', 'DOWNLOAD_THREAD_NUM', 'ALLOW_SIGNUP', + 'SECRET_KEY', 'ADMIN_NAME', 'POCKET_CONSUMER_KEY', 'HIDE_MAIL_TO_LOCAL'] + for key in keys: + cfgMap[key] = os.getenv(key) if key in os.environ else getattr(config, key) + os.environ[key] = cfgMap[key] + return cfgMap set_env() diff --git a/tools/nginx/default b/tools/nginx/default deleted file mode 100644 index 5811a022..00000000 --- a/tools/nginx/default +++ /dev/null @@ -1,105 +0,0 @@ -## -# You should look at the following URL's in order to grasp a solid understanding -# of Nginx configuration files in order to fully unleash the power of Nginx. -# https://www.nginx.com/resources/wiki/start/ -# https://www.nginx.com/resources/wiki/start/topics/tutorials/config_pitfalls/ -# https://wiki.debian.org/Nginx/DirectoryStructure -# -# In most cases, administrators will remove this file from sites-enabled/ and -# leave it as reference inside of sites-available where it will continue to be -# updated by the nginx packaging team. -# -# This file will automatically load configuration files provided by other -# applications, such as Drupal or Wordpress. These applications will be made -# available underneath a path with that package name, such as /drupal8. -# -# Please see /usr/share/doc/nginx-doc/examples/ for more detailed examples. -## - -# Default server configuration -# -server { - listen 80 default_server; - listen [::]:80 default_server; - charset utf-8; - client_max_body_size 32M; - - # SSL configuration - # - # listen 443 ssl default_server; - # listen [::]:443 ssl default_server; - # - # Note: You should disable gzip for SSL traffic. - # See: https://bugs.debian.org/773332 - # - # Read up on ssl_ciphers to ensure a secure configuration. - # See: https://bugs.debian.org/765782 - # - # Self signed certs generated by the ssl-cert package - # Don't use them in a production server! - # - # include snippets/snakeoil.conf; - - root /var/www/html; - - # Add index.php to the list if you are using PHP - index index.html index.htm index.nginx-debian.html; - - server_name _; - - location /static/ { - alias /home/ubuntu/site/kindleear/application/static/; - } - location /images/ { - alias /home/ubuntu/site/kindleear/application/images/; - } - location = /favicon.ico { - alias /home/ubuntu/site/kindleear/application/static/favicon.ico; - } - location = /robots.ico { - alias /home/ubuntu/site/kindleear/application/static/robots.ico; - } - location / { - proxy_pass http://127.0.0.1:8000; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - - # pass PHP scripts to FastCGI server - # - #location ~ \.php$ { - # include snippets/fastcgi-php.conf; - # - # # With php-fpm (or other unix sockets): - # fastcgi_pass unix:/run/php/php7.4-fpm.sock; - # # With php-cgi (or other tcp sockets): - # fastcgi_pass 127.0.0.1:9000; - #} - - # deny access to .htaccess files, if Apache's document root - # concurs with nginx's one - # - #location ~ /\.ht { - # deny all; - #} -} - - -# Virtual Host configuration for example.com -# -# You can move that to a different file under sites-available/ and symlink that -# to sites-enabled/ to enable it. -# -#server { -# listen 80; -# listen [::]:80; -# -# server_name example.com; -# -# root /var/www/example.com; -# index index.html; -# -# location / { -# try_files $uri $uri/ =404; -# } -#} diff --git a/tools/nginx/gunicorn.conf.py b/tools/nginx/gunicorn.conf.py index 5b138bd3..1d14010c 100644 --- a/tools/nginx/gunicorn.conf.py +++ b/tools/nginx/gunicorn.conf.py @@ -3,8 +3,8 @@ bind = "127.0.0.1:8000" workers = 1 threads = 3 -accesslog = "/home/ubuntu/log/gunicorn.access.log" -errorlog = "/home/ubuntu/log/gunicorn.error.log" +accesslog = "/var/log/gunicorn/error.log" +errorlog = "/var/log/gunicorn/access.log" capture_output = True enable_stdio_inheritance = True loglevel = "info" diff --git a/tools/nginx/gunicorn_logrotate b/tools/nginx/gunicorn_logrotate new file mode 100644 index 00000000..06d9f090 --- /dev/null +++ b/tools/nginx/gunicorn_logrotate @@ -0,0 +1,9 @@ +/var/log/gunicorn/*.log { + daily + dateext + dateformat -%Y-%m-%d + dateyesterday + rotate 90 + missingok + notifempty +} diff --git a/tools/nginx/nginx_default b/tools/nginx/nginx_default new file mode 100644 index 00000000..1e3bef6b --- /dev/null +++ b/tools/nginx/nginx_default @@ -0,0 +1,106 @@ +@@ -1,105 +0,0 @@ +## +# You should look at the following URL's in order to grasp a solid understanding +# of Nginx configuration files in order to fully unleash the power of Nginx. +# https://www.nginx.com/resources/wiki/start/ +# https://www.nginx.com/resources/wiki/start/topics/tutorials/config_pitfalls/ +# https://wiki.debian.org/Nginx/DirectoryStructure +# +# In most cases, administrators will remove this file from sites-enabled/ and +# leave it as reference inside of sites-available where it will continue to be +# updated by the nginx packaging team. +# +# This file will automatically load configuration files provided by other +# applications, such as Drupal or Wordpress. These applications will be made +# available underneath a path with that package name, such as /drupal8. +# +# Please see /usr/share/doc/nginx-doc/examples/ for more detailed examples. +## + +# Default server configuration +# +server { + listen 80 default_server; + listen [::]:80 default_server; + charset utf-8; + client_max_body_size 32M; + + # SSL configuration + # + # listen 443 ssl default_server; + # listen [::]:443 ssl default_server; + # + # Note: You should disable gzip for SSL traffic. + # See: https://bugs.debian.org/773332 + # + # Read up on ssl_ciphers to ensure a secure configuration. + # See: https://bugs.debian.org/765782 + # + # Self signed certs generated by the ssl-cert package + # Don't use them in a production server! + # + # include snippets/snakeoil.conf; + + root /var/www/html; + + # Add index.php to the list if you are using PHP + index index.html index.htm index.nginx-debian.html; + + server_name _; + + location /static/ { + alias /home/ubuntu/site/kindleear/application/static/; + } + location /images/ { + alias /home/ubuntu/site/kindleear/application/images/; + } + location = /favicon.ico { + alias /home/ubuntu/site/kindleear/application/static/favicon.ico; + } + location = /robots.txt { + alias /home/ubuntu/site/kindleear/application/static/robots.txt; + } + location / { + proxy_pass http://127.0.0.1:8000; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + # pass PHP scripts to FastCGI server + # + #location ~ \.php$ { + # include snippets/fastcgi-php.conf; + # + # # With php-fpm (or other unix sockets): + # fastcgi_pass unix:/run/php/php7.4-fpm.sock; + # # With php-cgi (or other tcp sockets): + # fastcgi_pass 127.0.0.1:9000; + #} + + # deny access to .htaccess files, if Apache's document root + # concurs with nginx's one + # + #location ~ /\.ht { + # deny all; + #} +} + + +# Virtual Host configuration for example.com +# +# You can move that to a different file under sites-available/ and symlink that +# to sites-enabled/ to enable it. +# +#server { +# listen 80; +# listen [::]:80; +# +# server_name example.com; +# +# root /var/www/example.com; +# index index.html; +# +# location / { +# try_files $uri $uri/ =404; +# } +#} \ No newline at end of file diff --git a/tools/update_req.py b/tools/update_req.py index af79c4fb..b5208bbb 100644 --- a/tools/update_req.py +++ b/tools/update_req.py @@ -77,16 +77,54 @@ def config_to_dict(cfgFile): exec(code, globals(), config_dict) return config_dict -if __name__ == '__main__': - print('\nThis script can help you to generate requirements.txt.\n') - usrInput = input('Press y to continue :') - if usrInput.lower() != 'y': - sys.exit(0) +#prepare config.py to build docker +def dockered_config_py(cfgFile): + default_cfg = {'APP_ID': 'kindleear', 'DATABASE_URL': 'sqlite:////data/kindleear.db', + 'TASK_QUEUE_SERVICE': 'apscheduler', 'TASK_QUEUE_BROKER_URL': 'memory', + 'KE_TEMP_DIR': '/tmp', 'DOWNLOAD_THREAD_NUM': '3', 'ALLOW_SIGNUP': 'no', + 'HIDE_MAIL_TO_LOCAL': 'yes', 'LOG_LEVEL': 'warning'} + ret = [] + inDocComment = False + pattern = r"^([_A-Z]+)\s*=\s*(.+)$" + with open(cfgFile, 'r', encoding='utf-8') as f: + lines = f.read().splitlines() + for line in lines: + line = line.strip() + if '"""' in line or "'''" in line: + inDocComment = not inDocComment + ret.append(line) + continue + elif not line or line.startswith('#') or inDocComment: + ret.append(line) + continue + + match = re.match(pattern, line) + if match: + name = match.group(1) + value = default_cfg.get(name, match.group(2)) + ret.append(f'{name} = "{value}"') + else: + ret.append(line) + + with open(cfgFile, 'w', encoding='utf-8') as f: + f.write('\n'.join(ret)) +if __name__ == '__main__': thisDir = os.path.abspath(os.path.dirname(__file__)) cfgFile = os.path.normpath(os.path.join(thisDir, '..', 'config.py')) reqFile = os.path.normpath(os.path.join(thisDir, '..', 'requirements.txt')) + if not os.path.exists(cfgFile): + cfgFile = os.path.normpath(os.path.join(thisDir, 'config.py')) + reqFile = os.path.normpath(os.path.join(thisDir, 'requirements.txt')) + if len(sys.argv) == 2 and sys.argv[1] == 'docker': + dockered_config_py(cfgFile) + else: + print('\nThis script can help you to update requirements.txt.\n') + usrInput = input('Press y to continue :') + if usrInput.lower() != 'y': + sys.exit(1) + cfg = config_to_dict(cfgFile) db = cfg['DATABASE_URL'].split('://')[0] task = cfg['TASK_QUEUE_SERVICE'] @@ -104,3 +142,5 @@ def config_to_dict(cfgFile): extras.add('sqlalchemy') write_req(reqFile, db, task, plat, *extras) print(f'Finished create {reqFile}') + sys.exit(0) +