porting to python3

.gitignore

@@ -5,7 +5,8 @@
 *.mobi
 *.epub
 *.azw3
-
+*.pkl
+*.bak
 tests/tools/*
 tests/rss/*
 tests/debug_mail/*

application/__init__.py

@@ -10,6 +10,7 @@
 from flask import Flask, render_template, session, request, g
 from flask_babel import Babel, gettext
 builtins.__dict__['_'] = gettext
+builtins.__dict__['appVer'] = __Version__
 
 #创建并初始化Flask wsgi对象
 def init_app(name, debug=False):

application/back_end/db_models.py

@@ -1,14 +1,14 @@
 #!/usr/bin/env python3
 # -*- coding:utf-8 -*-
 #数据库结构定义，使用这个文件隔离sql和nosql的差异，尽量向外提供一致的接口
-#Visit https://github.com/cdhigh/KindleEar for the latest version
+#Visit <https://github.com/cdhigh/KindleEar> for the latest version
 #Author:
 # cdhigh <https://github.com/cdhigh>
 import os, sys, random
 from operator import attrgetter
 from ..utils import ke_encrypt, ke_decrypt, tz_now
 
-if os.getenv('DATABASE_ENGINE') in ("datastore", "mongodb", "redis"):
+if os.getenv('DATABASE_URL').startswith(("datastore", "mongodb", "redis", "pickle")):
     from .db_models_nosql import *
 else:
     from .db_models_sql import *
@@ -187,8 +187,8 @@ class WhiteList(MyBaseModel):
 
 #Shared RSS links from other users [for kindleear.appspot.com only]
 class SharedRss(MyBaseModel):
-    title = CharField()
-    url = CharField(default='')
+    title = CharField(index=True)
+    url = CharField(default='', index=True)
     isfulltext = BooleanField(default=False)
     language = CharField(default='')
     category = CharField(default='')
@@ -239,24 +239,11 @@ def set_value(cls, name, value):
         cls.replace(name=name, value=value).execute()
 
 #创建数据库表格，一个数据库只需要创建一次
-#如果是sql数据库，可以使用force=True删掉之前的数据库文件(小心)
-def create_database_tables(force=False):
-    engine = os.getenv('DATABASE_ENGINE')
-    #if engine == "sqlite" and dbName:
-    #    if not force and os.path.exists(dbName):
-    #        #print(f'[Error] Database "{dbName}" already exists')
-    #        return
-    #    elif os.path.exists(dbName):
-    #        try:
-    #            os.remove(dbName)
-    #        except:
-    #            pass
-
-    if engine not in ["datastore", "mongodb"]:
-        #with dbInstance.connection_context():
-        #connect_database()
-        dbInstance.create_tables([KeUser, UserBlob, Recipe, BookedRecipe, DeliverLog, WhiteList,
-            SharedRss, SharedRssCategory, LastDelivered, AppInfo], safe=True)
-        #close_database()
+def create_database_tables():
+    #with dbInstance.connection_context():
+    #connect_database()
+    dbInstance.create_tables([KeUser, UserBlob, Recipe, BookedRecipe, DeliverLog, WhiteList,
+        SharedRss, SharedRssCategory, LastDelivered, AppInfo], safe=True)
+    #close_database()
 
     #print(f'Create database "{dbName}" finished')

application/back_end/db_models_nosql.py

@@ -6,28 +6,19 @@
 import os, json, datetime
 from weedata import *
 
-__DB_ENGINE = os.getenv('DATABASE_ENGINE')
-__DB_NAME = os.getenv('DATABASE_NAME', '')
-__APP_ID = os.getenv('APP_ID', 'kindleear')
+dbUrl = os.getenv('DATABASE_URL')
+appId = os.getenv('APP_ID', 'kindleear')
 
-if __DB_NAME.startswith('mongodb://'):
-    dbInstance = MongoDbClient(__APP_ID, __DB_NAME)
-elif __DB_NAME.startswith('redis://'):
-    dbInstance = RedisDbClient(__APP_ID, __DB_NAME)
-elif __DB_ENGINE == "datastore":
-    dbInstance = DatastoreClient(project=__APP_ID)
-elif __DB_ENGINE == "mongodb":
-    dbInstance = MongoDbClient(__APP_ID, host=os.getenv('DATABASE_HOST'), port=int(os.getenv('DATABASE_PORT')), 
-        username=(os.getenv('DATABASE_USERNAME') or None), password=(os.getenv('DATABASE_PASSWORD') or None))
-elif __DB_ENGINE == "redis":
-    try:
-        db_no = int(os.getenv('DATABASE_NAME') or '0')
-    except:
-        db_no = 0
-    dbInstance = RedisDbClient(__APP_ID, host=os.getenv('DATABASE_HOST'), port=int(os.getenv('DATABASE_PORT')), 
-        db=db_no, password=(os.getenv('DATABASE_PASSWORD') or None))
+if dbUrl.startswith('mongodb://'):
+    dbInstance = MongoDbClient(appId, dbUrl)
+elif dbUrl.startswith('redis://'):
+    dbInstance = RedisDbClient(appId, dbUrl)
+elif dbUrl.startswith("datastore"):
+    dbInstance = DatastoreClient(project=appId)
+elif dbUrl.startswith("pickle://"):
+    dbInstance = PickleDbClient(dbUrl)
 else:
-    raise Exception("database engine '{}' not supported yet".format(__DB_ENGINE))
+    raise ValueError("database engine '{}' not supported yet".format(dbUrl.split(":", 1)[0]))
 
 #调用此函数正式连接到数据库（打开数据库）
 def connect_database():

application/back_end/db_models_sql.py

@@ -10,33 +10,8 @@
 
 #用于在数据库结构升级后的兼容设计，数据库结构和前一版本不兼容则需要升级此版本号
 DB_VERSION = 1
-
-__DB_ENGINE = os.getenv('DATABASE_ENGINE')
-__DB_NAME = os.getenv('DATABASE_NAME', '')
-__DB_USERNAME = os.getenv('DATABASE_USERNAME') or None
-__DB_PASSWORD = os.getenv('DATABASE_PASSWORD') or None
-__DB_HOST = os.getenv('DATABASE_HOST')
-__DB_PORT = int(os.getenv('DATABASE_PORT'))
-
-dbName = ''
-if '://' in __DB_NAME:
-    dbInstance = connect(__DB_NAME)
-    __DB_ENGINE = __DB_NAME.split('://', 1)[0]
-elif __DB_ENGINE == "sqlite":
-    thisDir = os.path.dirname(os.path.abspath(__file__))
-    dbName = os.path.normpath(os.path.join(thisDir, "..", "..", __DB_NAME)) if __DB_NAME != ':memory:' else __DB_NAME
-    dbInstance = SqliteDatabase(dbName, check_same_thread=False)
-elif __DB_ENGINE == "mysql":
-    dbInstance = MySQLDatabase(__DB_NAME, user=__DB_USERNAME, password=__DB_PASSWORD,
-                         host=__DB_HOST, port=__DB_PORT)
-elif __DB_ENGINE == "postgresql":
-    dbInstance = PostgresqlDatabase(__DB_NAME, user=__DB_USERNAME, password=__DB_PASSWORD,
-                         host=__DB_HOST, port=__DB_PORT)
-elif __DB_ENGINE == "cockroachdb":
-    dbInstance = CockroachDatabase(__DB_NAME, user=__DB_USERNAME, password=__DB_PASSWORD,
-                         host=__DB_HOST, port=__DB_PORT)
-else:
-    raise Exception("database engine '{}' not supported yet".format(__DB_ENGINE))
+dbName = os.getenv('DATABASE_URL')
+dbInstance = connect(dbName)
 
 #调用此函数正式连接到数据库（打开数据库）
 def connect_database():
@@ -46,7 +21,7 @@ def connect_database():
 #关闭数据库连接
 def close_database():
     global dbInstance
-    if not dbInstance.is_closed() and dbName != ':memory:':
+    if not dbInstance.is_closed() and dbName != 'sqlite://:memory:':
         dbInstance.close()
 
 #自定义字段，在本应用用来保存列表

application/back_end/send_mail_adpt.py

@@ -30,13 +30,15 @@
 
 #返回当前可用的发送邮件服务列表
 def avaliable_sm_services():
-    sm = ['local']
+    sm = []
     if gae_mail:
         sm.append('gae')
     if SendGridAPIClient:
         sm.append('sendgrid')
     if smtp_send_mail:
         sm.append('smtp')
+    if not os.getenv('HIDE_MAIL_TO_LOCAL'):
+        sm.append('local')
     return sm
 
 #发送邮件

application/back_end/task_queue_apscheduler.py

@@ -2,27 +2,42 @@
 # -*- coding:utf-8 -*-
 #任务队列APScheduler
 #Author: cdhigh <https://github.com/cdhigh>
-import random
+import os, random
 
 from flask_apscheduler import APScheduler
-#from apscheduler.events import EVENT_JOB_EXECUTED, EVENT_JOB_ERROR, EVENT_JOB_MISSED
+from apscheduler.schedulers.background import BackgroundScheduler
 
-scheduler = APScheduler()
+_broker_url = os.getenv('TASK_QUEUE_BROKER_URL')
+if _broker_url.startswith('redis://'):
+    import redis
+    from apscheduler.jobstores.redis import RedisJobStore
+    _client = RedisJobStore()
+    _client.redis = redis.from_url(_broker_url)
+    jobstores = {"default": _client}
+elif _broker_url.startswith('mongodb://'):
+    import pymongo
+    from apscheduler.jobstores.mongodb import MongoDBJobStore
+    _client = pymongo.MongoClient(_broker_url)
+    jobstores = {"default": MongoDBJobStore(client=_client)}
+elif _broker_url.startswith(('sqlite://', 'mysql://', 'postgresql://')):
+    from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore
+    jobstores = {"default": SQLAlchemyJobStore(url=_broker_url)}
+elif _broker_url == '':
+    jobstores = None #default is memory store
+else:
+    raise ValueError('Unsupported TASK_QUEUE_BROKER_URL type: {_broker_url}')
+
+scheduler = APScheduler(scheduler=BackgroundScheduler(jobstores=jobstores))
 
 #https://viniciuschiele.github.io/flask-apscheduler/rst/api.html
 scheduler.api_enabled = True #提供/scheduler/jobs等几个有用的url
 
 def init_task_queue_service(app):
     scheduler.init_app(app)
-    #scheduler.add_listener(job_listener, EVENT_JOB_EXECUTED | EVENT_JOB_ERROR | EVENT_JOB_MISSED)
     scheduler.start()
     app.extensions["scheduler"] = scheduler
     return scheduler
 
-#APScheduler会自动删除trigger为date的任务，这个函数不需要了
-#def job_listener(event):
-#    scheduler.remove_job(event.job_id)
-
 #@scheduler.task('interval', id='check_deliver', hours=1, misfire_grace_time=20*60, coalesce=True)
 @scheduler.task('cron', minute=50, id='check_deliver', misfire_grace_time=20*60, coalesce=True)
 def check_deliver():

application/back_end/task_queue_celery.py

@@ -6,6 +6,7 @@
 #启动celery
 #celery -A main.celery_app worker --loglevel=info --logfile=d:\celery.log --concurrency=2 -P eventlet
 #celery -A main.celery_app beat -s /home/celery/var/run/celerybeat-schedule --loglevel=info --logfile=d:\celery.log --concurrency=2 -P eventlet
+import os
 from celery import Celery, Task, shared_task
 from celery.schedules import crontab
 
@@ -14,10 +15,40 @@ class FlaskTask(Task):
         def __call__(self, *args, **kwargs):
             with app.app_context():
                 return self.run(*args, **kwargs)
-
+
+    broker_url = app.config['TASK_QUEUE_BROKER_URL']
+    backend_url = broker_url
+    transport_opts = {}
+    if broker_url.startswith(('sqlite://', 'mysql://', 'postgresql://')):
+        broker_url = f'sqla+{broker_url}'
+    elif broker_url.startswith('file://'): #using a filesystem, ensure the folder exists
+        if broker_url.startswith('file:////?/'): #windows
+            dir_ = broker_url[11:]
+        elif broker_url.startswith('file:///'): #linux/mac
+            dir_ = broker_url[8:]
+        else:
+            raise ValueError('The value of TASK_QUEUE_BROKER_URL is invalid')
+        dir_in = os.path.join(dir_, 'data_in')
+        dir_out = os.path.join(dir_, 'data_out')
+        dir_procsed = os.path.join(dir_, 'processed')
+        transport_opts = {'data_folder_in': dir_in, 'data_folder_out': dir_out, 'processed_folder': dir_procsed, 
+            'store_processed': True}
+        for d in [dir_, dir_in, dir_out, dir_procsed]:
+            if not os.path.exists(d):
+                os.makedirs(d)
+        broker_url = 'filesystem://'
+
+    if backend_url.startswith(('sqlite://', 'mysql://', 'postgresql://')):
+        backend_url = f'db+{backend_url}'
+
     app.config.from_mapping(
-        CELERY={'broker_url': app.config['TASK_QUEUE_BROKER_URL'],
-            'result_backend': app.config['TASK_QUEUE_RESULT_BACKEND'],
+        CELERY={'broker_url': broker_url,
+            'result_backend': backend_url,
+            'mongodb_backend_settings': {
+                'database': 'kindleear',
+                'taskmeta_collection': 'kindleear_taskqueue',
+            },
+            'broker_transport_options': transport_opts,
             'task_ignore_result': True,
         },)
 
@@ -28,7 +59,7 @@ def __call__(self, *args, **kwargs):
     celery_app.conf.beat_schedule = {
         'check_deliver': {
             'task': 'check_deliver',
-            'schedule': crontab(minute=0, hour='*/1'), #每个小时
+            'schedule': crontab(minute=50, hour='*/1'), #每个小时
             'args': []
         },
         'remove_logs': {

application/back_end/task_queue_rq.py

@@ -15,8 +15,9 @@
 def init_task_queue_service(app):
     app.config['RQ_REDIS_URL'] = app.config['TASK_QUEUE_BROKER_URL']
     rq.init_app(app)
+    #windows不支持，暂时屏蔽，正式版本需要取消注释
     #check_deliver.cron('0 */1 * * *', 'check_deliver') #每隔一个小时执行一次
-    #remove_logs.cron('0 0 */1 * *', 'check_deliver') #每隔24小时执行一次
+    #remove_logs.cron('0 0 */1 * *', 'remove_logs') #每隔24小时执行一次
     return rq
 
 @rq.job

application/lib/smtp_mail.py

@@ -14,7 +14,7 @@ def smtp_send_mail(sender, to, subject, body, host, username, password, port=Non
         host, port = host.split(':', 2)
         port = int(port)
     else:
-        port = 25
+        port = 587 #587-TLS, 465-SSL
 
     to = to if isinstance(to, list) else [to]
     message = MIMEMultipart('alternative') if html else MIMEMultipart()
@@ -32,7 +32,7 @@ def smtp_send_mail(sender, to, subject, body, host, username, password, port=Non
         part.add_header('Content-Disposition', f'attachment; filename="{filename}"')
         message.attach(part)
 
-    with smtplib.SMTP_SSL(host=host, port=port) as smtp_server:
+    with smtplib.SMTP(host=host, port=port) as smtp_server:
         smtp_server.connect(host, port)
         smtp_server.ehlo()
         smtp_server.starttls()

application/static/base.css

@@ -797,6 +797,9 @@ div[class="schedule_daytimes"] input {
 .icon-schedule:before {
   content: "\ea5f";
 }
+.icon-edit:before {
+  content: "\e60c";
+}
 
 /* upload cover images */
 .imgFileUploade{
@@ -869,10 +872,11 @@ div[class="schedule_daytimes"] input {
     bottom: 20px;
     left: 50%;
     transform: translateX(-50%);
-    background-color: #ffe135;
+    background-color: #fcfab2;
     color: #333;
     padding: 15px 20px;
     border-radius: 10px;
+    border: 1px solid #ada802;
     box-shadow: 0 2px 5px rgba(0, 0, 0, 0.2);
     z-index: 9999;
 }

application/templates/admin.html

@@ -26,15 +26,15 @@
       </div>
       <div class="pure-control-group">
         <label for="sm_service">{{_("Signup type")}}</label>
-        <select class="pure-u-1 pure-u-sm-1-2" name="signup_type">
+        <select class="pure-u-1 pure-u-sm-1-2" name="signup_type" id="signup_type" onchange="SetInviteCodesVisualbility()" >
           <option value="public" {% if signupType == 'public' %} selected="selected" {% endif %}>{{_("Public")}}</option>
           <option value="oneTimeCode" {% if signupType == 'oneTimeCode' %} selected="selected" {% endif %}>{{_("One time code")}}</option>
           <option value="permanentCode" {% if signupType == 'permanentCode' %} selected="selected" {% endif %}>{{_("Permanent code")}}</option>
         </select>
       </div>
-      <div class="pure-control-group">
+      <div class="pure-control-group" id="invite_codes_group" style="display:none;">
         <label>{{_("Invitation codes")}}</label>
-        <textarea class="pure-u-1 pure-u-sm-1-2" name="invite_codes" placeholder="one code per line">{{inviteCodes}}</textarea>
+        <textarea class="pure-u-1 pure-u-sm-1-2" name="invite_codes" placeholder="{{_('one code per line')}}">{{inviteCodes}}</textarea>
       </div>
     </fieldset>
   </form>
@@ -82,7 +82,7 @@
           <td>{{u.expiration_days}} Days</td>
           {% endif -%}
           <td>
-            <button class="additional-btn btn-B" title="{{_('Change password')}}"><a href="/account/change/{{u.name}}"><i class="iconfont icon-upload"></i></a></button>
+            <button class="additional-btn btn-B" title="{{_('Change password')}}"><a href="/account/change/{{u.name}}"><i class="iconfont icon-edit"></i></a></button>
             {% if u.name == adminName -%}
             <button class="additional-btn btn-A additional-btn-disabled"><i class="iconfont icon-delete"></i></button>
             {% else -%}
@@ -97,3 +97,18 @@
   {% endif -%}
 </div>
 {% endblock -%}
+{% block js -%}
+<script type="text/javascript">
+$(document).ready(function() {
+  SetInviteCodesVisualbility();
+});
+function SetInviteCodesVisualbility() {
+  var type_ = $('#signup_type').val();
+  if (type_ == 'public') {
+    $('#invite_codes_group').hide();
+  } else {
+    $('#invite_codes_group').show();
+  }
+}
+</script>
+{% endblock -%}