v3.0.0beta

Ready for deployment

.gitignore

@@ -12,3 +12,4 @@ tests/rss/*
 tests/debug_mail/*
 tests/cov_html/*
 .idea/
+datastore/

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 cdhigh
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

app.yaml

@@ -1,4 +1,5 @@
 runtime: python38
+service: default
 
 #B1: 384MB/600MHz/manual_scaling
 instance_class: B1
@@ -23,21 +24,19 @@ inbound_services:
 
 handlers:
 - url: /robots.txt
-  static_files: static/robots.txt
-  upload: static/robots.txt
+  static_files: application/static/robots.txt
+  upload: application/static/robots.txt
 
 - url: /favicon.ico
-  static_files: static/favicon.ico
-  upload: static/favicon.ico
+  static_files: application/static/favicon.ico
+  upload: application/static/favicon.ico
   mime_type: image/x-icon
 
 - url: /static
-  static_dir: static
-  application_readable: true
+  static_dir: application/static
 
 - url: /images
-  static_dir: images
-  application_readable: true
+  static_dir: application/images
 
 - url: /.*
   secure: always

application/__init__.py

@@ -4,7 +4,7 @@
 # Author: cdhigh <https://github.com/cdhigh>
 __Author__ = "cdhigh"
 
-__Version__ = '3.0.0'
+__Version__ = '3.0.0b'
 
 import os, builtins, datetime
 from flask import Flask, render_template, session, request, g
@@ -22,12 +22,8 @@ def init_app(name, debug=False):
 
     app = Flask(name, template_folder=template_folder, static_folder=static_folder)
     app.config.from_pyfile(os.path.join(rootDir, 'config.py'))
-
-    from .utils import new_secret_key
-
-    app.config['SECRET_KEY'] = '12345678' # if debug else new_secret_key()
     app.config['MAX_CONTENT_LENGTH'] = 32 * 1024 * 1024 #32MB
-
+    
     from .view import setting
     app.config["BABEL_TRANSLATION_DIRECTORIES"] = i18n_folder
     babel = Babel(app)

application/back_end/db_models.py

@@ -17,11 +17,12 @@ class KeUser(MyBaseModel): # kindleEar User
     name = CharField(unique=True)
     passwd = CharField()
     email = CharField()
+    sender = CharField() #可能等于自己的email，也可能是管理员的email
     secret_key = CharField(default='')
     kindle_email = CharField(default='')
     enable_send = BooleanField(default=False)
     send_days = JSONField(default=JSONField.list_default)
-    send_time = IntegerField(default=0)
+    send_time = IntegerField(default=6)
     timezone = IntegerField(default=0)
     expiration_days = IntegerField(default=0) #账号超期设置值，0为永久有效
     expires = DateTimeField(null=True) #超过了此日期后账号自动停止推送
@@ -114,7 +115,7 @@ class UserBlob(MyBaseModel):
     name = CharField()
     user = CharField()
     time = DateTimeField(default=datetime.datetime.utcnow)
-    data = BlobField(null=True)
+    data = BlobField(null=True, index=False)
 
 #RSS订阅源，包括自定义RSS，上传的recipe，内置在zip里面的builtin_recipe不包括在内
 #每个Recipe的字符串表示为：custom:id, upload:id
@@ -125,7 +126,7 @@ class Recipe(MyBaseModel):
     isfulltext = BooleanField(default=False) #只有自定义RSS才有意义
     type_ = CharField() #'custom','upload'
     needs_subscription = BooleanField(default=False) #是否需要登陆网页，只有上传的recipe才有意义
-    src = TextField(default='') #保存上传的recipe的unicode字符串表示，已经解码
+    src = TextField(default='', index=False) #保存上传的recipe的unicode字符串表示，已经解码
     time = DateTimeField() #源被加入的时间，用于排序
     user = CharField() #哪个账号创建的，和nosql一致，保存用户名
     language = CharField(default='')
@@ -176,7 +177,7 @@ class DeliverLog(MyBaseModel):
     to = CharField()
     size = IntegerField(default=0)
     time_str = CharField() #每个用户的时区可能不同，为显示方便，创建记录时就生成推送时间字符串
-    datetime = DateTimeField(index=True)
+    datetime = DateTimeField()
     book = CharField(default='')
     status = CharField()
 
@@ -193,7 +194,7 @@ class SharedRss(MyBaseModel):
     language = CharField(default='')
     category = CharField(default='')
     recipe_url = CharField(default='') #客户端优先使用此字段获取recipe，为什么不用上面的url是要和以前的版本兼容
-    src = TextField(default='') #保存分享的recipe的unicode字符串表示，已经解码
+    src = TextField(default='', index=False) #保存分享的recipe的unicode字符串表示，已经解码
     description = CharField(default='')
     creator = CharField(default='') #保存贡献者的md5
     created_time = DateTimeField(default=datetime.datetime.utcnow)
@@ -246,4 +247,4 @@ def create_database_tables():
         SharedRss, SharedRssCategory, LastDelivered, AppInfo], safe=True)
     #close_database()
 
-    #print(f'Create database "{dbName}" finished')
+    return 'Created database tables successfully'

application/back_end/db_models_nosql.py

@@ -7,7 +7,7 @@
 from weedata import *
 
 dbUrl = os.getenv('DATABASE_URL')
-appId = os.getenv('APP_ID', 'kindleear')
+appId = os.getenv('APP_ID')
 
 if dbUrl.startswith('mongodb://'):
     dbInstance = MongoDbClient(appId, dbUrl)

application/back_end/send_mail_adpt.py

@@ -5,16 +5,21 @@
 #gae mail api
 #https://cloud.google.com/appengine/docs/standard/python3/reference/services/bundled/google/appengine/api/mail
 #https://cloud.google.com/appengine/docs/standard/python3/services/mail
-import os, datetime, zipfile
+import os, datetime, zipfile, base64
 from ..utils import local_time, ke_decrypt
 from ..base_handler import save_delivery_log
 from .db_models import KeUser
 
-try:
-    from google.appengine.api import mail as gae_mail
-    from google.appengine.api.mail_errors import InvalidSenderError, InvalidAttachmentTypeError, InvalidEmailError
-    from google.appengine.runtime.apiproxy_errors import OverQuotaError, DeadlineExceededError
-except ImportError:
+#google.appengine will apply patch for os.env module
+hideMailLocal = os.getenv('HIDE_MAIL_TO_LOCAL')
+
+#判断是否是部署在gae平台
+if os.getenv('DATABASE_URL') == 'datastore':
+    try:
+        from google.appengine.api import mail as gae_mail
+    except ImportError:
+        gae_mail = None
+else:
     gae_mail = None
 
 try:
@@ -23,6 +28,11 @@
 except ImportError:
     SendGridAPIClient = None
 
+try:
+    from mailjet_rest import Client as MailjetClient
+except ImportError:
+    MailjetClient = None
+
 try:
     from smtp_mail import smtp_send_mail
 except ImportError:
@@ -35,9 +45,11 @@ def avaliable_sm_services():
         sm.append('gae')
     if SendGridAPIClient:
         sm.append('sendgrid')
+    if MailjetClient:
+        sm.append('mailjet')
     if smtp_send_mail:
         sm.append('smtp')
-    if not os.getenv('HIDE_MAIL_TO_LOCAL'):
+    if not hideMailLocal:
         sm.append('local')
     return sm
 
@@ -53,27 +65,28 @@ def send_to_kindle(user, title, attachment, fileWithTime=True):
         lcTime = "({})".format(lcTime) if fileWithTime else ""
         fileName = f"{title}{lcTime}.{user.book_type}"
         attachment = (fileName, attachment)
-
-    if not isinstance(attachment, list):
-        attachments = [attachment]
 
+    if not isinstance(attachment, list):
+        attachment = [attachment]
+
     status = 'ok'
     body = "Deliver from KindleEar"
     try:
-        send_mail(user, user.kindle_email, subject, body, attachments)
+        send_mail(user, user.kindle_email, subject, body, attachment)
     except Exception as e:
         status = str(e)
         default_log.warning(f'Failed to send mail "{title}": {status}')
 
-    save_delivery_log(user, title, len(attachment), status=status)
+    size = sum([len(a[1]) for a in attachment])
+    save_delivery_log(user, title, size, status=status)
 
 #统一的发送邮件函数
 def send_mail(user, to, subject, body, attachments=None, html=None):
-    if not isinstance(to, list) and (',' in to):
+    if not isinstance(to, list):
         to = to.split(',')
     sm_service = user.get_send_mail_service()
-    srv_type = sm_service.get('service', 'gae')
-    data = {'sender': os.getenv('SRC_EMAIL'), 'to': to, 'subject': subject, 'body': body}
+    srv_type = sm_service.get('service', '')
+    data = {'sender': user.sender, 'to': to, 'subject': subject, 'body': body}
     if attachments:
         data['attachments'] = attachments
     if html:
@@ -85,10 +98,14 @@ def send_mail(user, to, subject, body, attachments=None, html=None):
     elif srv_type == 'sendgrid':
         apikey = sm_service.get('apikey', '')
         grid_send_mail(apikey=apikey, **data)
+    elif srv_type == 'mailjet':
+        apikey = sm_service.get('apikey', '')
+        secret_key = sm_service.get('secret_key', '')
+        mailjet_send_mail(apikey=apikey, secret_key=secret_key, **data)
     elif srv_type == 'smtp':
         data['host'] = sm_service.get('host', '')
         data['port'] = sm_service.get('port', 587)
-        data['username'] = sm_service.get('username', '')
+        data['username'] = user.sender
         data['password'] = ke_decrypt(sm_service.get('password', ''), user.secret_key)
         smtp_send_mail(**data)
     elif srv_type == 'local':
@@ -98,7 +115,7 @@ def send_mail(user, to, subject, body, attachments=None, html=None):
 
 #发送一个HTML邮件
 #user: KeUser实例
-#to: 收件地址，可以是一个单独的字符串，或一个字符串列表，对应发送到多个地址
+#to: 收件地址列表
 #subject: 邮件标题
 #html: 邮件正文的HTML内容
 #attachments: 附件文件名和二进制内容，[(fileName, content),...]
@@ -120,12 +137,11 @@ def send_html_mail(user, to, subject, html, attachments=None, body=None):
 
 #SendGrid发送邮件
 #sender:: 发送者地址
-#to: 收件地址，可以是一个单独的字符串，或一个字符串列表，对应发送到多个地址
+#to: 收件地址列表
 #subject: 邮件标题
 #body: 邮件正文纯文本内容
 #html: 邮件正文HTML
 #attachment: [(fileName, attachment),]
-#tz: 时区
 def grid_send_mail(apikey, sender, to, subject, body, html=None, attachments=None):
     global default_log
     sgClient = SendGridAPIClient(apikey)
@@ -146,6 +162,43 @@ def grid_send_mail(apikey, sender, to, subject, body, html=None, attachments=Non
     response = sgClient.send(message)
     if response.status_code not in (200, 202):
         raise Exception(f'sendgrid failed: {response.status_code}')
+
+#Mailjet发送邮件
+#sender:: 发送者地址
+#to: 收件地址列表
+#subject: 邮件标题
+#body: 邮件正文纯文本内容
+#html: 邮件正文HTML
+#attachment: [(fileName, attachment),]
+def mailjet_send_mail(apikey, secret_key, sender, to, subject, body, html=None, attachments=None):
+    global default_log
+    mjClient = MailjetClient(auth=(apikey, secret_key), version='v3.1')
+    to = [{'Email': t, 'Name': t} for t in to]
+    data = {'Messages': [{
+          "From": {"Email": sender, "Name": sender},
+          "To": to,
+          "Subject": subject,
+          "TextPart": body,
+        }],}
+
+    dataDict = data['Messages'][0]
+    if html:
+        dataDict['HTMLPart'] = html
+    if attachments:
+        dataDict['Attachments'] = []
+        for fileName, content in (attachments or []):
+            dataDict['Attachments'].append({"ContentType": "text/plain", "Filename": fileName,
+                "Base64Content": base64.b64encode(content).decode()})
+
+
+    resp = mjClient.send.create(data=data)
+    if resp.status_code in (200, 202):
+        status = resp.json()["Messages"][0]["Status"]
+        print(resp.json()) #TODO
+        if status != "success":
+            raise Exception(f'mailjet failed: {status}')
+    else:
+        raise Exception(f'mailjet failed: {resp.status_code}')
 
 def save_mail_to_local(dest_dir, subject, body, attachments=None, html=None, **kwargs):
     attachments = attachments or []

application/back_end/task_queue_apscheduler.py

@@ -23,7 +23,7 @@
     from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore
     jobstores = {"default": SQLAlchemyJobStore(url=_broker_url)}
 elif _broker_url == '':
-    jobstores = None #default is memory store
+    jobstores = {} #default is memory store
 else:
     raise ValueError('Unsupported TASK_QUEUE_BROKER_URL type: {_broker_url}')
 

application/back_end/task_queue_gae.py

@@ -1,10 +1,15 @@
 #!/usr/bin/env python3
 # -*- coding:utf-8 -*-
-#任务队列GAE
+#任务队列GAE接口实现
 #Author: cdhigh <https://github.com/cdhigh>
-import json
+import os, json
+from urllib.parse import urlencode, urlparse
+appId = os.getenv('APP_ID')
+serverLoc = os.getenv('SERVER_LOCATION')
+
 from google.cloud import tasks_v2
 DEFAULT_QUEUE_NAME = "default"
+TASK_HTTP_METHOD = tasks_v2.HttpMethod.GET
 
 def init_task_queue_service(app):
     pass
@@ -19,22 +24,18 @@ def create_url2book_task(payload: dict):
 #创建一个任务
 #url: 任务要调用的链接
 #payload: 要传递给url的参数，为一个Python字典
+#之所以使用HTTP.GET但是这里传入的是payload，而不是在调用方将payload合入url是为了兼容各种任务队列实现
 #返回创建的任务实例
 def create_http_task(url, payload):
     client = tasks_v2.CloudTasksClient()
+    taskParent = client.queue_path(appId, serverLoc, DEFAULT_QUEUE_NAME)
 
-    task = {"app_engine_http_request": {
-        "http_method": tasks_v2.HttpMethod.GET,
-        "relative_uri": url,}
-        }
-    if payload:
-        task["app_engine_http_request"]["headers"] = {"Content-type": "application/json"}
-        task["app_engine_http_request"]["body"] = json.dumps(payload).encode()
-    return client.create_task(task=task)
+    task = {"app_engine_http_request": {"http_method": TASK_HTTP_METHOD}}
 
-    #httpRequest = tasks_v2.HttpRequest(http_method=tasks_v2.HttpMethod.GET, url=url,
-    #        headers={"Content-type": "application/json"}, body=json.dumps(payload).encode(),)
-    #task = tasks_v2.Task(httpRequest=httpRequest)
-    #taskParent = client.queue_path(APP_ID, SERVER_LOCATION, DEFAULT_QUEUE_NAME)
-    #return client.create_task(tasks_v2.CreateTaskRequest(parent=taskParent, task=task))
-
+    if TASK_HTTP_METHOD == tasks_v2.HttpMethod.GET: #转换字典为查询字符串
+        params = {'relative_uri': urlparse(url)._replace(query=urlencode(payload, doseq=True)).geturl()}
+    else: #转换字典为post的body内容
+        params = {'relative_uri': url, 'headers': {"Content-type": "application/json"}, 'body': json.dumps(payload).encode()}
+
+    task["app_engine_http_request"].update(params)
+    return client.create_task(request={'parent': taskParent, 'task': task})