add gae deployment script

application/lib/readability/cleaners.py

@@ -1,6 +1,9 @@
 # strip out a set of nuisance html attributes that can mess up rendering in RSS feeds
 import re
-from lxml.html.clean import Cleaner
+try:
+    from lxml.html.clean import Cleaner
+except ImportError:
+    from lxml_html_clean import Cleaner
 
 bad_attrs = ["width", "height", "style", "[-a-z]*color", "background[-a-z]*", "on*"]
 single_quoted = "'[^']+'"

docker/Dockerfile

@@ -22,10 +22,10 @@ RUN pip install --upgrade pip
 COPY --from=req_builder /usr/site/requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
 
+COPY ./docker/gunicorn.conf.py .
 COPY ./main.py .
 COPY --from=req_builder /usr/site/config.py .
 COPY ./application/ ./application/
-COPY ./docker/gunicorn.conf.py .
 
 EXPOSE 8000
 

docker/gunicorn.conf.py

@@ -1,6 +1,6 @@
 # gunicorn.conf.py
 import os
-pythonpath = "/usr/local/lib/python3.10/site-packages"
+pythonpath = "/usr/local/lib/python3.9/site-packages"
 bind = "0.0.0.0:8000"
 workers = 1
 threads = 3
@@ -16,18 +16,18 @@
 logconfig_dict = {
     'version': 1,
     'disable_existing_loggers': False,
-    "root": {"level": "INFO", "handlers": ["error_file", "access_file"]},
+    "root": {"level": "INFO", "handlers": ["error_file"]},
     'loggers': {
         "gunicorn.error": {
             "level": "INFO", 
             "handlers": ["error_file"],
-            "propagate": 1,
+            "propagate": False,
             "qualname": "gunicorn.error"
         },
         "gunicorn.access": {
             "level": "INFO",
             "handlers": ["access_file"],
-            "propagate": 0,
+            "propagate": False,
             "qualname": "gunicorn.access"
         }
     },
@@ -37,25 +37,25 @@
             "maxBytes": 50*1024*1024, #50M
             "backupCount": 1,
             "formatter": "generic",
-            #'mode': 'w+',
             "filename": "/data/gunicorn.error.log"
         },
         "access_file": {
             "class": "logging.handlers.RotatingFileHandler",
             "maxBytes": 10*1024*1024, #10M
             "backupCount": 1,
-            "formatter": "generic",
+            "formatter": "access",
             "filename": "/data/gunicorn.access.log"
         }
     },
     'formatters':{
         "generic": {
-            "format": "'[%(asctime)s] %(levelname)s [%(filename)s:%(lineno)s] %(message)s'",
+            "format": "%(asctime)s %(levelname)s [%(filename)s:%(lineno)s] %(message)s",
             "datefmt": "[%Y-%m-%d %H:%M:%S %z]",
             "class": "logging.Formatter"
         },
         "access": {
-            "format": "'[%(asctime)s] %(levelname)s [%(filename)s:%(lineno)s] %(message)s'",
+            "format": "%(message)s",
+            "datefmt": "[%Y-%m-%d %H:%M:%S %z]",
             "class": "logging.Formatter"
         }
     }

tools/gae_deploy.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+gcloud app create   #select a region for app if it's first time
+python ./tools/update_req.py gae
+gcloud beta app deploy --version=1 app.yaml
+gcloud beta app deploy --version=1 cron.yaml
+gcloud beta app deploy --version=1 queue.yaml
+echo -e "The deployment is completed."
+echo -e "The access address is: https://$GOOGLE_CLOUD_PROJECT.appspot.com"

tools/update_req.py

@@ -2,12 +2,13 @@
 # -*- coding:utf-8 -*-
 """create/update requirments.txt of KindleEar
 """
-import re, os, sys, shutil
+import re, os, sys, shutil, subprocess
 
 REQ_COMM = [('requests', '>=2.31.0,<3.0.0'),
     ('chardet', '>=5.2.0,<6.0.0'),
     ('pillow', '>=10.2.0,<11.0.0'),
     ('lxml', '>=5.0.0,<6.0.0'),
+    ('lxml_html_clean', '>=0.1.1,<1.0.0'),
     ('sendgrid', '>=6.11.0,<7.0.0'),
     ('mailjet_rest', '>=1.3.4,<2.0.0'),
     ('python-dateutil', '>=2.8.2,<3.0.0'),
@@ -26,12 +27,12 @@
 REQ_DB = {
     'sqlite': [('peewee', '>=3.1.7,<4.0.0'),],
     'mysql': [('peewee', '>=3.1.7,<4.0.0'), ('pymysql', '>=1.1.0,<2.0.0'),],
-    'postgresql': [('peewee', '>=3.1.7,<4.0.0'), ('psycopg2', '>=2.9.9,<3.0.0'),],
-    'cockroachdb': [('peewee', '>=3.1.7,<=4.0.0'), ('psycopg2', '>=2.9.9,<3.0.0'),],
-    'datastore': [('weedata', '>=0.2.1,<1.0.0'), ('google-cloud-datastore', '>=2.19.0,<3.0.0'),],
-    'mongodb': [('weedata', '>=0.2.1,<1.0.0'), ('pymongo', '>=3.7.2,<4.0.0'),],
-    'redis': [('weedata', '>=0.2.1,<1.0.0'), ('redis', '>=4.5.0,<6.0.0'),],
-    'pickle': [('weedata', '>=0.2.1,<1.0.0'),],
+    'postgresql': [('peewee', '>=3.1.7,<4.0.0'), ('psycopg2-binary', '>=2.9.9,<3.0.0'),],
+    'cockroachdb': [('peewee', '>=3.1.7,<=4.0.0'), ('psycopg2-binary', '>=2.9.9,<3.0.0'),],
+    'datastore': [('weedata', '>=0.2.2,<1.0.0'), ('google-cloud-datastore', '>=2.19.0,<3.0.0'),],
+    'mongodb': [('weedata', '>=0.2.2,<1.0.0'), ('pymongo', '>=3.7.2,<4.0.0'),],
+    'redis': [('weedata', '>=0.2.2,<1.0.0'), ('redis', '>=4.5.0,<6.0.0'),],
+    'pickle': [('weedata', '>=0.2.2,<1.0.0'),],
 }
 
 REQ_TASK = {
@@ -42,8 +43,8 @@
 }
 
 REQ_PLAT = {'gae': [('appengine-python-standard', '>=1.1.6,<2.0.0'),],
-    'docker': [('weedata', '>=0.2.1,<1.0.0'),('pymysql', '>=1.1.0,<2.0.0'), #docker install all libs
-    ('psycopg2', '>=2.9.9,<3.0.0'),('pymongo', '>=3.7.2,<4.0.0'),('redis', '>=4.5.0,<6.0.0'),
+    'docker': [('weedata', '>=0.2.2,<1.0.0'),('pymysql', '>=1.1.0,<2.0.0'), #docker install all libs
+    ('psycopg2-binary', '>=2.9.9,<3.0.0'),('pymongo', '>=3.7.2,<4.0.0'),('redis', '>=4.5.0,<6.0.0'),
     ('celery', '>=5.3.6,<6.0.0'),('flask-rq2', '>=18.3,<19.0'),('sqlalchemy', '>=2.0.28,<3.0.0')],
 }
 
@@ -114,11 +115,24 @@ def dockerize_config_py(cfgFile):
         f.write('\n'.join(ret))
     print(f'Finished update {cfgFile}')
 
+def gae_location():
+    try:
+        output = subprocess.check_output(['gcloud', 'beta', 'app', 'describe'], universal_newlines=True)
+        lines = output.split('\n')
+        for line in lines:
+            if 'locationId:' in line:
+                loc = line[11:].strip()
+                return {'us-central': 'us-central1', 'europe-west': 'europe-west1'}.get(loc, loc)
+        return ''
+    except subprocess.CalledProcessError as e:
+        print(f"Subprocess error: {e}")
+        return ''
+
 #prepare config.py to deploy in gae
 def gaeify_config_py(cfgFile):
-    appId = os.getenv('GOOGLE_CLOUD_PROJECT', 'kindleear')
+    appId = os.getenv('GOOGLE_CLOUD_PROJECT')
     domain = f"https://{appId}.appspot.com"
-    default_cfg = {'APP_ID': appId, 'APP_DOMAIN': domain, 'SERVER_LOCATION': 'us-central1',
+    default_cfg = {'APP_ID': appId, 'APP_DOMAIN': domain, 'SERVER_LOCATION': gae_location(),
         'DATABASE_URL': 'datastore', 'TASK_QUEUE_SERVICE': 'gae', 'TASK_QUEUE_BROKER_URL': '',
         'KE_TEMP_DIR': '', 'DOWNLOAD_THREAD_NUM': '3', 'ALLOW_SIGNUP': 'no',
         'HIDE_MAIL_TO_LOCAL': 'yes', 'LOG_LEVEL': 'warning'}
@@ -160,9 +174,11 @@ def gaeify_config_py(cfgFile):
     dockerize = False
     gaeify = False
     if len(sys.argv) == 2 and sys.argv[1] == 'docker':
+        print('\nGenerating config.py and requirements.txt for Docker deployment.\n')
         dockerize_config_py(cfgFile)
         dockerize = True
     elif len(sys.argv) == 2 and sys.argv[1] == 'gae':
+        print('\nGenerating config.py and requirements.txt for GAE deployment.\n')
         gaeify_config_py(cfgFile)
         gaeify = True
     else: