Use sqlalchemy and upgrade to python 3 (#130)

* Use sqlalchemy and upgrade to python 3

* Update travis python version

* pre-commit update, remove python2.7 imports, update requirements

* update to python3.8

* pytest fixture update

* update venv-update

* update appengine config

* update frontend deps

* fix proxy middleware

* rename to main.py for appengine

* remove encoding pragma

* use in memory database for testing

* update tests

* update travis file

* update sendgrid, tests for postgres

* meta_data not metadata

Co-authored-by: Nicholas Ngorok <[email protected]>
Co-authored-by: Kent Wills <[email protected]>

.pre-commit-config.yaml

@@ -1,52 +1,57 @@
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.3.0
+    rev: v3.4.0
     hooks:
     -   id: trailing-whitespace
-        exclude: assets/vendor/*
-        language_version: python2.7
+        exclude: assets/vendor/
+        language_version: python3.8
     -   id: end-of-file-fixer
-        exclude: assets/vendor/*
-        language_version: python2.7
+        exclude: assets/vendor/|api/bin/
+        language_version: python3.8
     -   id: check-json
         files: \.(jshintrc|json)$
-        language_version: python2.7
+        language_version: python3.8
     -   id: check-yaml
-        language_version: python2.7
+        language_version: python3.8
     -   id: debug-statements
-        language_version: python2.7
+        language_version: python3.8
     -   id: name-tests-test
-        language_version: python2.7
-    -   id: flake8
-        language_version: python2.7
+        language_version: python3.8
     -   id: requirements-txt-fixer
-        language_version: python2.7
+        language_version: python3.8
     -   id: fix-encoding-pragma
-        language_version: python2.7
+        args:
+          - --remove
+        language_version: python3.8
     -   id: check-byte-order-marker
-        language_version: python2.7
+        language_version: python3.8
     -   id: check-merge-conflict
-        language_version: python2.7
--   repo: https://github.com/pre-commit/mirrors-autopep8
-    rev: v1.4.4
+        language_version: python3.8
+    -   id: fix-byte-order-marker
+        language_version: python3.8
+-   repo: https://gitlab.com/pycqa/flake8
+    rev: 3.8.4
     hooks:
-    -   id: autopep8
+    -   id: flake8
+        exclude: api/bin
+        args: [--max-line-length=131]
 -   repo: https://github.com/asottile/reorder_python_imports
-    rev: v1.7.0
+    rev: v2.3.6
     hooks:
     -   id: reorder-python-imports
+        exclude: api/bin
         args:
-        - --add-import
+        - --remove-import
         - from __future__ import absolute_import
-        - --add-import
+        - --remove-import
         - from __future__ import print_function
-        - --add-import
+        - --remove-import
         - from __future__ import unicode_literals
-        language_version: python2.7
+        language_version: python3.8
 -   repo: https://github.com/Yelp/detect-secrets
-    rev: v0.12.6
+    rev: v0.14.3
     hooks:
     -   id: detect-secrets
         args: ['--baseline', '.secrets.baseline']
-        exclude: .*tests/.*|.*yelp/testing/.*|\.pre-commit-config\.yaml
-        language_version: python2.7
+        exclude: .*tests/.*|.*yelp/testing/.*|\.pre-commit-config\.yaml|package-lock\.json|.*.html
+        language_version: python3.8

.secrets.baseline

@@ -1,9 +1,10 @@
 {
+  "custom_plugin_paths": [],
   "exclude": {
     "files": ".*tests/.*|.*yelp/testing/.*|\\.pre-commit-config\\.yaml",
     "lines": null
   },
-  "generated_at": "2019-09-21T22:16:33Z",
+  "generated_at": "2020-12-28T21:59:50Z",
   "plugins_used": [
     {
       "base64_limit": 4.5,
@@ -18,5 +19,9 @@
     }
   ],
   "results": {},
-  "version": "0.12.6"
+  "version": "0.14.3",
+  "word_list": {
+    "file": null,
+    "hash": null
+  }
 }

.travis.yml

@@ -1,24 +1,19 @@
 matrix:
     include:
         - language: python
-          python: 2.7
-          env: TOXENV=py27
+          python: 3.8
+          env: TOXENV=py38
           install: pip install tox
           before_script:
             - cd api/
-            - ls tests/
-            - curl https://storage.googleapis.com/appengine-sdks/featured/google_appengine_1.9.50.zip > google_appengine.zip
-            - unzip google_appengine.zip >/dev/null
-            - export APPENGINE_PATH=./google_appengine
-            - export PATH=$PATH:$APPENGINE_PATH
           script: make test
           cache:
             directories:
                 - $HOME/.cache/pip
                 - $HOME/.cache/pre-commit
 
         - language: node_js
-          node_js: 12
+          node_js: 14
           before_script:
             - cd frontend/
           script: make test

api/.coveragerc

@@ -6,7 +6,7 @@ omit =
     /usr/*
     setup.py
 
-    webapp.py
+    main.py
     templates/*
     venv/*
     google_appengine/*

api/.gcloudignore

@@ -0,0 +1,19 @@
+# This file specifies files that are *not* uploaded to Google Cloud Platform
+# using gcloud. It follows the same syntax as .gitignore, with the addition of
+# "#!include" directives (which insert the entries of the given .gitignore-style
+# file at that point).
+#
+# For more information, run:
+#   $ gcloud topic gcloudignore
+#
+.gcloudignore
+# If you would like to upload your .git directory, .gitignore file or files
+# from your .gitignore file, remove the corresponding line
+# below:
+.git
+.gitignore
+
+# Python pycache:
+__pycache__/
+# Ignored by the build system
+/setup.cfg

api/.gitignore

@@ -6,6 +6,7 @@
 .coverage
 .idea/
 .node-version
+.pylintrc
 .tox
 /coverage
 /lib/

api/Makefile

@@ -29,13 +29,13 @@ test: development install-hooks
 
 venv: $(VIRTUALENV_REQUIREMENTS) bin/venv-update
 	./bin/venv-update \
-	    venv= venv/ --python=python2.7 \
+	    venv= venv/ --python=python3.8 \
 	    install= $(patsubst %,-r %,$(VIRTUALENV_REQUIREMENTS)) \
 	    bootstrap-deps= -r requirements-bootstrap.txt \
 
 .PHONY: serve
 serve: development
-	dev_appserver.py .
+	FLASK_APP=main.py flask run
 
 .PHONY: clean
 clean:

api/app.yaml

@@ -1,37 +1,24 @@
-runtime: python27
-threadsafe: true
+runtime: python38
+
 instance_class: F2
+
 service: api
 
 handlers:
 - url: /tasks/.*
-  script: webapp.app
+  script: auto
   auth_fail_action: unauthorized
   secure: always
 
 - url: /preferences/.*
-  script: webapp.app
+  script: auto
   secure: always
 
 - url: /rule
-  script: webapp.app
-  login: admin
+  script: auto
   auth_fail_action: unauthorized
   secure: always
 
 - url: /.* # Anything not explicitly listed above
-  script: webapp.app
+  script: auto
   secure: always
-
-libraries:
-- name: jinja2
-  version: latest
-- name: ssl
-  version: latest
-
-skip_files:
-- ^(.*/)?#.*#$
-- ^(.*/)?.*~$
-- ^(.*/)?.*\.py[co]$
-- ^(.*/)?.*/RCS/.*$
-- ^(.*/)?\..*$

api/appengine_config.py

@@ -1,18 +1,13 @@
-# -*- coding: utf-8 -*-
 """
 `appengine_config.py` is automatically loaded when Google App Engine
 starts a new instance of your application. This runs before any
 WSGI applications specified in app.yaml are loaded.
 """
-from __future__ import absolute_import
-from __future__ import print_function
-from __future__ import unicode_literals
-
 import os.path
 
 from google.appengine.ext import vendor
 
 # Third-party libraries are stored in "site-packages" in the virtualenv,
 # vendoring will makes sure that they are importable by the application.
 virtualenv = os.environ.get('VIRTUAL_ENV', 'venv')
-vendor.add(os.path.join(virtualenv, 'lib', 'python2.7', 'site-packages'))
+vendor.add(os.path.join(virtualenv, 'lib', 'python3.8', 'site-packages'))

api/bin/venv-update

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 '''\
 usage: venv-update [-hV] [options]
 
@@ -463,3 +462,4 @@ def main():
 
 if __name__ == '__main__':
     exit(main())
+

api/database.py

@@ -0,0 +1,3 @@
+from flask_sqlalchemy import SQLAlchemy
+
+db = SQLAlchemy()

api/factory.py

@@ -0,0 +1,30 @@
+from database import db
+from flask import Flask
+from yelp_beans.logic.config import get_config
+
+
+def create_app():
+    app = Flask(__name__, template_folder='yelp_beans/templates')
+    app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
+    app.config['SQLALCHEMY_DATABASE_URI'] = get_config().get('DATABASE_URL_PROD', "sqlite://")
+    db.init_app(app)
+
+    from yelp_beans.routes.api.v1.meeting_requests import meeting_requests
+    from yelp_beans.routes.api.v1.metrics import metrics_blueprint
+    from yelp_beans.routes.api.v1.preferences import preferences_blueprint
+    from yelp_beans.routes.api.v1.user import user_blueprint
+    from yelp_beans.routes.tasks import tasks
+    # Cron Endpoint
+    app.register_blueprint(tasks, url_prefix='/tasks')
+
+    # Api Endpoints
+    app.register_blueprint(meeting_requests, url_prefix='/v1/meeting_request')
+    app.register_blueprint(metrics_blueprint, url_prefix='/v1/metrics')
+    app.register_blueprint(preferences_blueprint,
+                           url_prefix='/v1/user/preferences')
+    app.register_blueprint(user_blueprint, url_prefix='/v1/user')
+
+    with app.app_context():
+        db.create_all()
+
+    return app

api/main.py

@@ -0,0 +1,15 @@
+import os
+
+from factory import create_app
+
+
+app = create_app()
+app.app_context().push()
+
+if 'Development' in os.environ.get('SERVER_SOFTWARE', ''):
+    from tests.conftest import create_dev_data
+    from database import db
+    create_dev_data(db.session)
+
+if __name__ == "__main__":
+    app.run()

api/requirements-bootstrap.txt

@@ -1,5 +1,4 @@
-pip==9.0.1
-pip-custom-platform==0.3.1
-pymonkey==0.2.2
-venv-update==2.0.0
-wheel==0.29.0
+pip==18.1
+pymonkey==0.3.1
+venv-update==3.2.4
+wheel==0.33.6

api/requirements-dev.txt

@@ -1,33 +1,21 @@
-aspy.yaml==1.3.0
-atomicwrites==1.3.0
-attrs==19.1.0
-cfgv==2.0.1
-configparser==4.0.2
-contextlib2==0.6.0
-coverage==4.5.4
+appdirs==1.4.4
+attrs==20.3.0
+cfgv==3.2.0
+coverage==5.3.1
+distlib==0.3.1
 filelock==3.0.12
-funcsigs==1.0.2
-futures==3.3.0
-identify==1.4.7
-importlib-metadata==0.23
-importlib-resources==1.0.2
-mock==3.0.5
-more-itertools==5.0.0
-nodeenv==1.3.3
-packaging==19.2
-pathlib2==2.3.4
-pluggy==0.13.0
-pre-commit==1.18.3
-py==1.8.0
-pyparsing==2.4.2
-pytest==4.6.5
-requests-mock==1.3.0
-requirements-tools==1.2.1
-scandir==1.10.0
-six==1.12.0
-toml==0.10.0
-tox==3.14.0
-typing==3.7.4.1
-virtualenv==16.7.5
-wcwidth==0.1.7
-zipp==0.6.0
+identify==1.5.10
+iniconfig==1.1.1
+mock==4.0.3
+nodeenv==1.5.0
+packaging==20.8
+pluggy==0.13.1
+pre-commit==2.9.3
+py==1.10.0
+pyparsing==2.4.7
+pytest==6.2.1
+requests-mock==1.8.0
+requirements-tools==1.2.4
+toml==0.10.2
+tox==3.20.1
+virtualenv==20.2.2

api/requirements-minimal.txt

@@ -1,8 +1,10 @@
-boto
+boto3
 Flask
 flask-api-utils
+Flask-SQLAlchemy
 httplib2
 networkx
+psycopg2-binary
 pytz
 pyyaml
 requests