Try to add cloud profiler to website (#630)

* Try to add cloud profiler to website

* comment

* build deps

* only run profiler on app engine

* Split requirements.txt

* fix import

* error

Dockerfile

@@ -12,7 +12,7 @@ RUN npm run-script build
 FROM python:3.7-slim
 COPY server /website/server
 WORKDIR /website/server
-RUN pip install -r requirements.txt
+RUN pip install -r requirements-core.txt
 ENV FLASK_ENV="test"
 RUN python -m pytest
 

cloudbuild.py.yaml

@@ -5,7 +5,8 @@ steps:
     entrypoint: /bin/sh
     args:
       - -c
-      - ./run_test.sh -p
+      - |
+        ./run_test.sh -p
 
   - id: package_js
     name: "node"

cloudbuild.screenshot.yaml

@@ -42,7 +42,6 @@ steps:
       - |
         gsutil cp server/test_screenshots/*.png gs://datcom-browser-screenshot/$SHORT_SHA/
         rm -rf server/test_screenshots
-
   # # Fetch author email from git commit history
   # - id: fetch author email
   #   name: "gcr.io/cloud-builders/git"
@@ -59,7 +58,6 @@ steps:
   #         echo $(cut -d' ' -f2 <<<`git show $SHORT_SHA | head -2 | tail -1`) > /workspace/commit.txt
   #       fi
   #       echo `git show $(cat /workspace/commit.txt) | head -n 2 | tail -n 1 | cut -d "<" -f2 | cut -d ">" -f1` > /workspace/email.txt
-
   # # Send notify email
   # - id: send notify email
   #   name: python:3.7-slim
@@ -68,7 +66,7 @@ steps:
   #     - -c
   #     - |
   #       cd build-notifier
-  #       pip3 install -r requirements.txt -q
+  #       pip3 install -r requirements-core.txt -q
   #       echo "Receiver email: $(cat /workspace/email.txt)"
   #       python3 send_email.py \
   #         --email=$(cat /workspace/email.txt) \

run_server.sh

@@ -20,6 +20,6 @@ source .env/bin/activate
 export GOOGLE_CLOUD_PROJECT=datcom-browser-staging
 export FLASK_ENV=development
 cd server
-pip3 install -r requirements.txt
+pip3 install -r requirements-core.txt
 python3 main.py
 cd ..

run_test.sh

@@ -67,7 +67,7 @@ function run_py_test {
   source .env/bin/activate
   cd server
   export FLASK_ENV=test
-  pip3 install -r requirements.txt -q
+  pip3 install -r requirements-core.txt -q
   python3 -m pytest tests/**.py
   cd ..
   echo -e "#### Checking Python style"
@@ -89,7 +89,7 @@ function run_webdriver_test {
   fi
   export FLASK_ENV=webdriver
   export GOOGLE_CLOUD_PROJECT=datcom-browser-staging
-  pip3 install -r requirements.txt -q
+  pip3 install -r requirements-core.txt -q
   python3 -m pytest webdriver_tests/*.py
   cd ..
 }
@@ -105,7 +105,7 @@ function run_screenshot_test {
   fi
   export FLASK_ENV=webdriver
   export GOOGLE_CLOUD_PROJECT=datcom-browser-staging
-  pip3 install -r requirements.txt -q
+  pip3 install -r requirements-core.txt -q
   if [  -d test_screenshots  ]
   then
     echo "Delete the test_screenshots folder"

server/__init__.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import json
+import logging
 import os
 
 from flask import Flask
@@ -23,6 +24,17 @@
 def create_app():
     app = Flask(__name__, static_folder="dist", static_url_path="")
 
+    if os.environ.get('FLASK_ENV') in ['production', 'staging']:
+        import googlecloudprofiler
+        # Profiler initialization. It starts a daemon thread which continuously
+        # collects and uploads profiles. Best done as early as possible.
+        try:
+            # service and service_version can be automatically inferred when
+            # running on App Engine.
+            googlecloudprofiler.start(verbose=3)
+        except (ValueError, NotImplementedError) as exc:
+            logging.error(exc)
+
     # Setup flask config
     if os.environ.get('FLASK_ENV') == 'test':
         cfg = import_string('configmodule.TestConfig')()

server/requirements-core.txt

@@ -0,0 +1,16 @@
+Flask==1.1.2
+Flask-Caching
+attrs==19.3.0
+google-cloud-storage
+google-cloud-secret-manager==1.0.0
+jinja2
+parameterized
+pytest
+requests
+six
+wheel
+gunicorn
+selenium
+flask_testing
+yapf
+redis==3.5.3

server/requirements-gcp.txt

@@ -0,0 +1 @@
+google-cloud-profiler

server/requirements.txt

@@ -1,16 +1,2 @@
-Flask==1.1.2
-Flask-Caching
-attrs==19.3.0
-google-cloud-storage
-google-cloud-secret-manager==1.0.0
-jinja2
-parameterized
-pytest
-requests
-six
-wheel
-gunicorn
-selenium
-flask_testing
-yapf
-redis==3.5.3
+-r requirements-core.txt
+-r requirements-gcp.txt