From 9457c1a1c0e3d5c2c228a9e187e2f3a7e3a6a478 Mon Sep 17 00:00:00 2001 From: Bo Xu Date: Tue, 20 Oct 2020 15:46:43 -0700 Subject: [PATCH] Try to add cloud profiler to website (#630) * Try to add cloud profiler to website * comment * build deps * only run profiler on app engine * Split requirements.txt * fix import * error --- Dockerfile | 2 +- cloudbuild.py.yaml | 3 ++- cloudbuild.screenshot.yaml | 4 +--- run_server.sh | 2 +- run_test.sh | 6 +++--- server/__init__.py | 12 ++++++++++++ server/requirements-core.txt | 16 ++++++++++++++++ server/requirements-gcp.txt | 1 + server/requirements.txt | 18 ++---------------- 9 files changed, 39 insertions(+), 25 deletions(-) create mode 100644 server/requirements-core.txt create mode 100644 server/requirements-gcp.txt diff --git a/Dockerfile b/Dockerfile index 475a37044..534117e1b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ RUN npm run-script build FROM python:3.7-slim COPY server /website/server WORKDIR /website/server -RUN pip install -r requirements.txt +RUN pip install -r requirements-core.txt ENV FLASK_ENV="test" RUN python -m pytest diff --git a/cloudbuild.py.yaml b/cloudbuild.py.yaml index dd2154883..c290d5582 100644 --- a/cloudbuild.py.yaml +++ b/cloudbuild.py.yaml @@ -5,7 +5,8 @@ steps: entrypoint: /bin/sh args: - -c - - ./run_test.sh -p + - | + ./run_test.sh -p - id: package_js name: "node" diff --git a/cloudbuild.screenshot.yaml b/cloudbuild.screenshot.yaml index c9f0eb5e7..2e5286e30 100644 --- a/cloudbuild.screenshot.yaml +++ b/cloudbuild.screenshot.yaml @@ -42,7 +42,6 @@ steps: - | gsutil cp server/test_screenshots/*.png gs://datcom-browser-screenshot/$SHORT_SHA/ rm -rf server/test_screenshots - # # Fetch author email from git commit history # - id: fetch author email # name: "gcr.io/cloud-builders/git" @@ -59,7 +58,6 @@ steps: # echo $(cut -d' ' -f2 <<<`git show $SHORT_SHA | head -2 | tail -1`) > /workspace/commit.txt # fi # echo `git show $(cat /workspace/commit.txt) | head -n 2 | tail -n 1 | cut -d "<" -f2 | cut -d ">" -f1` > /workspace/email.txt - # # Send notify email # - id: send notify email # name: python:3.7-slim @@ -68,7 +66,7 @@ steps: # - -c # - | # cd build-notifier - # pip3 install -r requirements.txt -q + # pip3 install -r requirements-core.txt -q # echo "Receiver email: $(cat /workspace/email.txt)" # python3 send_email.py \ # --email=$(cat /workspace/email.txt) \ diff --git a/run_server.sh b/run_server.sh index 7bcb66cdf..bde5d34eb 100755 --- a/run_server.sh +++ b/run_server.sh @@ -20,6 +20,6 @@ source .env/bin/activate export GOOGLE_CLOUD_PROJECT=datcom-browser-staging export FLASK_ENV=development cd server -pip3 install -r requirements.txt +pip3 install -r requirements-core.txt python3 main.py cd .. diff --git a/run_test.sh b/run_test.sh index 498c549b7..d1cdba959 100755 --- a/run_test.sh +++ b/run_test.sh @@ -67,7 +67,7 @@ function run_py_test { source .env/bin/activate cd server export FLASK_ENV=test - pip3 install -r requirements.txt -q + pip3 install -r requirements-core.txt -q python3 -m pytest tests/**.py cd .. echo -e "#### Checking Python style" @@ -89,7 +89,7 @@ function run_webdriver_test { fi export FLASK_ENV=webdriver export GOOGLE_CLOUD_PROJECT=datcom-browser-staging - pip3 install -r requirements.txt -q + pip3 install -r requirements-core.txt -q python3 -m pytest webdriver_tests/*.py cd .. } @@ -105,7 +105,7 @@ function run_screenshot_test { fi export FLASK_ENV=webdriver export GOOGLE_CLOUD_PROJECT=datcom-browser-staging - pip3 install -r requirements.txt -q + pip3 install -r requirements-core.txt -q if [ -d test_screenshots ] then echo "Delete the test_screenshots folder" diff --git a/server/__init__.py b/server/__init__.py index ca2a0216b..63a997871 100644 --- a/server/__init__.py +++ b/server/__init__.py @@ -13,6 +13,7 @@ # limitations under the License. import json +import logging import os from flask import Flask @@ -23,6 +24,17 @@ def create_app(): app = Flask(__name__, static_folder="dist", static_url_path="") + if os.environ.get('FLASK_ENV') in ['production', 'staging']: + import googlecloudprofiler + # Profiler initialization. It starts a daemon thread which continuously + # collects and uploads profiles. Best done as early as possible. + try: + # service and service_version can be automatically inferred when + # running on App Engine. + googlecloudprofiler.start(verbose=3) + except (ValueError, NotImplementedError) as exc: + logging.error(exc) + # Setup flask config if os.environ.get('FLASK_ENV') == 'test': cfg = import_string('configmodule.TestConfig')() diff --git a/server/requirements-core.txt b/server/requirements-core.txt new file mode 100644 index 000000000..d8108570a --- /dev/null +++ b/server/requirements-core.txt @@ -0,0 +1,16 @@ +Flask==1.1.2 +Flask-Caching +attrs==19.3.0 +google-cloud-storage +google-cloud-secret-manager==1.0.0 +jinja2 +parameterized +pytest +requests +six +wheel +gunicorn +selenium +flask_testing +yapf +redis==3.5.3 \ No newline at end of file diff --git a/server/requirements-gcp.txt b/server/requirements-gcp.txt new file mode 100644 index 000000000..ffbe653f1 --- /dev/null +++ b/server/requirements-gcp.txt @@ -0,0 +1 @@ +google-cloud-profiler \ No newline at end of file diff --git a/server/requirements.txt b/server/requirements.txt index d8108570a..74506ec67 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -1,16 +1,2 @@ -Flask==1.1.2 -Flask-Caching -attrs==19.3.0 -google-cloud-storage -google-cloud-secret-manager==1.0.0 -jinja2 -parameterized -pytest -requests -six -wheel -gunicorn -selenium -flask_testing -yapf -redis==3.5.3 \ No newline at end of file +-r requirements-core.txt +-r requirements-gcp.txt \ No newline at end of file