diff --git a/.gitignore b/.gitignore index 4057f250..8122e558 100644 --- a/.gitignore +++ b/.gitignore @@ -178,7 +178,8 @@ classified_structural_variants /src/frontend_celery/webapp/static/packages/ /logs /classified_variants_* -/src/frontend_celery/tests/keycloak* +/src/frontend_celery/playwright/keycloak* +/src/frontend_celery/playwright/screenshots /Python-* /data_makefile /.localpython \ No newline at end of file diff --git a/src/frontend_celery/playwright/conftest.py b/src/frontend_celery/playwright/conftest.py new file mode 100644 index 00000000..102907c6 --- /dev/null +++ b/src/frontend_celery/playwright/conftest.py @@ -0,0 +1,46 @@ +import sys +from os import path +sys.path.append(path.dirname(path.dirname(path.abspath(__file__)))) +from webapp import create_app +import pytest +import os + + +@pytest.fixture +def app(): + app = create_app() + yield app + +@pytest.fixture +def test_client(app): + with app.test_client() as client: + yield client + +@pytest.fixture +def config(app): + return app.config + +@pytest.fixture() +def client(): + return app.test_client() + +@pytest.fixture(autouse=True) +def _push_request_context(request, app): + ctx = app.test_request_context() # create context + ctx.push() + + def teardown(): + ctx.pop() + + request.addfinalizer(teardown) + + +#@pytest.fixture() +#def page2(playwright): +# browser = playwright.firefox.launch(args=['--no-proxy-server']) +# context = browser.new_context( +# #user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.0", +# color_scheme=r"light" +# ) +# page = context.new_page() +# return page \ No newline at end of file diff --git a/src/frontend_celery/tests/script/prepare_keycloak.sh b/src/frontend_celery/playwright/prepare_keycloak.sh similarity index 79% rename from src/frontend_celery/tests/script/prepare_keycloak.sh rename to src/frontend_celery/playwright/prepare_keycloak.sh index bcca78af..5904aa6b 100755 --- a/src/frontend_celery/tests/script/prepare_keycloak.sh +++ b/src/frontend_celery/playwright/prepare_keycloak.sh @@ -14,5 +14,5 @@ unzip keycloak-18.0.0.zip rm keycloak-18.0.0.zip # initialize keycloak -keycloak-18.0.0/bin/kc.sh import --file ../../../resources/backups/keycloak_export/Heredivar-realm-test.json +keycloak-18.0.0/bin/kc.sh import --file ../../resources/backups/keycloak_export/Heredivar-realm-test.json #/mnt/storage2/users/ahdoebm1/HerediVar/resources/backups/keycloak_export/Heredivar-realm-test.json \ No newline at end of file diff --git a/src/frontend_celery/playwright/pytest.ini b/src/frontend_celery/playwright/pytest.ini new file mode 100644 index 00000000..ee279b8b --- /dev/null +++ b/src/frontend_celery/playwright/pytest.ini @@ -0,0 +1,5 @@ +[pytest] +# Run firefox with UI +#addopts = --headed --browser firefox +#addopts = --base-url http://srv018.img.med.uni-tuebingen.de:4999 +live_server_scope = function \ No newline at end of file diff --git a/src/frontend_celery/playwright/start_heredivar_for_tests.sh b/src/frontend_celery/playwright/start_heredivar_for_tests.sh new file mode 100755 index 00000000..ba029f54 --- /dev/null +++ b/src/frontend_celery/playwright/start_heredivar_for_tests.sh @@ -0,0 +1,40 @@ + +#!/bin/bash + +SCRIPT=$(readlink -f "$0") +SCRIPTPATH=$(dirname "$SCRIPT") +cd $SCRIPTPATH +pwd +ROOT=$(dirname $(dirname $(dirname "$SCRIPTPATH"))) + +function waitForServer { + dist=$1 + echo -n "Starting $dist" + # Give the server some time to start up. Look for a well-known + # bit of text in the log file. Try at most 50 times before giving up. + C=50 + while : + do + grep "* Debugger PIN: " heredivar.log + if [ $? -eq 0 ]; then + echo " server started." + break + elif [ $C -gt 0 ]; then + echo -n "." + C=$((C-1)) + sleep 1 + else + echo " timeout!" + cat heredivar.log + exit 1 + fi + done +} + +set -o allexport +extension=env_ +source $ROOT/.$extension$WEBAPP_ENV +set +o allexport + +python3 $ROOT/src/frontend_celery/main.py > heredivar.log 2>&1 & +waitForServer "HerediVar" \ No newline at end of file diff --git a/src/frontend_celery/playwright/start_keycloak_for_tests.sh b/src/frontend_celery/playwright/start_keycloak_for_tests.sh new file mode 100755 index 00000000..0c11aff3 --- /dev/null +++ b/src/frontend_celery/playwright/start_keycloak_for_tests.sh @@ -0,0 +1,40 @@ + +#!/bin/bash + +SCRIPT=$(readlink -f "$0") +SCRIPTPATH=$(dirname "$SCRIPT") +cd $SCRIPTPATH +pwd +ROOT=$(dirname $(dirname $(dirname "$SCRIPTPATH"))) + +function waitForServer { + dist=$1 + echo -n "Starting $dist" + # Give the server some time to start up. Look for a well-known + # bit of text in the log file. Try at most 50 times before giving up. + C=50 + while : + do + grep "$dist .* (powered by Quarkus .*) started" keycloak.log + if [ $? -eq 0 ]; then + echo " server started." + break + elif [ $C -gt 0 ]; then + echo -n "." + C=$((C-1)) + sleep 1 + else + echo " timeout!" + cat keycloak.log + exit 1 + fi + done +} + +set -o allexport +extension=env_ +source $ROOT/.$extension$WEBAPP_ENV +set +o allexport + +$SCRIPTPATH/keycloak-18.0.0/bin/kc.sh start-dev --hostname $KEYCLOAK_HOST --http-port $KEYCLOAK_PORT > keycloak.log 2>&1 & # --log-level debug +waitForServer "Keycloak" \ No newline at end of file diff --git a/src/frontend_celery/playwright/start_tests.sh b/src/frontend_celery/playwright/start_tests.sh new file mode 100755 index 00000000..0a21f29c --- /dev/null +++ b/src/frontend_celery/playwright/start_tests.sh @@ -0,0 +1,56 @@ +#!/bin/bash +#set -e +#set -o pipefail +#set -o verbose + +export WEBAPP_ENV=localtest + +SCRIPT=$(readlink -f "$0") +ROOT=$(dirname $(dirname $(dirname $(dirname "$SCRIPT")))) +cd $ROOT +pwd + +TESTDIR=$ROOT/src/frontend_celery/playwright +TEST_DATA_DIR=$TESTDIR/data +DB_STRUCTURE=$TEST_DATA_DIR/db_structure/structure.sql.gz +DB_STATIC_SEED=$TEST_DATA_DIR/db_seeds/static.sql + + +set -o allexport +extension=env_ +source $ROOT/.$extension$WEBAPP_ENV +set +o allexport + +source $ROOT/.venv/bin/activate + + + +# init structure of the database +zcat $DB_STRUCTURE | mysql -h $DB_HOST -P $DB_PORT -u$DB_ADMIN -p$DB_ADMIN_PW $DB_NAME + +# init static information +mysql_errors=$(mysql -h $DB_HOST -P $DB_PORT -u$DB_ADMIN -p$DB_ADMIN_PW $DB_NAME < $DB_STATIC_SEED 2>&1) +if [ $? = "1" ]; then + echo $mysql_errors + exit 1 +fi + +#mysql --host sql.img.med.uni-tuebingen.de -uahdoebm1 -p20220303 HerediVar_ahdoebm1_test < $DATABASE_DUMPER_DIR/users/user_privileges_test.sql +#mysql --host sql.img.med.uni-tuebingen.de -uahdoebm1 -p20220303 HerediVar_ahdoebm1_test -e "SHOW TABLES" + + +# start keycloak +$TESTDIR/start_keycloak_for_tests.sh + + +# start heredivar +$TESTDIR/start_heredivar_for_tests.sh + + +# run tests +cd $TESTDIR +pytest #-k 'test_dev' + +# stop keycloak +pkill -s 0 -e java +pkill -s 0 -e python3 diff --git a/src/frontend_celery/playwright/tests/test_scenarios.py b/src/frontend_celery/playwright/tests/test_scenarios.py new file mode 100644 index 00000000..e09b355b --- /dev/null +++ b/src/frontend_celery/playwright/tests/test_scenarios.py @@ -0,0 +1,36 @@ +from os import path +import sys +sys.path.append(path.dirname(path.dirname(path.abspath(__file__)))) +import re +#from playwright.sync_api import Page, expect, sync_playwright +from flask import url_for + + +def test_screenshot(page): + response = page.goto(url_for("main.index", _external=True )) # "http://localhost:4000" + + print(response.request.all_headers()) + page.screenshot(path="screenshots/index.png") + +def test_login(page): + response = page.goto(url_for("auth.login", _external=True)) + page.screenshot(path="screenshots/login.png") + assert response.status == 200 + + page.locator("#username").fill("superuser") + page.locator("#password").fill("12345") + page.screenshot(path="screenshots/login_filled.png") + + page.locator("#kc-login").click() + page.screenshot(path="screenshots/login_done.png") + + + + +#def test_get_started_link(page: Page): +# page.goto("https://playwright.dev/") +# +# # Click the get started link. +# page.get_by_role("link", name="Get started").click() +# +# # Expects page to have a heading with the name of Installation. +# expect(page.get_by_role("heading", name="Installation")).to_be_visible() \ No newline at end of file diff --git a/src/frontend_celery/playwright/update_static_data.sh b/src/frontend_celery/playwright/update_static_data.sh new file mode 100755 index 00000000..0b6140e0 --- /dev/null +++ b/src/frontend_celery/playwright/update_static_data.sh @@ -0,0 +1,82 @@ +#!/bin/bash +set -e +set -o pipefail +#set -o verbose + + +helpFunction() +{ + echo "" + echo "Usage: $0 -w env" + echo "This script dumps the static data from HerediVar for tests. This data is used for seeding the database during tests." + echo -e "\t-w Provide 'dev' for development server and 'prod' for production config." + exit 1 # Exit script after printing help +} + +while getopts "w:" opt +do + case "$opt" in + w ) we="$OPTARG" ;; + ? ) helpFunction ;; # Print helpFunction in case parameter is non-existent + esac +done + +# Print helpFunction in case parameters are empty +if [ -z "$we" ] +then + echo "Some or all of the parameters are empty"; + helpFunction +fi + + +# set paths +SCRIPT=$(readlink -f "$0") +ROOT=$(dirname $(dirname $(dirname "$SCRIPT"))) +cd $ROOT +pwd + + +export WEBAPP_ENV=$we +if [ -z "${WEBAPP_ENV}" ] +then + echo Environment variable WEBAPP_ENV not set. Use -w option + exit 1 +fi + +echo using "${WEBAPP_ENV}" config + + +SCRIPTPATH=$(dirname "$SCRIPT") +echo script: $SCRIPT +echo scriptpath: $SCRIPTPATH +ROOT=$(dirname $(dirname $(dirname $(dirname "$SCRIPT")))) +echo Root dir: $ROOT + +#cd $SCRIPTPATH + + +# load environment variables +set -o allexport +extension=env_ +source $ROOT/.$extension$WEBAPP_ENV +set +o allexport + + +# setup paths +path_to_structure=$SCRIPTPATH/data/db_structure +mkdir -p $path_to_structure +path_to_structure=$path_to_structure/structure.sql +path_to_data=$SCRIPTPATH/data/db_seeds +mkdir -p $path_to_data +path_to_data=$path_to_data/static.sql + +echo $path_to_structure +echo $path_to_data + +# dump structure +mysqldump --quick --column-statistics=0 $DB_NAME -P $DB_PORT -h $DB_HOST -u $DB_ADMIN -p$DB_ADMIN_PW --no-tablespaces --no-data -r $path_to_structure +gzip --force $path_to_structure + +# dump static data +mysqldump --quick --column-statistics=0 $DB_NAME -P $DB_PORT -h $DB_HOST -u $DB_ADMIN -p$DB_ADMIN_PW --no-tablespaces --no-create-info -r $path_to_data annotation_type classification_scheme classification_criterium classification_criterium_strength classification_scheme_alias mutually_exclusive_criteria mutually_inclusive_criteria +#gzip $path_to_data diff --git a/src/frontend_celery/playwright/utils/functions.py b/src/frontend_celery/playwright/utils/functions.py new file mode 100644 index 00000000..a52711a0 --- /dev/null +++ b/src/frontend_celery/playwright/utils/functions.py @@ -0,0 +1,8 @@ + + + + + +def subscribe_request_response(page): + page.on("request", lambda request: print(">>", request.method, request.url)) + page.on("response", lambda response: print("<<", response.status, response.url)) \ No newline at end of file diff --git a/src/frontend_celery/start_webapp.sh b/src/frontend_celery/start_webapp.sh index df566b9d..9bd1f6b4 100755 --- a/src/frontend_celery/start_webapp.sh +++ b/src/frontend_celery/start_webapp.sh @@ -53,6 +53,11 @@ then python3 main.py fi +if [ "${WEBAPP_ENV}" == "localtest" ] +then + python3 main.py +fi + if [ "${WEBAPP_ENV}" == "prod" ] then #export CURL_CA_BUNDLE="" diff --git a/src/frontend_celery/tests/script/start_keycloak_for_tests.sh b/src/frontend_celery/tests/script/start_keycloak_for_tests.sh index 39069321..36a4c97b 100755 --- a/src/frontend_celery/tests/script/start_keycloak_for_tests.sh +++ b/src/frontend_celery/tests/script/start_keycloak_for_tests.sh @@ -6,7 +6,7 @@ SCRIPTPATH=$(dirname "$SCRIPT") cd $SCRIPTPATH cd ../ pwd - +ROOT=$(dirname $(dirname $(dirname $(dirname "$SCRIPTPATH")))) function waitForServer { dist=$1 @@ -32,5 +32,10 @@ function waitForServer { done } -keycloak-18.0.0/bin/kc.sh start-dev --hostname 127.0.0.1 --http-port 5050 > keycloak.log 2>&1 & # --log-level debug +set -o allexport +extension=env_ +source $ROOT/.$extension$WEBAPP_ENV +set +o allexport + +keycloak-18.0.0/bin/kc.sh start-dev --hostname $KEYCLOAK_HOST --http-port $KEYCLOAK_PORT > keycloak.log 2>&1 & # --log-level debug waitForServer "Keycloak" \ No newline at end of file diff --git a/src/frontend_celery/webapp/auth/auth_routes.py b/src/frontend_celery/webapp/auth/auth_routes.py index 0f118841..0ca8bf8d 100644 --- a/src/frontend_celery/webapp/auth/auth_routes.py +++ b/src/frontend_celery/webapp/auth/auth_routes.py @@ -407,42 +407,42 @@ def profile(): @auth_blueprint.route('/login') def login(): - # you can not use the regular login route during testing because this redirects to the keycloak form which can not - # be filled computationally - if current_app.config.get('TESTING', False): - issuer = current_app.config['ISSUER'] - url = f'{issuer}/protocol/openid-connect/token' - data = {'client_id':current_app.config['CLIENTID'], 'client_secret': current_app.config['CLIENTSECRET'], 'grant_type': 'password', 'username': 'superuser', 'password': '12345'} - token_response = requests.post(url = url, data=data) - assert token_response.status_code == 200 - session['tokenResponse'] = token_response.json() - - url = f'{issuer}/protocol/openid-connect/userinfo' - data = {'token': session["tokenResponse"]["access_token"], 'token_type_hint': 'access_token', 'client_secret': current_app.config['CLIENTSECRET'], 'client_id': current_app.config['CLIENTID']} - header = {'Authorization': f'Bearer {session["tokenResponse"]["access_token"]}'} - user_response = requests.post(url = url, data=data, headers=header) - assert user_response.status_code == 200 - user_info = user_response.json() - - # this is only to record which user made which actions in the database and has nothing to do with authenitication - username = user_info['preferred_username'] - first_name = user_info['given_name'] - last_name = user_info['family_name'] - affiliation = user_info.get('affiliation') - - # init the session - session['user'] = user_info - - if affiliation is None or affiliation.strip() == '': - flash('LOGIN ERROR: You are missing the affiliation tag ask a HerediVar administrator to add it!', 'alert-danger') - current_app.logger.error("Could not login user " + username + ", because the user was missing affiliation tag in keycloak.") - return redirect(url_for('auth.logout', auto_logout='True')) - conn = Connection(session['user']['roles']) - conn.insert_user(username, first_name, last_name, affiliation) # this inserts only if the user is not already in the database and updates the information if the information changed (except for username this one has to stay) - user_info['user_id'] = conn.get_user_id(username) - conn.close() - - return save_redirect(request.args.get('next_login', url_for('main.index'))) + ## you can not use the regular login route during testing because this redirects to the keycloak form which can not + ## be filled computationally + #if current_app.config.get('TESTING', False): + # issuer = current_app.config['ISSUER'] + # url = f'{issuer}/protocol/openid-connect/token' + # data = {'client_id':current_app.config['CLIENTID'], 'client_secret': current_app.config['CLIENTSECRET'], 'grant_type': 'password', 'username': 'superuser', 'password': '12345'} + # token_response = requests.post(url = url, data=data) + # assert token_response.status_code == 200 + # session['tokenResponse'] = token_response.json() + # + # url = f'{issuer}/protocol/openid-connect/userinfo' + # data = {'token': session["tokenResponse"]["access_token"], 'token_type_hint': 'access_token', 'client_secret': current_app.config['CLIENTSECRET'], 'client_id': current_app.config['CLIENTID']} + # header = {'Authorization': f'Bearer {session["tokenResponse"]["access_token"]}'} + # user_response = requests.post(url = url, data=data, headers=header) + # assert user_response.status_code == 200 + # user_info = user_response.json() + # + # # this is only to record which user made which actions in the database and has nothing to do with authenitication + # username = user_info['preferred_username'] + # first_name = user_info['given_name'] + # last_name = user_info['family_name'] + # affiliation = user_info.get('affiliation') + # + # # init the session + # session['user'] = user_info + # + # if affiliation is None or affiliation.strip() == '': + # flash('LOGIN ERROR: You are missing the affiliation tag ask a HerediVar administrator to add it!', 'alert-danger') + # current_app.logger.error("Could not login user " + username + ", because the user was missing affiliation tag in keycloak.") + # return redirect(url_for('auth.logout', auto_logout='True')) + # conn = Connection(session['user']['roles']) + # conn.insert_user(username, first_name, last_name, affiliation) # this inserts only if the user is not already in the database and updates the information if the information changed (except for username this one has to stay) + # user_info['user_id'] = conn.get_user_id(username) + # conn.close() + # + # return save_redirect(request.args.get('next_login', url_for('main.index'))) # construct redirect uri: first redirect to keycloak login page # then redirect to auth with the next param which defaults to the '/' route diff --git a/src/frontend_celery/webapp/templates/index.html b/src/frontend_celery/webapp/templates/index.html index b214cc89..0e7dbb61 100644 --- a/src/frontend_celery/webapp/templates/index.html +++ b/src/frontend_celery/webapp/templates/index.html @@ -107,6 +107,7 @@

Changelog

  • Fixed a bug where some variants got raw SpliceAI scores instead of the masked ones
  • When constructing an URL for the variant details page it was previously not possible to omit the "chr" prefix for the chromosome
  • Task force protein domains are now shown correctly
    • +
  • Transcripts on the negative strand were shown as if they were on the positive strand in the IGV browser
    • Known issues: