From f907546a41d74bfd49d46addc44cf06fa48c2460 Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Fri, 23 Sep 2022 10:39:21 -0400 Subject: [PATCH 1/6] Set IS_FARGATE_DEPLOY to true --- dockstore_launcher_config/compose.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockstore_launcher_config/compose.config b/dockstore_launcher_config/compose.config index b06a079..b70aee7 100644 --- a/dockstore_launcher_config/compose.config +++ b/dockstore_launcher_config/compose.config @@ -39,7 +39,7 @@ "GOOGLE_CLIENT_ID":"potato", "GOOGLE_CLIENT_SECRET":"potato", "HTTPS":false, -"IS_FARGATE_DEPLOY":false, +"IS_FARGATE_DEPLOY":true, "LOGSTASH":false, "LOGSTASH_HOST":"replaceme", "NEXTFLOW_PARSING_LAMBDA_VERSION":"n/a", From c64a3c744c8fca6eee92308303660bfbb7c29c9e Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Fri, 23 Sep 2022 10:45:35 -0400 Subject: [PATCH 2/6] Revert "Set IS_FARGATE_DEPLOY to true" This reverts commit b1d712488ececed4607488a0b70276f9dff7bd6f. --- dockstore_launcher_config/compose.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockstore_launcher_config/compose.config b/dockstore_launcher_config/compose.config index b70aee7..b06a079 100644 --- a/dockstore_launcher_config/compose.config +++ b/dockstore_launcher_config/compose.config @@ -39,7 +39,7 @@ "GOOGLE_CLIENT_ID":"potato", "GOOGLE_CLIENT_SECRET":"potato", "HTTPS":false, -"IS_FARGATE_DEPLOY":true, +"IS_FARGATE_DEPLOY":false, "LOGSTASH":false, "LOGSTASH_HOST":"replaceme", "NEXTFLOW_PARSING_LAMBDA_VERSION":"n/a", From 44ea6803bdcdd56fc8d73bbda1ee2ed07a7dc813 Mon Sep 17 00:00:00 2001 From: Kathy Tran Date: Fri, 23 Sep 2022 10:47:12 -0400 Subject: [PATCH 3/6] Remove log volume --- docker-compose.yml | 3 --- templates/init_migration.sh.template | 6 +++--- templates/init_webservice.sh.template | 2 +- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 1d4776d..26fc671 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,6 @@ services: migration: image: quay.io/dockstore/dockstore-webservice@${DOCKSTORE_IMAGE_DIGEST} volumes: - - log_volume:/dockstore_logs - ./config/web.yml:/home/web.yml - ./config/init_migration.sh:/home/init_migration.sh command: ["bash", "/home/init_migration.sh"] @@ -21,7 +20,6 @@ services: depends_on: - migration volumes: - - log_volume:/dockstore_logs - ./config/web.yml:/home/web.yml - ./config/init_webservice.sh:/home/init_webservice.sh - ${GITHUB_APP_PRIVATE_KEY_FILE}:/dockstore/github-key/dockstore-github-private-key.pem @@ -75,6 +73,5 @@ services: # awslogs-stream: "metricbeat" volumes: - log_volume: esdata1: driver: local diff --git a/templates/init_migration.sh.template b/templates/init_migration.sh.template index afb5824..4c3444f 100755 --- a/templates/init_migration.sh.template +++ b/templates/init_migration.sh.template @@ -4,12 +4,12 @@ cd "$(dirname "$0")" {{#DATABASE_GENERATED}} -java -Ddw.database.user=postgres -Ddw.database.password="{{{ POSTGRES_DBPASSWORD }}}" -jar /home/dockstore-webservice-*.jar db migrate web.yml --include 1.3.0.generated,1.3.1.consistency,1.4.0,1.5.0,1.6.0,1.7.0 | tee --append /dockstore_logs/webservice.out +java -Ddw.database.user=postgres -Ddw.database.password="{{{ POSTGRES_DBPASSWORD }}}" -jar /home/dockstore-webservice-*.jar db migrate web.yml --include 1.3.0.generated,1.3.1.consistency,1.4.0,1.5.0,1.6.0,1.7.0 {{/DATABASE_GENERATED}} {{^DATABASE_GENERATED}} -java -Ddw.database.user=postgres -Ddw.database.password="{{{ POSTGRES_DBPASSWORD }}}" -jar /home/dockstore-webservice-*.jar db migrate web.yml --include 1.3.1.consistency,1.4.0,1.5.0,1.6.0,1.7.0 | tee --append /dockstore_logs/webservice.out +java -Ddw.database.user=postgres -Ddw.database.password="{{{ POSTGRES_DBPASSWORD }}}" -jar /home/dockstore-webservice-*.jar db migrate web.yml --include 1.3.1.consistency,1.4.0,1.5.0,1.6.0,1.7.0 {{/DATABASE_GENERATED}} # this particular migration needs to run as postgres because only postgres can surrender ownership java -Ddw.database.user=postgres -Ddw.database.password="{{{ POSTGRES_DBPASSWORD }}}" -jar /home/dockstore-webservice-*.jar db migrate web.yml --include 1.7.0.relinquish # future migrations will start here and should be run as dockstore -java -Ddw.database.user=dockstore -Ddw.database.password="{{{ DOCKSTORE_DBPASSWORD }}}" -jar /home/dockstore-webservice-*.jar db migrate web.yml --include 1.8.0,1.9.0,1.10.0,1.11.0,1.12.0,1.13.0 | tee --append /dockstore_logs/webservice.out +java -Ddw.database.user=dockstore -Ddw.database.password="{{{ DOCKSTORE_DBPASSWORD }}}" -jar /home/dockstore-webservice-*.jar db migrate web.yml --include 1.8.0,1.9.0,1.10.0,1.11.0,1.12.0,1.13.0 diff --git a/templates/init_webservice.sh.template b/templates/init_webservice.sh.template index 6ea6adc..26b913d 100755 --- a/templates/init_webservice.sh.template +++ b/templates/init_webservice.sh.template @@ -2,6 +2,6 @@ cd "$(dirname "$0")" -java -XX:MaxRAMPercentage=50.0 -XX:+ExitOnOutOfMemoryError -jar /home/dockstore-webservice-*.jar server web.yml | tee --append /dockstore_logs/webservice.out +java -XX:MaxRAMPercentage=50.0 -XX:+ExitOnOutOfMemoryError -jar /home/dockstore-webservice-*.jar server web.yml From 1d3eac309b2b3e50c87e2a50aeb20559895c7d06 Mon Sep 17 00:00:00 2001 From: Steve Von Worley Date: Tue, 25 Oct 2022 12:53:30 -0700 Subject: [PATCH 4/6] include galaxy plugin in webservice docker image https://ucsc-cgl.atlassian.net/browse/SEAB-4880 --- docker-compose.yml | 1 - install_bootstrap | 14 -------------- scripts/webservice-image-digest.py | 7 ++++--- 3 files changed, 4 insertions(+), 18 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 26fc671..2a18ae8 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,7 +23,6 @@ services: - ./config/web.yml:/home/web.yml - ./config/init_webservice.sh:/home/init_webservice.sh - ${GITHUB_APP_PRIVATE_KEY_FILE}:/dockstore/github-key/dockstore-github-private-key.pem - - ../language-plugins/:/root/.dockstore/language-plugins command: ["bash", "/home/init_webservice.sh"] ports: - "8081:8081" diff --git a/install_bootstrap b/install_bootstrap index 02dbf3a..f103bc0 100755 --- a/install_bootstrap +++ b/install_bootstrap @@ -66,18 +66,6 @@ function template() for f in $(ls templates/rules/); do mustache dockstore_launcher_config/compose.config templates/rules/$f > config/rules/$f; done } -# Delete any galaxy plugin versions already in folder and download if a version has been specified. -function download_galaxy() -{ - if [ ! -d "../language-plugins" ] ; then - mkdir ../language-plugins - fi - - find ../language-plugins -regextype posix-extended -regex '.*dockstore-galaxy-interface.*' -delete - wget -nc -P ../language-plugins/ https://artifacts.oicr.on.ca/artifactory/collab-release/com/github/galaxyproject/dockstore-galaxy-interface/dockstore-galaxy-interface/${GALAXY_PLUGIN_VERSION}/dockstore-galaxy-interface-${GALAXY_PLUGIN_VERSION}.jar - -} - #Read the config file if it exists if [ -f dockstore_launcher_config/compose.config ] ; then source <(jq -r 'to_entries|map("\(.key)=\"\(.value|tostring)\"")|.[]' dockstore_launcher_config/compose.config) @@ -85,8 +73,6 @@ fi template -download_galaxy "$*" - if [ $IS_FARGATE_DEPLOY == "false" ]; then # We need to set the environment variable for the image digest source .env diff --git a/scripts/webservice-image-digest.py b/scripts/webservice-image-digest.py index 1d7dbe6..d50bf87 100755 --- a/scripts/webservice-image-digest.py +++ b/scripts/webservice-image-digest.py @@ -58,9 +58,10 @@ def get_commit_from_github(tag_or_branch): def get_digest_from_s3(directory): # downloads the image-digest.txt from a directory in S3 base_url = "https://gui.dockstore.org" - response = requests.get("{}/{}/image-digest.txt".format(base_url, directory)) + digest_url = "{}/{}/image-digest.txt".format(base_url, directory.replace("/", "_")) + response = requests.get(digest_url) if (response.status_code != 200): - print("Expected a file at {}".format("{}/{}/image-digest.txt".format(base_url, directory))) + print("Expected a file at {}".format(digest_url)) print("The image-digest.txt was not found in S3, did the build succeed?") exit(1) # There is a newline at the end of the file we rstrip @@ -70,7 +71,7 @@ def get_digest_from_s3(directory): # slashes are replaced with _ in docker image tags # check to see if input includes a dash followed by 7 chars parsed = args.tag.split('-') - if len(parsed) == 2 and len(parsed[1]) == 7 and all(c in string.hexdigits for c in parsed[1]): + if len(parsed) >= 2 and len(parsed[-1]) == 7 and all(c in string.hexdigits for c in parsed[-1]): directory = args.tag else: commit = get_commit_from_github(args.tag) From 69e8fffd182960e50886b78d9785cf77112acab6 Mon Sep 17 00:00:00 2001 From: David Steinberg Date: Mon, 24 Oct 2022 15:26:22 -0700 Subject: [PATCH 5/6] Add a header for noindex and nofollow This will hopefully stop most search engines from indexing non-production Dockstore instances --- templates/default.nginx_http.security.conf.template | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/templates/default.nginx_http.security.conf.template b/templates/default.nginx_http.security.conf.template index e692a95..e177735 100644 --- a/templates/default.nginx_http.security.conf.template +++ b/templates/default.nginx_http.security.conf.template @@ -27,3 +27,7 @@ proxy_hide_header Server; add_header X-Content-Type-Options "nosniff" always; add_header Strict-Transport-Security $hsts_header always; + +{{^PRODUCTION}} +add_header X-Robots-Tag "noindex, nofollow" always; +{{/PRODUCTION}} \ No newline at end of file From 7eeccb9f607e2a487d845a9c6b0125abfac28769 Mon Sep 17 00:00:00 2001 From: David Steinberg Date: Wed, 26 Oct 2022 09:35:01 -0700 Subject: [PATCH 6/6] Add newline end of file --- templates/default.nginx_http.security.conf.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/default.nginx_http.security.conf.template b/templates/default.nginx_http.security.conf.template index e177735..c740558 100644 --- a/templates/default.nginx_http.security.conf.template +++ b/templates/default.nginx_http.security.conf.template @@ -30,4 +30,4 @@ add_header Strict-Transport-Security $hsts_header always; {{^PRODUCTION}} add_header X-Robots-Tag "noindex, nofollow" always; -{{/PRODUCTION}} \ No newline at end of file +{{/PRODUCTION}}