Merge branch 'release/1.9.0'

dockstore · Jun 29, 2020 · 23e4bd1 · 23e4bd1
2 parents 868644d + 5161708
commit 23e4bd1
Show file tree

Hide file tree

Showing 16 changed files with 221 additions and 202 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -2,6 +2,11 @@ sudo: required
 services:
 - docker
 
+addons:
+  apt:
+    packages:
+    - nginx
+
 before_install:
 - gem install mustache
 
@@ -10,3 +15,5 @@ language: bash
 install:
 - travis_wait 30 bash install_bootstrap --script
 - docker-compose version
+# validate the nginx configuration files using nginx via docker
+- docker run -v $PWD/config/default.nginx_http.conf:/etc/nginx/conf.d/default.conf:ro -v $PWD/config/default.nginx_http.shared.conf:/etc/nginx/conf.d/default.nginx_http.shared.conf:ro nginx:1.13.1 nginx -t -c /etc/nginx/nginx.conf
diff --git a/DEV-README.md b/DEV-README.md
@@ -4,16 +4,7 @@ There are 3 different sets of metric logs being sent to logstash's elasticsearch
 2. Dropwizard Metrics => Metricbeats => elasticsearch
 3. Postgres Metrics => Metricbeats => elasticsearch
 
-## Additional Security Setup
-
-### HTTPS Renewal
-`/home/ubuntu/certbot-auto renew &> /home/ubuntu/logs/certbot-auto.log`
-
-### Password Generation
-Use `htpasswd -c ./nginx/kibana.htpasswd user1` to set up a password for nginx.  If the password doesn't work, try using a different type of encryption.
-
 ## Ports
-- Port 9200 must be opened for metricbeats to send data to elasticsearch directly
 - Port 5055 must be opened for the production webservice to send data to logstash
 - Port 5066 must be opened for the staging webservice to send data to logstash
 - Port 5601 must be opened for developers to view the kibana dashboard

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
@@ -60,8 +60,8 @@ services:
     restart: always
     volumes:
       - ./staticConfig/kibana.yml:/usr/share/kibana/config/kibana.yml
-    expose:
-      - "5601"
+    ports:
+      - "35943:5601"
     depends_on:
       - elasticsearch-logstash
     environment:
@@ -95,27 +95,6 @@ services:
         awslogs-group: ${LOG_GROUP_NAME}
         awslogs-stream: "elastalert"
 
-  # courtesy of https://blog.t1cg.io/post/elk-aws-deployment.html#installing-docker-nginx-certbot-and-the-elk-stack
-  nginx:
-    build:
-      context: nginx/
-    restart: always
-    ports:
-      - "35943:443"
-      - "80:80"
-    volumes:
-      - "/etc/letsencrypt/live/logging.dockstore.org/fullchain.pem:/etc/letsencrypt/live/logging.dockstore.org/fullchain.pem"
-      - "/etc/letsencrypt/live/logging.dockstore.org/privkey.pem:/etc/letsencrypt/live/logging.dockstore.org/privkey.pem"
-      - "./nginx/kibana.htpasswd:/etc/nginx/conf.d/kibana.htpasswd"
-      - "./nginx/default.conf:/etc/nginx/conf.d/default.conf"
-    depends_on:
-      - kibana
-    logging:
-      driver: "awslogs"
-      options:
-        awslogs-group: ${LOG_GROUP_NAME}
-        awslogs-stream: "nginx"
-
 volumes:
   esdata2:
     driver: local
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,18 +1,6 @@
 version: '2'
 
 services:
-  postgres:
-    image: postgres:10.10
-    restart: always
-    volumes:
-      - ./postgres-data:/var/lib/postgresql/data
-    expose: 
-      - "5432"
-    logging:
-      driver: "awslogs"
-      options:
-        awslogs-group: ${LOG_GROUP_NAME}
-        awslogs-stream: "postgres"
 
   elasticsearch:
     image: docker.elastic.co/elasticsearch/elasticsearch:5.6.3
@@ -40,33 +28,31 @@ services:
         awslogs-stream: "elasticsearch"
 
   migration: 
-    build:
-      context: .
-      dockerfile: config/Dockerfile_webservice
-    depends_on:
-      - postgres
+    image: quay.io/dockstore/dockstore-webservice:${DOCKSTORE_VERSION}
     volumes:
       - log_volume:/dockstore_logs
       - ./config/web.yml:/home/web.yml
-    command: ["dockerize", "-wait", "tcp://postgres:5432", "-timeout", "60s", "/home/init_migration.sh"]
+      - ./config/init_migration.sh:/home/init_migration.sh
+    command: ["bash", "/home/init_migration.sh"]
     logging:
       driver: "awslogs"
       options:
         awslogs-group: ${LOG_GROUP_NAME}
         awslogs-stream: "migration"
 
   webservice: 
-    build:
-      context: .
-      dockerfile: config/Dockerfile_webservice
+    image: quay.io/dockstore/dockstore-webservice:${DOCKSTORE_VERSION}
     restart: always
     depends_on:
-      - postgres
       - elasticsearch
       - migration
     volumes:
       - log_volume:/dockstore_logs
       - ./config/web.yml:/home/web.yml
+      - ./config/init_webservice.sh:/home/init_webservice.sh
+      - ${GITHUB_APP_PRIVATE_KEY_FILE}:/home/dockstore_github_app_private_key.pem
+      - ../language-plugins/:/root/.dockstore/language-plugins
+    command: ["bash", "/home/init_webservice.sh"]
     expose:
       - "8081"
     logging:
@@ -100,7 +86,6 @@ services:
 #    restart: always
 #    depends_on:
 #      - webservice
-#      - postgres
 #    volumes:
 #      - /proc:/hostfs/proc:ro
 #      - /sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro

diff --git a/dockstore_launcher_config/compose.config b/dockstore_launcher_config/compose.config
@@ -1,11 +1,10 @@
 {
 "PUBLIC_LAUNCHER_IP_ADDRESS":"",
+"PRODUCTION":false,
 "DOCKSTORE_VERSION":"1.8.0-beta.4",
-"GITHUB_CLIENT1_ID":"foobared",
+"UI2_HASH":"develop",
 "GITHUB_CLIENT2_ID":"foobared",
-"GITHUB_CLIENT1_SECRET":"foobared",
 "GITHUB_CLIENT2_SECRET":"foobared",
-"UI2_HASH":"develop",
 "QUAY_CLIENT_ID":"foobared",
 "QUAY_CLIENT_SECRET":"foobared",
 "BITBUCKET_CLIENT_ID":"foobared",
@@ -14,19 +13,36 @@
 "HTTPS":false,
 "DISCOURSE_URL":"foobar",
 "DISCOURSE_KEY":"foobar",
-"GOOGLE_VERIFICATION_NAME":"foobar",
+"DISCOURSE_CATEGORY_ID":"10",
+"DOCUMENTATION_URL":"foobar",
+"FEATURED_CONTENT_URL":"foobar",
+"GITHUB_APP_PRIVATE_KEY_FILE": "/replaceme",
+"GITHUB_APP_ID": "1234",
+"GITHUB_APP_NAME": "foobar",
+"TOOLTESTER_BUCKET_NAME": "replaceme",
 "TAG_MANAGER_ID":"foobar",
 "GITLAB_CLIENT_ID":"foobar",
 "GITLAB_CLIENT_SECRET":"foobar",
 "GOOGLE_CLIENT_ID":"potato",
 "GOOGLE_CLIENT_SECRET":"potato",
 "LOGSTASH":false,
-"ENABLE_CWL_VIEWER":"false",
-"ENABLE_LAUNCH_WITH_FIRECLOUD":"false",
-"FIRECLOUD_IMPORT_URL":"foobar",
-"GITHUB_APP_PRIVATE_KEY_FILE":"config/web.yml",
-"GITHUB_APP_ID":"1234",
-"GITHUB_APP_NAME":"foobar",
-"DOCUMENTATION_URL":"foobar",
-"FEATURED_CONTENT_URL":"foobar"
+"LOGSTASH_HOST":"replaceme",
+"TERRA_IMPORT_URL":"replaceme",
+"BD_CATALYST_SEVEN_BRIDGES_IMPORT_URL":"replaceme",
+"BD_CATALYST_TERRA_IMPORT_URL":"replaceme",
+"AUTHORIZER_TYPE":"replaceme",
+"EXTERNAL_GOOGLE_CLIENT_PREFIX1":"replaceme",
+"SAM_PATH":"replaceme",
+"DATABASE_GENERATED":false,
+"ORCID_CLIENT_ID":"replaceme",
+"ORCID_CLIENT_SECRET":"replaceme",
+"ORCID_URL":"replaceme",
+"ZENODO_CLIENT_ID":"replaceme",
+"ZENODO_CLIENT_SECRET":"replaceme",
+"ZENODO_URL":"replaceme",
+"SLACK_URL":"replaeceme",
+"GALAXY_PLUGIN_VERSION":"0.0.4",
+"DATABASE_DOMAIN":"postgres",
+"DBUSER":"dockstore",
+"DBPASSWORD":"dockstore"
 }