Merge pull request #314 from KPMP/develop

Q1 2024 Atlas Release

atlas/knowledge-environment/.env.example

@@ -54,3 +54,6 @@ MEM_LIMIT=1073741824
 
 # Authorization token for Enterprise Search API
 ENV_ES_API_TOKEN=
+
+# Most recent date for the Recently Released file category
+RECENTLY_RELEASED_DATE=

atlas/knowledge-environment/docker-compose.dev.yml

@@ -37,6 +37,7 @@ services:
       MYSQL_HOST: ${ENV_MYSQL_HOST}
       MYSQL_USER: ${ENV_MYSQL_USER}
       MYSQL_PASSWORD: ${ENV_MYSQL_PASSWORD}
+      RECENTLY_RELEASED_DATE: ${RECENTLY_RELEASED_DATE}
       TZ: "America/Detroit"
     privileged: true
     logging:
@@ -87,6 +88,8 @@ services:
 
   atlas-file-service:
     image: kingstonduo/atlas-file-service:3.3
+    depends_on: 
+      - mariadb    
     ports:
       - "5000:5000"
     volumes:
@@ -100,6 +103,9 @@ services:
       - MYSQL_USER=${ENV_MYSQL_USER}
       - MYSQL_PASSWORD=${ENV_MYSQL_PASSWORD}      
       - BUCKET_NAME=${ENV_BUCKET_NAME}
+      - MYSQL_HOST=${ENV_MYSQL_HOST}
+      - MYSQL_USER=${ENV_MYSQL_USER}
+      - MYSQL_PASSWORD=${ENV_MYSQL_PASSWORD}
       - TZ=America/Detroit
     networks:
       local:

atlas/knowledge-environment/docker-compose.local.yml

@@ -37,6 +37,7 @@ services:
       MYSQL_HOST: ${ENV_MYSQL_HOST}
       MYSQL_USER: ${ENV_MYSQL_USER}
       MYSQL_PASSWORD: ${ENV_MYSQL_PASSWORD}
+      RECENTLY_RELEASED_DATE: ${RECENTLY_RELEASED_DATE}
       TZ: "America/Detroit"
     privileged: true
     logging:
@@ -61,6 +62,7 @@ services:
       MYSQL_HOST: ${ENV_MYSQL_HOST}
       MYSQL_USER: ${ENV_MYSQL_USER}
       MYSQL_PASSWORD: ${ENV_MYSQL_PASSWORD}
+      RECENTLY_RELEASED_DATE: ${RECENTLY_RELEASED_DATE}
       TZ: "America/Detroit"
     privileged: true
     logging:
@@ -111,6 +113,8 @@ services:
 
   atlas-file-service:
     image: kingstonduo/atlas-file-service:3.3
+    depends_on: 
+      - mariadb    
     ports:
       - "5000:5000"
     volumes:
@@ -124,6 +128,9 @@ services:
       - MYSQL_USER=${ENV_MYSQL_USER}
       - MYSQL_PASSWORD=${ENV_MYSQL_PASSWORD}      
       - BUCKET_NAME=${ENV_BUCKET_NAME}
+      - MYSQL_HOST=${ENV_MYSQL_HOST}
+      - MYSQL_USER=${ENV_MYSQL_USER}
+      - MYSQL_PASSWORD=${ENV_MYSQL_PASSWORD}
       - "TZ=America/Detroit"
     networks:
       local:

atlas/knowledge-environment/docker-compose.prod.yml

@@ -38,6 +38,7 @@ services:
       MYSQL_USER: ${ENV_MYSQL_USER}
       MYSQL_PASSWORD: ${ENV_MYSQL_PASSWORD}
       ES_API_TOKEN: ${ENV_ES_API_TOKEN}
+      RECENTLY_RELEASED_DATE: ${RECENTLY_RELEASED_DATE}
       TZ: "America/Detroit"
     privileged: true
     logging:
@@ -63,6 +64,7 @@ services:
       MYSQL_USER: ${ENV_MYSQL_USER}
       MYSQL_PASSWORD: ${ENV_MYSQL_PASSWORD}
       ES_API_TOKEN: ${ENV_ES_API_TOKEN}
+      RECENTLY_RELEASED_DATE: ${RECENTLY_RELEASED_DATE}
       TZ: "America/Detroit"
     privileged: true
     logging:
@@ -131,6 +133,9 @@ services:
       - MYSQL_USER=${ENV_MYSQL_USER}
       - MYSQL_PASSWORD=${ENV_MYSQL_PASSWORD}      
       - BUCKET_NAME=${ENV_BUCKET_NAME}
+      - MYSQL_HOST=${ENV_MYSQL_HOST}
+      - MYSQL_USER=${ENV_MYSQL_USER}
+      - MYSQL_PASSWORD=${ENV_MYSQL_PASSWORD}
       - "TZ=America/Detroit"
     networks:
       local:

cassiopeia/.env.example

@@ -3,7 +3,7 @@ ENV_DOCKER_ENVIRONMENT=development
 
 # Spring
 ENV_APACHE_TOMCAT_PORT=3030
-ENV_SPRING_BOOT_APPDIR=/path/to/your/spring/code
+ENV_SPRING_BOOT_CONTAINER=dockerImage/cassiopiea
 ENV_WSI_FILES_DIR=/data/deepZoomImages
 ENV_WSI_ORIG_FILES_DIR=/data/knowledgeEnvironment/deepZoom
 ENV_FLUENTD_ADDRESS=path.to.your.site:24224

cassiopeia/docker-compose.dev.yml

@@ -39,7 +39,7 @@ services:
 
   spring:
     container_name: cassie-spring
-    image: kingstonduo/cassiopeia-data:latest
+    image: ${ENV_SPRING_BOOT_CONTAINER}
     environment:
       - "TZ=America/Detroit"
     ports:

cassiopeia/docker-compose.prod.yml

@@ -40,7 +40,7 @@ services:
 
   spring:
     container_name: cassie-spring
-    image: kingstonduo/cassiopeia-data:1.2
+    image: ${ENV_SPRING_BOOT_CONTAINER}
     environment:
       - "TZ=America/Detroit"
     ports:

images/apache/apache-dl-proxy/Dockerfile

@@ -1,48 +1,32 @@
-FROM oraclelinux:8.5
+FROM debian:buster-slim
 
-# Define args and set a default value
-ARG maintainer=tier
-ARG imagename=shibboleth_sp
-ARG version=2.6.1
+SHELL ["/bin/bash", "-c"]
 
-LABEL Maintainer=$maintainer
-LABEL Vendor="Internet2"
-LABEL ImageType="Base"
-LABEL ImageName=$imagename
-LABEL ImageOS=centos7
-LABEL Version=$version
-
-LABEL Build docker build --rm --tag $maintainer/$imagename .
-
-# Add starters and installers
 ADD ./container_files /opt
 
-RUN curl -o /etc/yum.repos.d/security:shibboleth.repo \
-      http://download.opensuse.org/repositories/security://shibboleth/CentOS_7/security:shibboleth.repo \
-      && yum -y update \
-      && yum -y install \
-        httpd \
-        mod_ssl \
-        shibboleth.x86_64 \
-        dos2unix \
-      && yum clean all \
-      && rm /etc/httpd/conf.d/autoindex.conf \
-      && rm /etc/httpd/conf.d/ssl.conf \
-      && rm /etc/httpd/conf.d/userdir.conf \
-      && rm /etc/httpd/conf.d/welcome.conf \
-      && rm /etc/localtime \
-      && chmod +x /opt/bin/httpd-shib-foreground \
-      && chmod +x /opt/bin/shibboleth_keygen.sh
-
+RUN apt-get -qq update && \
+    apt-get -qq -y --no-install-recommends install \
+        apache2 \
+        libapache2-mod-shib2 \
+        curl \
+        openssl \ 
+        && rm -rf /var/lib/apt/lists/*
+
+RUN rm /etc/localtime \
+    && chmod +x /opt/bin/httpd-shib-foreground \
+    && chmod +x /opt/bin/shibboleth_keygen.sh
+
 # Export this variable so that shibd can find its CURL library
 RUN LD_LIBRARY_PATH="/opt/shibboleth/lib64"
 RUN export LD_LIBRARY_PATH
 
-#Script to start service, Added ssl default conf, Added shib module apache
-RUN ln -s /opt/bin/httpd-shib-foreground  /usr/local/bin && ln -s /opt/etc/httpd/conf.d/ssl.conf /etc/httpd/conf.d/ssl.conf && ln -s /opt/etc/httpd/conf.d/virt.conf /etc/httpd/conf.d/virt.conf && ln -s /opt/etc/httpd/conf.modules.d/00-shib.conf /etc/httpd/conf.modules.d/00-shib.conf && ln -s /usr/lib64/shibboleth/mod_shib_24.so /etc/httpd/modules/mod_shib_24.so && ln -s /usr/share/zoneinfo/America/Detroit /etc/localtime
+RUN chown -R _shibd:_shibd /etc/shibboleth/
+RUN chown -R _shibd:_shibd /var/cache/shibboleth/
+
+RUN ln -s /opt/bin/httpd-shib-foreground /usr/local/bin && ln -s /opt/etc/httpd/conf.d/ssl.conf /etc/apache2/conf-enabled/ssl.conf && ln -s /opt/etc/httpd/conf.d/virt.conf /etc/apache2/conf-enabled/virt.conf && ln -s /usr/share/zoneinfo/America/Detroit /etc/localtime
+RUN cp /etc/apache2/mods-available/ssl.load /etc/apache2/mods-enabled && cp /etc/apache2/mods-available/socache_shmcb.load /etc/apache2/mods-enabled && cp /etc/apache2/mods-available/rewrite.load /etc/apache2/mods-enabled
 
-# KPMP Specific Configuration - MAKE EDITS HERE
-COPY ./certs/inc-md-cert.pem /etc/pki/tls/certs/inc-md-cert.pem
+RUN curl -k https://ds.incommon.org/certs/inc-md-cert.pem -o /etc/ssl/certs/inc-md-cert.pem
 COPY ./container_files/etc/shibboleth/attribute-map.xml /etc/shibboleth/attribute-map.xml
 
 RUN echo "************** Built Apache WITH Shibboleth **************"

images/apache/apache-dl-proxy/container_files/bin/httpd-shib-foreground

@@ -4,4 +4,4 @@ set -e
 # Apache gets grumpy about PID files pre-existing
 rm -f /etc/httpd/logs/httpd.pid
 
-(/usr/sbin/shibd) & httpd -DFOREGROUND
+/etc/init.d/shibd start & exec apache2ctl -D FOREGROUND

images/apache/apache-dl-proxy/container_files/etc/httpd/conf.d/ssl.conf

@@ -1,4 +1,4 @@
-SSLPassPhraseDialog exec:/usr/libexec/httpd-ssl-pass-dialog
+SSLPassPhraseDialog builtin
 SSLSessionCache         shmcb:/run/httpd/sslcache(512000)
 SSLSessionCacheTimeout  300
 SSLRandomSeed startup file:/dev/urandom  256

images/apache/apache-dl-proxy/container_files/etc/httpd/conf.d/virt.conf

@@ -1,5 +1,5 @@
-NameVirtualHost *:443
-Listen 443
+#NameVirtualHost *:443
+#Listen 443
 # qa-upload
 <VirtualHost *:80>
 
@@ -28,7 +28,7 @@ Listen 443
 
   <Location "/">
     AuthType shibboleth
-    ShibRequireSession On
+    ShibRequestSetting requireSession 1
     ShibUseHeaders On
     Require valid-user
   </Location>
@@ -82,14 +82,14 @@ Listen 443
 
   <Location "/">
     AuthType shibboleth
-    ShibRequireSession On
+    ShibRequestSetting requireSession 1
     ShibUseHeaders On
     Require valid-user
   </Location>
 
   <Location "/api/v1/redcap">
     AuthType shibboleth
-    ShibRequireSession Off
+    ShibRequestSetting requireSession 0
     require shibboleth
   </Location>
 
@@ -137,14 +137,14 @@ Listen 443
 
   <Location "/">
     AuthType shibboleth
-    ShibRequireSession On
+    ShibRequestSetting requireSession 1
     ShibUseHeaders On
     Require valid-user
   </Location>
 
   <Location "/api/v1/redcap">
     AuthType shibboleth
-    ShibRequireSession Off
+    ShibRequestSetting requireSession 0
     require shibboleth
   </Location>
 
@@ -199,7 +199,7 @@ Listen 443
 
   <Location "/">
     AuthType shibboleth
-    ShibRequireSession On
+    ShibRequestSetting requireSession 1
     ShibUseHeaders On
     Require valid-user
   </Location>
@@ -251,7 +251,7 @@ Listen 443
 
   <Location "/">
     AuthType shibboleth
-    ShibRequireSession On
+    ShibRequestSetting requireSession 1
     ShibUseHeaders On
     Require valid-user
   </Location>
@@ -303,7 +303,7 @@ Listen 443
 
   <Location "/">
     AuthType shibboleth
-    ShibRequireSession On
+    ShibRequestSetting requireSession 1
     ShibUseHeaders On
     Require valid-user
   </Location>

libra/.env.example

@@ -23,6 +23,7 @@ ENV_SHIB_CONF_DIR=/path/to/shib/conf
 
 # DMD Stuff
 ENV_DMD_SERVICE_CONTAINER=kingstonduo/data-management:1.0
+ENV_DLU_WATCHER_CONTAINER=
 mysql_user=
 mysql_pwd==
 mysql_host=mariadb
@@ -37,3 +38,4 @@ spectrack_base_url=
 # Globus and Data Lake filesystem mounts
 ENV_DATALAKE_FILE_DIR=
 ENV_GLOBUS_FILE_MOUNT=
+INSIDE_DOCKER=true

libra/docker-compose.dev.yml

@@ -42,6 +42,30 @@ services:
         aliases:
           - data-manager-service
 
+  dlu-watcher-service:
+      container_name: dlu-watcher
+      image: ${ENV_DLU_WATCHER_CONTAINER}
+      environment:
+        - mysql_user=${mysql_user}
+        - mysql_pwd=${mysql_pwd}
+        - mysql_host=${mysql_host}
+        - mysql_port=${mysql_port}
+        - mysql_db=${mysql_db}
+        - mongo_host=${mongo_host}
+        - mongo_port=${mongo_port}
+        - mongo_db=${mongo_db}
+        - globus_data_directory=${ENV_GLOBUS_FILE_MOUNT}
+        - dlu_data_directory=${ENV_DATALAKE_FILE_DIR}
+        - spectrack_token=${spectrack_token}
+        - spectrack_base_url=${spectrack_base_url}
+        - INSIDE_DOCKER=${INSIDE_DOCKER}
+      volumes:
+        - "${ENV_GLOBUS_FILE_MOUNT}:/globus"
+        - "${ENV_DATALAKE_FILE_DIR}:/data"
+      networks:
+        dataLake:
+
+
 volumes:
   esdata:
 

libra/docker-compose.local.yml

@@ -1,9 +1,9 @@
 version: "3"
 services:
-   data-manager-service:
+  data-manager-service:
     container_name: data-manager-service
     image: ${ENV_DMD_SERVICE_CONTAINER}
-    entrypoint: ["flask", "run"]
+    entrypoint: [ "flask", "run" ]
     expose:
       - 5000
     environment:
@@ -19,15 +19,38 @@ services:
       - spectrack_base_url=${spectrack_base_url}
     volumes:
       - "${ENV_DATALAKE_FILE_DIR}:/data"
-      - "${ENV_GLOBUS_FILE_MOUNT}:/globus"      
+      - "${ENV_GLOBUS_FILE_MOUNT}:/globus"
     networks:
       dataLake:
         aliases:
           - data-manager-service
 
+  dlu-watcher-service:
+    container_name: dlu-watcher
+    image: ${ENV_DLU_WATCHER_CONTAINER}
+    environment:
+      - mysql_user=${mysql_user}
+      - mysql_pwd=${mysql_pwd}
+      - mysql_host=${mysql_host}
+      - mysql_port=${mysql_port}
+      - mysql_db=${mysql_db}
+      - mongo_host=${mongo_host}
+      - mongo_port=${mongo_port}
+      - mongo_db=${mongo_db}
+      - spectrack_token=${spectrack_token}
+      - spectrack_base_url=${spectrack_base_url}
+      - INSIDE_DOCKER=${INSIDE_DOCKER}
+    volumes:
+      - "${ENV_DATALAKE_FILE_DIR}:/data"
+      - "${ENV_GLOBUS_FILE_MOUNT}:/globus"
+    networks:
+      dataLake:
+
+
 volumes:
   esdata:
 
+
 networks:
   dataLake:
     external: true