From 7d5f85330fdaf4bd217e5ffa42a9999800e2614d Mon Sep 17 00:00:00 2001 From: tmaeno Date: Tue, 5 Jul 2022 13:58:10 +0200 Subject: [PATCH 01/17] Dockerfile --- Dockerfile | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5bfccae2..06aca6dc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -47,6 +47,8 @@ RUN chown atlpan -R /opt/idds # RUN chown atlpan -R /opt/idds_source RUN chown atlpan /var/log/idds RUN chown apache -R /var/idds/wsgisocks/ +RUN chmod -R 777 /var/log/idds +RUN chmod -R 777 /var/idds # setup conda virtual env ADD requirements.yaml /opt/idds/ @@ -65,10 +67,30 @@ RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip ins RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip install --no-cache-dir --upgrade requests SQLAlchemy urllib3 retrying mod_wsgi flask futures stomp.py cx-Oracle unittest2 pep8 flake8 pytest nose sphinx recommonmark sphinx-rtd-theme nevergrad RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip install --no-cache-dir --upgrade psycopg2-binary RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip install --no-cache-dir --upgrade rucio-clients-atlas rucio-clients panda-client -RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip install --no-cache-dir --upgrade idds-common==$TAG idds-workflow==$TAG idds-server==$TAG idds-client==$TAG idds-doma==$TAG idds-atlas==$TAG idds-website==$TAG idds-monitor==$TAG + +WORKDIR /tmp/src +COPY . . +RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; \ + if [[ -z "$TAG" ]] ; then \ + cd - && cd common && python3 setup.py sdist; python3 -m pip install `ls dist/i*.tar.gz` && \ + cd - && cd workflow && python3 setup.py sdist; python3 -m pip install `ls dist/i*.tar.gz` && \ + cd - && cd server && python3 setup.py sdist; python3 -m pip install `ls dist/i*.tar.gz` && \ + cd - && cd client && python3 setup.py sdist; python3 -m pip install `ls dist/i*.tar.gz` && \ + cd - && cd doma && python3 setup.py sdist; python3 -m pip install `ls dist/i*.tar.gz` && \ + cd - && cd atlas && python3 setup.py sdist; python3 -m pip install `ls dist/i*.tar.gz` && \ + cd - && cd website && python3 setup.py sdist; python3 -m pip install `ls dist/i*.tar.gz` && \ + cd - && cd monitor && python3 setup.py sdist; python3 -m pip install `ls dist/i*.tar.gz` && \ + cd - ; \ + else \ + python3 -m pip install --no-cache-dir --upgrade idds-common==$TAG idds-workflow==$TAG idds-server==$TAG idds-client==$TAG idds-doma==$TAG idds-atlas==$TAG idds-website==$TAG idds-monitor==$TAG ; \ + fi + +WORKDIR /tmp/src +RUN rm -rf /tmp/src RUN mkdir /opt/idds/config RUN mkdir /opt/idds/config/idds +RUN chmod -R 777 /opt/idds/config # RUN mkdir /opt/idds/config_default # ADD idds.cfg.default /opt/idds/config @@ -77,6 +99,8 @@ RUN mkdir /opt/idds/config/idds # RUN ls /opt/idds/config; ls /opt/idds/config/idds; # for rest service +RUN chmod -r 777 /etc/grid-security + # RUN ln -fs /opt/idds/config/hostkey.pem /etc/grid-security/hostkey.pem # RUN ln -fs /opt/idds/config/hostcert.pem /etc/grid-security/hostcert.pem From c100997608932117736c299b3999ad1ef0832fff Mon Sep 17 00:00:00 2001 From: tmaeno Date: Tue, 5 Jul 2022 14:13:35 +0200 Subject: [PATCH 02/17] hpo doc --- Dockerfile | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 06aca6dc..26155589 100644 --- a/Dockerfile +++ b/Dockerfile @@ -72,15 +72,15 @@ WORKDIR /tmp/src COPY . . RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; \ if [[ -z "$TAG" ]] ; then \ - cd - && cd common && python3 setup.py sdist; python3 -m pip install `ls dist/i*.tar.gz` && \ - cd - && cd workflow && python3 setup.py sdist; python3 -m pip install `ls dist/i*.tar.gz` && \ - cd - && cd server && python3 setup.py sdist; python3 -m pip install `ls dist/i*.tar.gz` && \ - cd - && cd client && python3 setup.py sdist; python3 -m pip install `ls dist/i*.tar.gz` && \ - cd - && cd doma && python3 setup.py sdist; python3 -m pip install `ls dist/i*.tar.gz` && \ - cd - && cd atlas && python3 setup.py sdist; python3 -m pip install `ls dist/i*.tar.gz` && \ - cd - && cd website && python3 setup.py sdist; python3 -m pip install `ls dist/i*.tar.gz` && \ - cd - && cd monitor && python3 setup.py sdist; python3 -m pip install `ls dist/i*.tar.gz` && \ - cd - ; \ + python3 setup.py sdist && \ + python3 -m pip install `ls common/dist/*.tar.gz` && \ + python3 -m pip install `ls workflow/dist/*.tar.gz` && \ + python3 -m pip install `ls server/dist/*.tar.gz` && \ + python3 -m pip install `ls client/dist/*.tar.gz` && \ + python3 -m pip install `ls doma/dist/*.tar.gz` && \ + python3 -m pip install `ls atlas/dist/*.tar.gz` && \ + python3 -m pip install `ls website/dist/*.tar.gz` && \ + python3 -m pip install `ls monitor/dist/*.tar.gz` ; \ else \ python3 -m pip install --no-cache-dir --upgrade idds-common==$TAG idds-workflow==$TAG idds-server==$TAG idds-client==$TAG idds-doma==$TAG idds-atlas==$TAG idds-website==$TAG idds-monitor==$TAG ; \ fi From 1bd3831f09740c5638e8e4df63c1306fa3b42247 Mon Sep 17 00:00:00 2001 From: tmaeno Date: Tue, 5 Jul 2022 14:20:14 +0200 Subject: [PATCH 03/17] hpo doc --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 26155589..b706ae0a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -85,7 +85,7 @@ RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; \ python3 -m pip install --no-cache-dir --upgrade idds-common==$TAG idds-workflow==$TAG idds-server==$TAG idds-client==$TAG idds-doma==$TAG idds-atlas==$TAG idds-website==$TAG idds-monitor==$TAG ; \ fi -WORKDIR /tmp/src +WORKDIR /tmp RUN rm -rf /tmp/src RUN mkdir /opt/idds/config From 0c1b356c14ef7920562d2ac1d72c19e186cb9cf0 Mon Sep 17 00:00:00 2001 From: tmaeno Date: Tue, 5 Jul 2022 14:30:37 +0200 Subject: [PATCH 04/17] hpo doc --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b706ae0a..91f7f8da 100644 --- a/Dockerfile +++ b/Dockerfile @@ -75,7 +75,7 @@ RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; \ python3 setup.py sdist && \ python3 -m pip install `ls common/dist/*.tar.gz` && \ python3 -m pip install `ls workflow/dist/*.tar.gz` && \ - python3 -m pip install `ls server/dist/*.tar.gz` && \ + python3 -m pip install `ls main/dist/*.tar.gz` && \ python3 -m pip install `ls client/dist/*.tar.gz` && \ python3 -m pip install `ls doma/dist/*.tar.gz` && \ python3 -m pip install `ls atlas/dist/*.tar.gz` && \ From b64aa957e6a5049833f26a1b0bdeca52f3d203ad Mon Sep 17 00:00:00 2001 From: tmaeno Date: Tue, 5 Jul 2022 14:35:41 +0200 Subject: [PATCH 05/17] Dockerfile --- Dockerfile | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 91f7f8da..cecb87b9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -73,14 +73,10 @@ COPY . . RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; \ if [[ -z "$TAG" ]] ; then \ python3 setup.py sdist && \ - python3 -m pip install `ls common/dist/*.tar.gz` && \ - python3 -m pip install `ls workflow/dist/*.tar.gz` && \ - python3 -m pip install `ls main/dist/*.tar.gz` && \ - python3 -m pip install `ls client/dist/*.tar.gz` && \ - python3 -m pip install `ls doma/dist/*.tar.gz` && \ - python3 -m pip install `ls atlas/dist/*.tar.gz` && \ - python3 -m pip install `ls website/dist/*.tar.gz` && \ - python3 -m pip install `ls monitor/dist/*.tar.gz` ; \ + for package in common main client workflow doma atlas website monitor ; \ + do \ + python3 -m pip install `ls $package/dist/*.tar.gz` \ + done ; \ else \ python3 -m pip install --no-cache-dir --upgrade idds-common==$TAG idds-workflow==$TAG idds-server==$TAG idds-client==$TAG idds-doma==$TAG idds-atlas==$TAG idds-website==$TAG idds-monitor==$TAG ; \ fi From fdf4efa66ed994e74fecd7918e791ce246a1e2d7 Mon Sep 17 00:00:00 2001 From: tmaeno Date: Tue, 5 Jul 2022 15:21:16 +0200 Subject: [PATCH 06/17] Dockerfile --- Dockerfile | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index cecb87b9..1be307c5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -70,13 +70,17 @@ RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip ins WORKDIR /tmp/src COPY . . + +RUN echo $'#!/bin/bash \n\ +set -m \n\ +for package in common main client workflow doma atlas website monitor ; \n\ +do \n\ + python3 -m pip install `ls $package/dist/*.tar.gz` \n\ +done \n ' > inst_packages.sh + RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; \ if [[ -z "$TAG" ]] ; then \ - python3 setup.py sdist && \ - for package in common main client workflow doma atlas website monitor ; \ - do \ - python3 -m pip install `ls $package/dist/*.tar.gz` \ - done ; \ + python3 setup.py sdist && chmod +x inst_packages.sh && ./inst_packages.sh ; \ else \ python3 -m pip install --no-cache-dir --upgrade idds-common==$TAG idds-workflow==$TAG idds-server==$TAG idds-client==$TAG idds-doma==$TAG idds-atlas==$TAG idds-website==$TAG idds-monitor==$TAG ; \ fi From dab5d60fceaddd30eae88b8f6dad5307583fe809 Mon Sep 17 00:00:00 2001 From: tmaeno Date: Tue, 5 Jul 2022 15:31:53 +0200 Subject: [PATCH 07/17] Dockerfile --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1be307c5..77a068d7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -99,7 +99,8 @@ RUN chmod -R 777 /opt/idds/config # RUN ls /opt/idds/config; ls /opt/idds/config/idds; # for rest service -RUN chmod -r 777 /etc/grid-security + +RUN mkdir /etc/grid-security && chmod -r 777 /etc/grid-security # RUN ln -fs /opt/idds/config/hostkey.pem /etc/grid-security/hostkey.pem # RUN ln -fs /opt/idds/config/hostcert.pem /etc/grid-security/hostcert.pem From 9dac15d477f829017f5f5d5220631f2c29dff16e Mon Sep 17 00:00:00 2001 From: tmaeno Date: Tue, 5 Jul 2022 15:40:41 +0200 Subject: [PATCH 08/17] Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 77a068d7..7ea9eccd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -100,7 +100,7 @@ RUN chmod -R 777 /opt/idds/config # for rest service -RUN mkdir /etc/grid-security && chmod -r 777 /etc/grid-security +RUN chmod -R 777 /etc/grid-security # RUN ln -fs /opt/idds/config/hostkey.pem /etc/grid-security/hostkey.pem # RUN ln -fs /opt/idds/config/hostcert.pem /etc/grid-security/hostcert.pem From a334ce2c3ab4fc61e42d89fb2a84bf9d371de36a Mon Sep 17 00:00:00 2001 From: tmaeno Date: Tue, 5 Jul 2022 15:57:45 +0200 Subject: [PATCH 09/17] Dockerfile --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 7ea9eccd..fa92a382 100644 --- a/Dockerfile +++ b/Dockerfile @@ -49,6 +49,8 @@ RUN chown atlpan /var/log/idds RUN chown apache -R /var/idds/wsgisocks/ RUN chmod -R 777 /var/log/idds RUN chmod -R 777 /var/idds +RUN chmod -R 777 /etc/httpd/conf.d +RUN chmod -R 777 /run/httpd # setup conda virtual env ADD requirements.yaml /opt/idds/ From ae3e926fb6e2ec4893e6cecf40356cabffeb4132 Mon Sep 17 00:00:00 2001 From: tmaeno Date: Tue, 5 Jul 2022 16:24:31 +0200 Subject: [PATCH 10/17] Dockerfile --- Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Dockerfile b/Dockerfile index fa92a382..ade29b5a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -51,6 +51,7 @@ RUN chmod -R 777 /var/log/idds RUN chmod -R 777 /var/idds RUN chmod -R 777 /etc/httpd/conf.d RUN chmod -R 777 /run/httpd +RUN chmod -R 777 /var/log/httpd # setup conda virtual env ADD requirements.yaml /opt/idds/ @@ -104,6 +105,9 @@ RUN chmod -R 777 /opt/idds/config RUN chmod -R 777 /etc/grid-security +# to grant low-numbered port to non-root +RUN setcap CAP_NET_BIND_SERVICE=+eip /usr/sbin/httpd + # RUN ln -fs /opt/idds/config/hostkey.pem /etc/grid-security/hostkey.pem # RUN ln -fs /opt/idds/config/hostcert.pem /etc/grid-security/hostcert.pem From fc821fdc9074e1b962c9b76122428ff4cbbf0115 Mon Sep 17 00:00:00 2001 From: tmaeno Date: Tue, 5 Jul 2022 16:50:04 +0200 Subject: [PATCH 11/17] Dockerfile --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index ade29b5a..b2ab8353 100644 --- a/Dockerfile +++ b/Dockerfile @@ -107,6 +107,7 @@ RUN chmod -R 777 /etc/grid-security # to grant low-numbered port to non-root RUN setcap CAP_NET_BIND_SERVICE=+eip /usr/sbin/httpd +RUN chmod a+r /etc/pki/tls/certs/localhost.crt # RUN ln -fs /opt/idds/config/hostkey.pem /etc/grid-security/hostkey.pem # RUN ln -fs /opt/idds/config/hostcert.pem /etc/grid-security/hostcert.pem From ad6734c918aecf471454ba75d6e962c1ddca00df Mon Sep 17 00:00:00 2001 From: tmaeno Date: Tue, 5 Jul 2022 17:02:54 +0200 Subject: [PATCH 12/17] Dockerfile --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index b2ab8353..4892bfb2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -108,6 +108,7 @@ RUN chmod -R 777 /etc/grid-security # to grant low-numbered port to non-root RUN setcap CAP_NET_BIND_SERVICE=+eip /usr/sbin/httpd RUN chmod a+r /etc/pki/tls/certs/localhost.crt +RUN chmod a+r /etc/pki/tls/private/localhost.key # RUN ln -fs /opt/idds/config/hostkey.pem /etc/grid-security/hostkey.pem # RUN ln -fs /opt/idds/config/hostcert.pem /etc/grid-security/hostcert.pem From 4a00321971254f3f8ad52116a2d5a0312dac16c3 Mon Sep 17 00:00:00 2001 From: tmaeno Date: Tue, 5 Jul 2022 17:06:50 +0200 Subject: [PATCH 13/17] Dockerfile --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4892bfb2..fe1285cc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -107,8 +107,7 @@ RUN chmod -R 777 /etc/grid-security # to grant low-numbered port to non-root RUN setcap CAP_NET_BIND_SERVICE=+eip /usr/sbin/httpd -RUN chmod a+r /etc/pki/tls/certs/localhost.crt -RUN chmod a+r /etc/pki/tls/private/localhost.key +RUN rm /etc/httpd/conf.d/ssl.conf # RUN ln -fs /opt/idds/config/hostkey.pem /etc/grid-security/hostkey.pem # RUN ln -fs /opt/idds/config/hostcert.pem /etc/grid-security/hostcert.pem From d6f37ed4d1b55ab5eab257a92164b648de1760d7 Mon Sep 17 00:00:00 2001 From: tmaeno Date: Tue, 5 Jul 2022 17:20:49 +0200 Subject: [PATCH 14/17] Dockerfile --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index fe1285cc..4892bfb2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -107,7 +107,8 @@ RUN chmod -R 777 /etc/grid-security # to grant low-numbered port to non-root RUN setcap CAP_NET_BIND_SERVICE=+eip /usr/sbin/httpd -RUN rm /etc/httpd/conf.d/ssl.conf +RUN chmod a+r /etc/pki/tls/certs/localhost.crt +RUN chmod a+r /etc/pki/tls/private/localhost.key # RUN ln -fs /opt/idds/config/hostkey.pem /etc/grid-security/hostkey.pem # RUN ln -fs /opt/idds/config/hostcert.pem /etc/grid-security/hostcert.pem From d15b420c6eb60033bb87a3eb9aa468afa89d8a5f Mon Sep 17 00:00:00 2001 From: tmaeno Date: Tue, 5 Jul 2022 17:31:49 +0200 Subject: [PATCH 15/17] Dockerfile --- Dockerfile | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4892bfb2..9c477215 100644 --- a/Dockerfile +++ b/Dockerfile @@ -47,6 +47,8 @@ RUN chown atlpan -R /opt/idds # RUN chown atlpan -R /opt/idds_source RUN chown atlpan /var/log/idds RUN chown apache -R /var/idds/wsgisocks/ + +# to run with non-root PID RUN chmod -R 777 /var/log/idds RUN chmod -R 777 /var/idds RUN chmod -R 777 /etc/httpd/conf.d @@ -93,7 +95,10 @@ RUN rm -rf /tmp/src RUN mkdir /opt/idds/config RUN mkdir /opt/idds/config/idds + +# to run with non-root PID RUN chmod -R 777 /opt/idds/config + # RUN mkdir /opt/idds/config_default # ADD idds.cfg.default /opt/idds/config @@ -103,10 +108,11 @@ RUN chmod -R 777 /opt/idds/config # for rest service +# to grant low-numbered port access to non-root +RUN setcap CAP_NET_BIND_SERVICE=+eip /usr/sbin/httpd RUN chmod -R 777 /etc/grid-security -# to grant low-numbered port to non-root -RUN setcap CAP_NET_BIND_SERVICE=+eip /usr/sbin/httpd +# required for ssl.conf to run with non-root PID RUN chmod a+r /etc/pki/tls/certs/localhost.crt RUN chmod a+r /etc/pki/tls/private/localhost.key From 573147ed4162d8408de0364a7952fd4373777e1b Mon Sep 17 00:00:00 2001 From: tmaeno Date: Tue, 5 Jul 2022 17:57:52 +0200 Subject: [PATCH 16/17] Dockerfile --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9c477215..da634d18 100644 --- a/Dockerfile +++ b/Dockerfile @@ -95,11 +95,11 @@ RUN rm -rf /tmp/src RUN mkdir /opt/idds/config RUN mkdir /opt/idds/config/idds +RUN mkdir /opt/idds/config_default # to run with non-root PID RUN chmod -R 777 /opt/idds/config - -# RUN mkdir /opt/idds/config_default +RUN chmod -R 777 /opt/idds/config_default # ADD idds.cfg.default /opt/idds/config From e07880e622c2395b7937071494a1dd553e6bcccb Mon Sep 17 00:00:00 2001 From: tmaeno Date: Tue, 5 Jul 2022 18:06:09 +0200 Subject: [PATCH 17/17] Dockerfile --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index da634d18..34d021f6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -95,7 +95,6 @@ RUN rm -rf /tmp/src RUN mkdir /opt/idds/config RUN mkdir /opt/idds/config/idds -RUN mkdir /opt/idds/config_default # to run with non-root PID RUN chmod -R 777 /opt/idds/config