From ab3f088564ec434ee0a5f973e7ca22500a1f1f10 Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Wed, 22 Nov 2023 10:40:32 +0100 Subject: [PATCH 01/74] revamp CI --- .github/workflows/dockerimage.yml | 60 +++++++++++++++++++++++++------ .travis.yml | 20 ----------- CHANGES.txt => CHANGES.md | 0 Dockerfile | 4 +-- 4 files changed, 51 insertions(+), 33 deletions(-) delete mode 100644 .travis.yml rename CHANGES.txt => CHANGES.md (100%) diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml index 5181ad7..8625e2e 100644 --- a/.github/workflows/dockerimage.yml +++ b/.github/workflows/dockerimage.yml @@ -1,19 +1,57 @@ name: Docker Image CI -on: [push] +on: ["push", "pull_request"] jobs: - build: - + lint: runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v1 - - name: Build the Docker image + - uses: actions/checkout@v4 + - uses: hadolint/hadolint-action@v3.1.0 + with: + dockerfile: Dockerfile + + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Build and test the Docker image run: | - docker build . --file Dockerfile --tag docker.pkg.github.com/genouest/biomaj-docker/biomaj:$GITHUB_SHA - docker login docker.pkg.github.com -u ${{ secrets.GHR_USER }} -p ${{ secrets.GHR_TOKEN }} - docker push docker.pkg.github.com/genouest/biomaj-docker/biomaj:$GITHUB_SHA - - + echo "BIOMAJ_DIR=$(pwd)" > .env + echo "BIOMAJ_DATA_DIR=$(pwd)/biomaj" >> .env + mkdir -p biomaj biomaj/conf biomaj/log biomaj/lock biomaj/db biomaj/process biomaj/cache + cp test-local/etc/biomaj/conf.d/alu.properties biomaj/conf/ + cp test-local/etc/biomaj/conf.d/local.properties biomaj/conf/ + docker-compose config --q + docker build -t quay.io/genouest/biomaj . + docker-compose up -d + sleep 20 + ./docker-test.sh local + docker-compose down + + publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.GHR_USER }} + password: ${{ secrets.GHR_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: genouest/biomaj-docker + + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: . + file: ./Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 10d96e4..0000000 --- a/.travis.yml +++ /dev/null @@ -1,20 +0,0 @@ -sudo: required - -services: - - docker - -before_install: - - echo "BIOMAJ_DIR=$(pwd)" > .env - - echo "BIOMAJ_DATA_DIR=$(pwd)/biomaj" >> .env - - mkdir -p biomaj biomaj/conf biomaj/log biomaj/lock biomaj/db biomaj/process biomaj/cache - - cp test-local/etc/biomaj/conf.d/alu.properties biomaj/conf/ - - cp test-local/etc/biomaj/conf.d/local.properties biomaj/conf/ - -script: - - docker run --rm -i hadolint/hadolint < Dockerfile || true - - docker-compose config --q - - docker build -t quay.io/osallou/biomaj . - - docker-compose up -d - - sleep 20 - - ./docker-test.sh local - - docker-compose down diff --git a/CHANGES.txt b/CHANGES.md similarity index 100% rename from CHANGES.txt rename to CHANGES.md diff --git a/Dockerfile b/Dockerfile index ba7179d..f087318 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM quay.io/osallou/debian:buster +FROM debian:buster WORKDIR /root ENV BIOMAJ_CONFIG=/root/config.yml @@ -102,7 +102,7 @@ RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86 /opt/conda/bin/conda config --add channels bioconda && \ /opt/conda/bin/conda upgrade -y conda && \ chmod 777 -R /opt/conda/ - + From 520ff59fa5df050f9ac9b317f4467e7eeeb2710b Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Wed, 22 Nov 2023 12:17:38 +0100 Subject: [PATCH 02/74] lint --- Dockerfile | 118 ++++++++++++++++++++++++----------------------------- 1 file changed, 53 insertions(+), 65 deletions(-) diff --git a/Dockerfile b/Dockerfile index f087318..2d9c6d5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,83 +4,71 @@ WORKDIR /root ENV BIOMAJ_CONFIG=/root/config.yml ENV prometheus_multiproc_dir=/tmp/biomaj-prometheus-multiproc -RUN rm -rf /tmp/biomaj-prometheus-multiproc -RUN mkdir -p /tmp/biomaj-prometheus-multiproc +RUN rm -rf /tmp/biomaj-prometheus-multiproc && \ + mkdir -p /tmp/biomaj-prometheus-multiproc RUN apt-get update && apt-get install -y apt-transport-https curl libcurl4-openssl-dev python3-pycurl python3-setuptools python3-pip git unzip bzip2 ca-certificates jq --no-install-recommends # Install docker to allow docker execution from process-message RUN buildDeps='gnupg2 dirmngr software-properties-common' \ && set -x \ - && apt-get install -y $buildDeps --no-install-recommends \ + && apt-get install -y "$buildDeps" --no-install-recommends \ && curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - \ && add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable" \ && apt-get update \ - && apt-get install -y docker-ce-cli \ - && apt-get purge -y --auto-remove $buildDeps + && apt-get install --no-install-recommends -y docker-ce-cli \ + && apt-get purge -y --auto-remove "$buildDeps" ENV BIOMAJ_RELEASE="shahikorma-v13" -RUN git clone https://github.com/genouest/biomaj-core.git -#RUN easy_install3 pip -RUN pip3 install setuptools --upgrade - -RUN git clone https://github.com/genouest/biomaj-zipkin.git - -RUN git clone https://github.com/genouest/biomaj-user.git - -RUN git clone https://github.com/genouest/biomaj-cli.git - -RUN git clone https://github.com/genouest/biomaj-process.git - -RUN git clone https://github.com/genouest/biomaj-download.git - -RUN git clone https://github.com/genouest/biomaj.git && echo "Install biomaj" - -RUN git clone https://github.com/genouest/biomaj-daemon.git && echo "Install daemon" - -RUN git clone https://github.com/genouest/biomaj-watcher.git && echo "Install biomaj-watcher" - -RUN git clone https://github.com/genouest/biomaj-ftp.git - -RUN git clone https://github.com/genouest/biomaj-release.git - -RUN git clone https://github.com/genouest/biomaj-data.git +RUN git clone https://github.com/genouest/biomaj-core.git && \ + git clone https://github.com/genouest/biomaj-zipkin.git && \ + git clone https://github.com/genouest/biomaj-user.git && \ + git clone https://github.com/genouest/biomaj-cli.git && \ + git clone https://github.com/genouest/biomaj-process.git && \ + git clone https://github.com/genouest/biomaj-download.git && \ + git clone https://github.com/genouest/biomaj.git && \ + git clone https://github.com/genouest/biomaj-daemon.git && \ + git clone https://github.com/genouest/biomaj-watcher.git && \ + git clone https://github.com/genouest/biomaj-ftp.git && \ + git clone https://github.com/genouest/biomaj-release.git && \ + git clone https://github.com/genouest/biomaj-data.git ENV BIOMAJ_CONFIG=/etc/biomaj/config.yml RUN mkdir -p /var/log/biomaj -RUN pip3 install greenlet==0.4.17 -RUN pip3 install gevent==1.4.0 -RUN pip3 install graypy -RUN pip3 install pymongo==3.12.3 -RUN pip3 install redis==3.5.3 -RUN pip3 install wheel -RUN pip3 install PyYAML==5.3.1 +RUN pip3 install setuptools --upgrade && \ + pip3 install --no-cache-dir greenlet==0.4.17 && \ + pip3 install --no-cache-dir gevent==1.4.0 && \ + pip3 install --no-cache-dir graypy && \ + pip3 install --no-cache-dir pymongo==3.12.3 && \ + pip3 install --no-cache-dir redis==3.5.3 && \ + pip3 install --no-cache-dir wheel && \ + pip3 install --no-cache-dir PyYAML==5.3.1 ENV SUDO_FORCE_REMOVE=yes -RUN buildDeps='gcc python3-dev protobuf-compiler' \ +RUN buildDeps="gcc python3-dev protobuf-compiler" \ && set -x \ - && apt-get install -y $buildDeps --no-install-recommends \ - && cd /root/biomaj-core && python3 setup.py build && pip3 install . \ - && cd /root/biomaj-zipkin && python3 setup.py build && pip3 install . \ - && cd /root/biomaj-user && python3 setup.py build && pip3 install . \ - && cd /root/biomaj-cli && python3 setup.py build && pip3 install . \ + && apt-get install -y "$buildDeps" --no-install-recommends \ + && cd /root/biomaj-core && python3 setup.py build && pip3 install --no-cache-dir . \ + && cd /root/biomaj-zipkin && python3 setup.py build && pip3 install --no-cache-dir . \ + && cd /root/biomaj-user && python3 setup.py build && pip3 install --no-cache-dir . \ + && cd /root/biomaj-cli && python3 setup.py build && pip3 install --no-cache-dir . \ && cd /root/biomaj-process/biomaj_process/message && protoc --python_out=. procmessage.proto \ - && cd /root/biomaj-process && python3 setup.py build && pip3 install . \ + && cd /root/biomaj-process && python3 setup.py build && pip3 install --no-cache-dir . \ && cd /root/biomaj-download/biomaj_download/message && protoc --python_out=. downmessage.proto \ - && cd /root/biomaj-download && python3 setup.py build && pip3 install . \ - && cd /root/biomaj && python3 setup.py build && pip3 install . \ - && cd /root/biomaj-daemon && python3 setup.py build && pip3 install . \ - && cd /root/biomaj-watcher && pip3 install -r requirements.txt && pip3 install . \ - && cd /root/biomaj-ftp && python3 setup.py build && pip3 install . \ - && cd /root/biomaj-release && python3 setup.py build && pip3 install . \ - && cd /root/biomaj-data && python3 setup.py build && pip3 install . \ - && apt-get install -y wget bzip2 ca-certificates curl git \ + && cd /root/biomaj-download && python3 setup.py build && pip3 install --no-cache-dir . \ + && cd /root/biomaj && python3 setup.py build && pip3 install --no-cache-dir . \ + && cd /root/biomaj-daemon && python3 setup.py build && pip3 install --no-cache-dir . \ + && cd /root/biomaj-watcher && pip3 install --no-cache-dir -r requirements.txt && pip3 install --no-cache-dir . \ + && cd /root/biomaj-ftp && python3 setup.py build && pip3 install --no-cache-dir . \ + && cd /root/biomaj-release && python3 setup.py build && pip3 install --no-cache-dir . \ + && cd /root/biomaj-data && python3 setup.py build && pip3 install --no-cache-dir . \ + && apt-get install --no-install-recommends -y wget bzip2 ca-certificates curl git \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* \ - && apt-get purge -y --auto-remove $buildDeps + && apt-get purge -y --auto-remove "$buildDeps" @@ -88,19 +76,19 @@ RUN buildDeps='gcc python3-dev protobuf-compiler' \ # apt-get install -y wget bzip2 ca-certificates curl git && \ # apt-get clean && \ # rm -rf /var/lib/apt/lists/* -RUN pip3 uninstall -y gunicorn && pip3 install gunicorn==19.9.0 -RUN pip3 uninstall -y greenlet && pip3 install greenlet==0.4.15 -RUN pip3 uninstall -y gevent && pip3 install gevent==1.3.7 +RUN pip3 uninstall -y gunicorn && pip3 install --no-cache-dir gunicorn==19.9.0 && \ + pip3 uninstall -y greenlet && pip3 install --no-cache-dir greenlet==0.4.15 && \ + pip3 uninstall -y gevent && pip3 install --no-cache-dir gevent==1.3.7 #Conda installation and give write permissions to conda folder RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh && \ /bin/bash ~/miniconda.sh -b -p /opt/conda && \ - rm ~/miniconda.sh && \ - /opt/conda/bin/conda config --add channels r && \ - /opt/conda/bin/conda config --add channels bioconda && \ - /opt/conda/bin/conda upgrade -y conda && \ + rm ~/miniconda.sh && \ + /opt/conda/bin/conda config --add channels r && \ + /opt/conda/bin/conda config --add channels bioconda && \ + /opt/conda/bin/conda upgrade -y conda && \ chmod 777 -R /opt/conda/ @@ -125,11 +113,11 @@ COPY biomaj-config/gunicorn_conf.py /etc/biomaj/gunicorn_conf.py COPY watcher.sh /root/watcher.sh # Local test configuration -RUN mkdir -p /etc/biomaj/conf.d -RUN mkdir -p /var/log/biomaj -RUN mkdir -p /etc/biomaj/process.d -RUN mkdir -p /var/cache/biomaj -RUN mkdir -p /var/run/biomaj +RUN mkdir -p /etc/biomaj/conf.d && \ + mkdir -p /var/log/biomaj && \ + mkdir -p /etc/biomaj/process.d && \ + mkdir -p /var/cache/biomaj && \ + mkdir -p /var/run/biomaj COPY test-local/etc/biomaj/global_local.properties /etc/biomaj/global_local.properties COPY test-local/etc/biomaj/conf.d/alu.properties /etc/biomaj/conf.d/alu.properties From 72a7cdf9d933c4b14090395871a2763e830afbef Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Fri, 24 Nov 2023 10:03:39 +0100 Subject: [PATCH 03/74] some linting --- .hadolint.yaml | 3 +++ Dockerfile | 15 ++++++++++----- 2 files changed, 13 insertions(+), 5 deletions(-) create mode 100644 .hadolint.yaml diff --git a/.hadolint.yaml b/.hadolint.yaml new file mode 100644 index 0000000..73481fe --- /dev/null +++ b/.hadolint.yaml @@ -0,0 +1,3 @@ +ignored: + - DL3008 + - DL3003 diff --git a/Dockerfile b/Dockerfile index 2d9c6d5..5e1812b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,10 @@ ENV prometheus_multiproc_dir=/tmp/biomaj-prometheus-multiproc RUN rm -rf /tmp/biomaj-prometheus-multiproc && \ mkdir -p /tmp/biomaj-prometheus-multiproc -RUN apt-get update && apt-get install -y apt-transport-https curl libcurl4-openssl-dev python3-pycurl python3-setuptools python3-pip git unzip bzip2 ca-certificates jq --no-install-recommends +RUN apt-get update \ + && apt-get install -y apt-transport-https curl libcurl4-openssl-dev python3-pycurl python3-setuptools python3-pip git unzip bzip2 ca-certificates jq --no-install-recommends \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* # Install docker to allow docker execution from process-message RUN buildDeps='gnupg2 dirmngr software-properties-common' \ @@ -17,7 +20,9 @@ RUN buildDeps='gnupg2 dirmngr software-properties-common' \ && add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable" \ && apt-get update \ && apt-get install --no-install-recommends -y docker-ce-cli \ - && apt-get purge -y --auto-remove "$buildDeps" + && apt-get purge -y --auto-remove "$buildDeps" \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* ENV BIOMAJ_RELEASE="shahikorma-v13" @@ -38,7 +43,7 @@ ENV BIOMAJ_CONFIG=/etc/biomaj/config.yml RUN mkdir -p /var/log/biomaj -RUN pip3 install setuptools --upgrade && \ +RUN pip3 install --no-cache-dir setuptools --upgrade && \ pip3 install --no-cache-dir greenlet==0.4.17 && \ pip3 install --no-cache-dir gevent==1.4.0 && \ pip3 install --no-cache-dir graypy && \ @@ -66,9 +71,9 @@ RUN buildDeps="gcc python3-dev protobuf-compiler" \ && cd /root/biomaj-release && python3 setup.py build && pip3 install --no-cache-dir . \ && cd /root/biomaj-data && python3 setup.py build && pip3 install --no-cache-dir . \ && apt-get install --no-install-recommends -y wget bzip2 ca-certificates curl git \ + && apt-get purge -y --auto-remove "$buildDeps" \ && apt-get clean \ - && rm -rf /var/lib/apt/lists/* \ - && apt-get purge -y --auto-remove "$buildDeps" + && rm -rf /var/lib/apt/lists/* From df7346d27346d127dfd1959456868c63dc015d5c Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Fri, 24 Nov 2023 10:58:22 +0100 Subject: [PATCH 04/74] lint --- .hadolint.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.hadolint.yaml b/.hadolint.yaml index 73481fe..6bf8263 100644 --- a/.hadolint.yaml +++ b/.hadolint.yaml @@ -1,3 +1,6 @@ ignored: - DL3008 - DL3003 + - DL4006 + - DL4001 + - DL3013 From a84405a36a87291ca7a8a8895eaa61fc31a0a8af Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Fri, 24 Nov 2023 11:03:16 +0100 Subject: [PATCH 05/74] fix --- .hadolint.yaml | 1 + Dockerfile | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.hadolint.yaml b/.hadolint.yaml index 6bf8263..77ebb91 100644 --- a/.hadolint.yaml +++ b/.hadolint.yaml @@ -4,3 +4,4 @@ ignored: - DL4006 - DL4001 - DL3013 + - SC2086 diff --git a/Dockerfile b/Dockerfile index 5e1812b..73660cf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,12 +15,12 @@ RUN apt-get update \ # Install docker to allow docker execution from process-message RUN buildDeps='gnupg2 dirmngr software-properties-common' \ && set -x \ - && apt-get install -y "$buildDeps" --no-install-recommends \ + && apt-get install -y $buildDeps --no-install-recommends \ && curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - \ && add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable" \ && apt-get update \ && apt-get install --no-install-recommends -y docker-ce-cli \ - && apt-get purge -y --auto-remove "$buildDeps" \ + && apt-get purge -y --auto-remove $buildDeps \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* @@ -55,7 +55,7 @@ RUN pip3 install --no-cache-dir setuptools --upgrade && \ ENV SUDO_FORCE_REMOVE=yes RUN buildDeps="gcc python3-dev protobuf-compiler" \ && set -x \ - && apt-get install -y "$buildDeps" --no-install-recommends \ + && apt-get install -y $buildDeps --no-install-recommends \ && cd /root/biomaj-core && python3 setup.py build && pip3 install --no-cache-dir . \ && cd /root/biomaj-zipkin && python3 setup.py build && pip3 install --no-cache-dir . \ && cd /root/biomaj-user && python3 setup.py build && pip3 install --no-cache-dir . \ @@ -71,7 +71,7 @@ RUN buildDeps="gcc python3-dev protobuf-compiler" \ && cd /root/biomaj-release && python3 setup.py build && pip3 install --no-cache-dir . \ && cd /root/biomaj-data && python3 setup.py build && pip3 install --no-cache-dir . \ && apt-get install --no-install-recommends -y wget bzip2 ca-certificates curl git \ - && apt-get purge -y --auto-remove "$buildDeps" \ + && apt-get purge -y --auto-remove $buildDeps \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* From e832e9d38b2aa9270e0db92ede5a040f06ea9efa Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Fri, 24 Nov 2023 11:04:51 +0100 Subject: [PATCH 06/74] single build on PR --- .github/workflows/dockerimage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml index 8625e2e..0a134c9 100644 --- a/.github/workflows/dockerimage.yml +++ b/.github/workflows/dockerimage.yml @@ -1,6 +1,6 @@ name: Docker Image CI -on: ["push", "pull_request"] +on: ["push"] jobs: From 94e82aa94ae445b9d487d88563ff071fa94e3e6e Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Fri, 24 Nov 2023 11:07:34 +0100 Subject: [PATCH 07/74] fix --- Dockerfile | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/Dockerfile b/Dockerfile index 73660cf..6db5fcb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,18 +7,13 @@ ENV prometheus_multiproc_dir=/tmp/biomaj-prometheus-multiproc RUN rm -rf /tmp/biomaj-prometheus-multiproc && \ mkdir -p /tmp/biomaj-prometheus-multiproc -RUN apt-get update \ - && apt-get install -y apt-transport-https curl libcurl4-openssl-dev python3-pycurl python3-setuptools python3-pip git unzip bzip2 ca-certificates jq --no-install-recommends \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - # Install docker to allow docker execution from process-message RUN buildDeps='gnupg2 dirmngr software-properties-common' \ - && set -x \ - && apt-get install -y $buildDeps --no-install-recommends \ - && curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - \ && add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable" \ && apt-get update \ + && apt-get install -y apt-transport-https curl libcurl4-openssl-dev python3-pycurl python3-setuptools python3-pip git unzip bzip2 ca-certificates jq --no-install-recommends \ + && apt-get install -y $buildDeps --no-install-recommends \ + && curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - \ && apt-get install --no-install-recommends -y docker-ce-cli \ && apt-get purge -y --auto-remove $buildDeps \ && apt-get clean \ @@ -55,6 +50,7 @@ RUN pip3 install --no-cache-dir setuptools --upgrade && \ ENV SUDO_FORCE_REMOVE=yes RUN buildDeps="gcc python3-dev protobuf-compiler" \ && set -x \ + && apt-get update \ && apt-get install -y $buildDeps --no-install-recommends \ && cd /root/biomaj-core && python3 setup.py build && pip3 install --no-cache-dir . \ && cd /root/biomaj-zipkin && python3 setup.py build && pip3 install --no-cache-dir . \ @@ -76,11 +72,6 @@ RUN buildDeps="gcc python3-dev protobuf-compiler" \ && rm -rf /var/lib/apt/lists/* - -#RUN apt-get update --fix-missing && \ -# apt-get install -y wget bzip2 ca-certificates curl git && \ -# apt-get clean && \ -# rm -rf /var/lib/apt/lists/* RUN pip3 uninstall -y gunicorn && pip3 install --no-cache-dir gunicorn==19.9.0 && \ pip3 uninstall -y greenlet && pip3 install --no-cache-dir greenlet==0.4.15 && \ pip3 uninstall -y gevent && pip3 install --no-cache-dir gevent==1.3.7 From 5cf9626445fd12a2370ca719059f0e41e9ac8863 Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Fri, 24 Nov 2023 11:18:25 +0100 Subject: [PATCH 08/74] fix --- Dockerfile | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6db5fcb..76176f1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ RUN rm -rf /tmp/biomaj-prometheus-multiproc && \ # Install docker to allow docker execution from process-message RUN buildDeps='gnupg2 dirmngr software-properties-common' \ - && add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable" \ + && echo 'deb [arch=amd64] https://download.docker.com/linux/debian buster stable' | sudo tee /etc/apt/sources.list.d/docker.list \ && apt-get update \ && apt-get install -y apt-transport-https curl libcurl4-openssl-dev python3-pycurl python3-setuptools python3-pip git unzip bzip2 ca-certificates jq --no-install-recommends \ && apt-get install -y $buildDeps --no-install-recommends \ @@ -94,10 +94,6 @@ RUN mkdir /data /config ENV PATH=$PATH:/opt/conda/bin -#RUN conda config --add channels r -#RUN conda config --add channels bioconda -#RUN conda upgrade -y conda - VOLUME ["/data", "/config"] RUN mkdir -p /var/lib/biomaj/data From e47fe0f0c216379be4d86f36fc7986c8592fad79 Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Fri, 24 Nov 2023 11:22:48 +0100 Subject: [PATCH 09/74] fix --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 76176f1..66f6502 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ RUN rm -rf /tmp/biomaj-prometheus-multiproc && \ # Install docker to allow docker execution from process-message RUN buildDeps='gnupg2 dirmngr software-properties-common' \ - && echo 'deb [arch=amd64] https://download.docker.com/linux/debian buster stable' | sudo tee /etc/apt/sources.list.d/docker.list \ + && echo 'deb [arch=amd64] https://download.docker.com/linux/debian buster stable' | tee /etc/apt/sources.list.d/docker.list \ && apt-get update \ && apt-get install -y apt-transport-https curl libcurl4-openssl-dev python3-pycurl python3-setuptools python3-pip git unzip bzip2 ca-certificates jq --no-install-recommends \ && apt-get install -y $buildDeps --no-install-recommends \ From 3f797a8961402f0d01986334a947512a10f59351 Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Fri, 24 Nov 2023 14:13:21 +0100 Subject: [PATCH 10/74] fix --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 66f6502..31dc859 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,9 +9,10 @@ RUN rm -rf /tmp/biomaj-prometheus-multiproc && \ # Install docker to allow docker execution from process-message RUN buildDeps='gnupg2 dirmngr software-properties-common' \ - && echo 'deb [arch=amd64] https://download.docker.com/linux/debian buster stable' | tee /etc/apt/sources.list.d/docker.list \ && apt-get update \ && apt-get install -y apt-transport-https curl libcurl4-openssl-dev python3-pycurl python3-setuptools python3-pip git unzip bzip2 ca-certificates jq --no-install-recommends \ + && echo 'deb [arch=amd64] https://download.docker.com/linux/debian buster stable' | tee /etc/apt/sources.list.d/docker.list \ + && apt-get update \ && apt-get install -y $buildDeps --no-install-recommends \ && curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - \ && apt-get install --no-install-recommends -y docker-ce-cli \ From ae31de0e859f25976234c73b6b87bc895704b2d7 Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Fri, 24 Nov 2023 14:14:36 +0100 Subject: [PATCH 11/74] fix --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 31dc859..a3ccd31 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,9 +12,9 @@ RUN buildDeps='gnupg2 dirmngr software-properties-common' \ && apt-get update \ && apt-get install -y apt-transport-https curl libcurl4-openssl-dev python3-pycurl python3-setuptools python3-pip git unzip bzip2 ca-certificates jq --no-install-recommends \ && echo 'deb [arch=amd64] https://download.docker.com/linux/debian buster stable' | tee /etc/apt/sources.list.d/docker.list \ + && curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - \ && apt-get update \ && apt-get install -y $buildDeps --no-install-recommends \ - && curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - \ && apt-get install --no-install-recommends -y docker-ce-cli \ && apt-get purge -y --auto-remove $buildDeps \ && apt-get clean \ From c83a3d17f31ccbc7102e9ba4aa646385dea8bb33 Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Fri, 24 Nov 2023 14:17:55 +0100 Subject: [PATCH 12/74] fix --- Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index a3ccd31..8a73056 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,13 +8,12 @@ RUN rm -rf /tmp/biomaj-prometheus-multiproc && \ mkdir -p /tmp/biomaj-prometheus-multiproc # Install docker to allow docker execution from process-message -RUN buildDeps='gnupg2 dirmngr software-properties-common' \ +RUN buildDeps='gnupg2 dirmngr' \ && apt-get update \ - && apt-get install -y apt-transport-https curl libcurl4-openssl-dev python3-pycurl python3-setuptools python3-pip git unzip bzip2 ca-certificates jq --no-install-recommends \ + && apt-get install -y apt-transport-https curl libcurl4-openssl-dev python3-pycurl python3-setuptools python3-pip git unzip bzip2 ca-certificates jq $buildDeps --no-install-recommends \ && echo 'deb [arch=amd64] https://download.docker.com/linux/debian buster stable' | tee /etc/apt/sources.list.d/docker.list \ && curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - \ && apt-get update \ - && apt-get install -y $buildDeps --no-install-recommends \ && apt-get install --no-install-recommends -y docker-ce-cli \ && apt-get purge -y --auto-remove $buildDeps \ && apt-get clean \ From 908dc9f24a96aac250c5d5a118c53a8c5c7e3377 Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Fri, 24 Nov 2023 14:23:32 +0100 Subject: [PATCH 13/74] ci test --- .github/workflows/dockerimage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml index 0a134c9..ed36c3b 100644 --- a/.github/workflows/dockerimage.yml +++ b/.github/workflows/dockerimage.yml @@ -38,7 +38,7 @@ jobs: - name: Log in to Docker Hub uses: docker/login-action@v3 with: - username: ${{ secrets.GHR_USER }} + username: ${{ github.actor }} password: ${{ secrets.GHR_TOKEN }} - name: Extract metadata (tags, labels) for Docker From acf407a95f9db9c1e0186f6f6c9f356840aa4183 Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Fri, 24 Nov 2023 14:25:38 +0100 Subject: [PATCH 14/74] consul image moved --- docker-compose-otherdb.yml | 2 +- docker-compose-traefik.yml | 2 +- docker-compose.yml | 2 +- kubernetes/helm/templates/biomaj-consul.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-compose-otherdb.yml b/docker-compose-otherdb.yml index 958cdd3..ca5862b 100644 --- a/docker-compose-otherdb.yml +++ b/docker-compose-otherdb.yml @@ -28,7 +28,7 @@ - RABBITMQ_DEFAULT_PASS=biomaj biomaj-consul: - image: consul + image: hashicorp/consul:1.17 hostname: consulnode1 ports: - "8400:8400" diff --git a/docker-compose-traefik.yml b/docker-compose-traefik.yml index 1f1410c..7551465 100644 --- a/docker-compose-traefik.yml +++ b/docker-compose-traefik.yml @@ -50,7 +50,7 @@ - RABBITMQ_DEFAULT_PASS=biomaj biomaj-consul: - image: consul + image: hashicorp/consul:1.17 hostname: consulnode1 ports: - "8400:8400" diff --git a/docker-compose.yml b/docker-compose.yml index b15d3fa..00a4406 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,7 +28,7 @@ - RABBITMQ_DEFAULT_PASS=biomaj biomaj-consul: - image: consul + image: hashicorp/consul:1.17 hostname: consulnode1 ports: - "8400:8400" diff --git a/kubernetes/helm/templates/biomaj-consul.yml b/kubernetes/helm/templates/biomaj-consul.yml index dad6b1d..122a410 100644 --- a/kubernetes/helm/templates/biomaj-consul.yml +++ b/kubernetes/helm/templates/biomaj-consul.yml @@ -8,6 +8,6 @@ spec: hostname: consulnode1 containers: - name: biomaj-consul - image: consul + image: hashicorp/consul:1.17 command: ["consul"] args: ["agent", "-dev", "-client", "0.0.0.0", "-log-level", "info"] From bc06800e45e2f686da6ad0a4d9858d6275d136da Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Tue, 9 Apr 2024 14:54:40 +0200 Subject: [PATCH 15/74] trigger tests From 03aa65d69b1d3219f846c4a40839184420b8a3c8 Mon Sep 17 00:00:00 2001 From: Remy Siminel <95074810+rsiminel@users.noreply.github.com> Date: Tue, 9 Apr 2024 14:23:38 +0000 Subject: [PATCH 16/74] test commit --- .github/workflows/dockerimage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml index ed36c3b..039fe13 100644 --- a/.github/workflows/dockerimage.yml +++ b/.github/workflows/dockerimage.yml @@ -1,6 +1,6 @@ name: Docker Image CI -on: ["push"] +on: ["push"] # Test comment jobs: From a270b60daae222854687beeb982b453c1ee0ee8f Mon Sep 17 00:00:00 2001 From: remy siminel Date: Tue, 9 Apr 2024 17:08:16 +0200 Subject: [PATCH 17/74] test commit --- .github/workflows/dockerimage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml index 039fe13..ed36c3b 100644 --- a/.github/workflows/dockerimage.yml +++ b/.github/workflows/dockerimage.yml @@ -1,6 +1,6 @@ name: Docker Image CI -on: ["push"] # Test comment +on: ["push"] jobs: From 9398a903d36ba95f4ebd67a4506f4e46b5a38891 Mon Sep 17 00:00:00 2001 From: remy siminel Date: Tue, 9 Apr 2024 17:21:45 +0200 Subject: [PATCH 18/74] enable manual workflow triggering --- .github/workflows/dockerimage.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml index ed36c3b..b49edd5 100644 --- a/.github/workflows/dockerimage.yml +++ b/.github/workflows/dockerimage.yml @@ -1,6 +1,7 @@ name: Docker Image CI -on: ["push"] +on: ["push", "workflow_dispatch"] + jobs: From 5b3866a0f95693c8aff3e2fd2838201bef0e64c8 Mon Sep 17 00:00:00 2001 From: remy siminel Date: Wed, 10 Apr 2024 11:16:40 +0200 Subject: [PATCH 19/74] fix01 --- .github/workflows/dockerimage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml index b49edd5..3e20dc5 100644 --- a/.github/workflows/dockerimage.yml +++ b/.github/workflows/dockerimage.yml @@ -24,7 +24,7 @@ jobs: mkdir -p biomaj biomaj/conf biomaj/log biomaj/lock biomaj/db biomaj/process biomaj/cache cp test-local/etc/biomaj/conf.d/alu.properties biomaj/conf/ cp test-local/etc/biomaj/conf.d/local.properties biomaj/conf/ - docker-compose config --q + docker-compose config -q docker build -t quay.io/genouest/biomaj . docker-compose up -d sleep 20 From e5fbe2a7e7cdd71d101475165d61b8caaf852193 Mon Sep 17 00:00:00 2001 From: remy siminel Date: Wed, 10 Apr 2024 15:08:49 +0200 Subject: [PATCH 20/74] update .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 61310a5..90a36d9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ .env biomaj +biomaj/conf/alu.properties +biomaj/conf/local.properties !*biomaj/ mongo redis From 30d0d3f68f1f4bd0594a64efbe6ff3e7f5fd2c5d Mon Sep 17 00:00:00 2001 From: remy siminel Date: Wed, 10 Apr 2024 17:47:52 +0200 Subject: [PATCH 21/74] fix02 --- .github/workflows/dockerimage.yml | 2 +- docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml index 3e20dc5..f52aa37 100644 --- a/.github/workflows/dockerimage.yml +++ b/.github/workflows/dockerimage.yml @@ -25,7 +25,7 @@ jobs: cp test-local/etc/biomaj/conf.d/alu.properties biomaj/conf/ cp test-local/etc/biomaj/conf.d/local.properties biomaj/conf/ docker-compose config -q - docker build -t quay.io/genouest/biomaj . + docker-compose build docker-compose up -d sleep 20 ./docker-test.sh local diff --git a/docker-compose.yml b/docker-compose.yml index 00a4406..5ac6036 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -305,6 +305,6 @@ build: context: proxy/nginx volumes: - - ${PWD}/proxy/nginx/default.conf:/etc/nginx/conf.d/default.conf:ro + - ./proxy/nginx/default.conf:/etc/nginx/conf.d/default.conf:ro - ${BIOMAJ_DIR}/biomaj/db:/db:ro command: [nginx-debug, '-g', 'daemon off;'] From 2443b6723b6a7dfe25220d5b8ece53136d5beedc Mon Sep 17 00:00:00 2001 From: remy siminel Date: Thu, 11 Apr 2024 09:44:11 +0200 Subject: [PATCH 22/74] de-osallou-ify docker-compose.yml --- docker-compose.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 5ac6036..538f848 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -82,7 +82,7 @@ - biomaj-consul biomaj-user-web: - image: quay.io/osallou/biomaj + image: quay.io/genouest/biomaj build: context: . volumes: @@ -100,7 +100,7 @@ biomaj-download-web: - image: quay.io/osallou/biomaj + image: quay.io/genouest/biomaj build: context: . volumes: @@ -117,7 +117,7 @@ command: ["/startup.sh", "gunicorn", "-c", "/etc/biomaj/gunicorn_conf.py", "-b", "0.0.0.0:5000", "--log-config", "/etc/biomaj/production.ini biomaj_download.biomaj_download_web:app"] biomaj-download-message: - image: quay.io/osallou/biomaj + image: quay.io/genouest/biomaj build: context: . volumes: @@ -137,7 +137,7 @@ command: ["/startup.sh", "python3", "/root/biomaj-download/bin/biomaj_download_consumer.py"] biomaj-process-web: - image: quay.io/osallou/biomaj + image: quay.io/genouest/biomaj build: context: . volumes: @@ -155,7 +155,7 @@ command: ["/startup.sh", "gunicorn", "-c", "/etc/biomaj/gunicorn_conf.py", "-b", "0.0.0.0:5000", "--log-config", "/etc/biomaj/production.ini biomaj_process.biomaj_process_web:app"] biomaj-process-message: - image: quay.io/osallou/biomaj + image: quay.io/genouest/biomaj build: context: . volumes: @@ -178,7 +178,7 @@ command: ["/startup.sh", "python3", "/root/biomaj-process/bin/biomaj_process_consumer.py"] biomaj-daemon-web: - image: quay.io/osallou/biomaj + image: quay.io/genouest/biomaj build: context: . volumes: @@ -203,7 +203,7 @@ command: ["/startup.sh", "gunicorn", "-c", "/etc/biomaj/gunicorn_conf.py", "-b", "0.0.0.0:5000", "--log-config", "/etc/biomaj/production.ini biomaj_daemon.daemon.biomaj_daemon_web:app"] biomaj-daemon-message: - image: quay.io/osallou/biomaj + image: quay.io/genouest/biomaj build: context: . volumes: @@ -227,7 +227,7 @@ command: ["/startup.sh", "python3", "/root/biomaj-daemon/bin/biomaj_daemon_consumer.py"] biomaj-watcher-web: - image: quay.io/osallou/biomaj + image: quay.io/genouest/biomaj build: context: . volumes: @@ -244,7 +244,7 @@ command: ["/startup.sh", "/root/watcher.sh"] biomaj-cron: - image: quay.io/osallou/biomaj-cron + image: quay.io/genouest/biomaj-cron volumes: - ${BIOMAJ_DIR}/biomaj:/var/lib/biomaj/data environment: @@ -253,7 +253,7 @@ - biomaj-daemon-web biomaj-ftp: - image: quay.io/osallou/biomaj + image: quay.io/genouest/biomaj build: context: . volumes: @@ -276,7 +276,7 @@ command: ["/startup.sh", "python3", "/root/biomaj-ftp/bin/biomaj_ftp_service.py"] biomaj-release-web: - image: quay.io/osallou/biomaj + image: quay.io/genouest/biomaj build: context: . volumes: @@ -294,7 +294,7 @@ command: ["/startup.sh", "python3", "/root/biomaj-release/bin/biomaj_release.py"] biomaj-expose: - image: quay.io/osallou/biomaj-expose + image: quay.io/genouest/biomaj-expose labels: - traefik.enable=true - traefik.frontend.rule=PathPrefix:/db From d65dba59413d33f54a9d335bbbf4e49e1b0ce828 Mon Sep 17 00:00:00 2001 From: remy siminel Date: Thu, 11 Apr 2024 10:22:20 +0200 Subject: [PATCH 23/74] clarify actions output --- .github/workflows/dockerimage.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml index f52aa37..8fb2cf7 100644 --- a/.github/workflows/dockerimage.yml +++ b/.github/workflows/dockerimage.yml @@ -17,18 +17,28 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Build and test the Docker image + - name: Set up environment run: | echo "BIOMAJ_DIR=$(pwd)" > .env echo "BIOMAJ_DATA_DIR=$(pwd)/biomaj" >> .env mkdir -p biomaj biomaj/conf biomaj/log biomaj/lock biomaj/db biomaj/process biomaj/cache cp test-local/etc/biomaj/conf.d/alu.properties biomaj/conf/ cp test-local/etc/biomaj/conf.d/local.properties biomaj/conf/ + - name: Validate Docker Compose configuration + run: | docker-compose config -q + - name: Build Docker image + run: | docker-compose build + - name: Start Docker containers + run: | docker-compose up -d + - name: Execute test script + run: | sleep 20 ./docker-test.sh local + - name: Shut down Docker containers + run: | docker-compose down publish: From 8c2a68b5247d283af2526e88750f871c6b1e4b10 Mon Sep 17 00:00:00 2001 From: remy siminel Date: Thu, 11 Apr 2024 13:52:48 +0200 Subject: [PATCH 24/74] fix03 --- .github/workflows/dockerimage.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml index 8fb2cf7..4c269c6 100644 --- a/.github/workflows/dockerimage.yml +++ b/.github/workflows/dockerimage.yml @@ -24,6 +24,7 @@ jobs: mkdir -p biomaj biomaj/conf biomaj/log biomaj/lock biomaj/db biomaj/process biomaj/cache cp test-local/etc/biomaj/conf.d/alu.properties biomaj/conf/ cp test-local/etc/biomaj/conf.d/local.properties biomaj/conf/ + apt-get update && apt-get install -y jq - name: Validate Docker Compose configuration run: | docker-compose config -q From 17b4d224b417ec8b11417b6eda85b0c254b5736c Mon Sep 17 00:00:00 2001 From: remy siminel Date: Thu, 11 Apr 2024 17:18:17 +0200 Subject: [PATCH 25/74] remove jq dependancy --- .github/workflows/dockerimage.yml | 1 - docker-test.sh | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml index 4c269c6..8fb2cf7 100644 --- a/.github/workflows/dockerimage.yml +++ b/.github/workflows/dockerimage.yml @@ -24,7 +24,6 @@ jobs: mkdir -p biomaj biomaj/conf biomaj/log biomaj/lock biomaj/db biomaj/process biomaj/cache cp test-local/etc/biomaj/conf.d/alu.properties biomaj/conf/ cp test-local/etc/biomaj/conf.d/local.properties biomaj/conf/ - apt-get update && apt-get install -y jq - name: Validate Docker Compose configuration run: | docker-compose config -q diff --git a/docker-test.sh b/docker-test.sh index 7f22721..d1f60a0 100755 --- a/docker-test.sh +++ b/docker-test.sh @@ -6,7 +6,7 @@ BANK=$1 TS=$(date +%s) USER="biomaj$TS" docker-compose ps -APIKEY=`docker-compose exec biomaj-user-web biomaj-users.py -A add -E biomaj@fake.fr -U $USER -P biomaj --json | jq -r '.apikey'` +APIKEY=`docker-compose exec biomaj-user-web biomaj-users.py -A add -E biomaj@fake.fr -U $USER -P biomaj --json | awk -F'"apikey":' '{print $2}' | awk -F'"' '{print $2}'` echo "APIKEY=$APIKEY" echo "update $BANK" From 61baad4f897f693195a65656a0057627f5e409fb Mon Sep 17 00:00:00 2001 From: remy siminel Date: Thu, 11 Apr 2024 18:33:21 +0200 Subject: [PATCH 26/74] fix04 --- .github/workflows/dockerimage.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml index 8fb2cf7..e722fd1 100644 --- a/.github/workflows/dockerimage.yml +++ b/.github/workflows/dockerimage.yml @@ -26,20 +26,20 @@ jobs: cp test-local/etc/biomaj/conf.d/local.properties biomaj/conf/ - name: Validate Docker Compose configuration run: | - docker-compose config -q + docker compose config -q - name: Build Docker image run: | - docker-compose build + docker compose build -q - name: Start Docker containers run: | - docker-compose up -d + docker compose up -d - name: Execute test script run: | sleep 20 ./docker-test.sh local - name: Shut down Docker containers run: | - docker-compose down + docker compose down publish: runs-on: ubuntu-latest From 4ee3fc1cb3ecdf541c03f8c27cdcbc311ec0be72 Mon Sep 17 00:00:00 2001 From: remy siminel Date: Fri, 12 Apr 2024 12:05:29 +0200 Subject: [PATCH 27/74] fix dockerfile dependencies --- Dockerfile | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8a73056..bb3e454 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,6 +15,8 @@ RUN buildDeps='gnupg2 dirmngr' \ && curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - \ && apt-get update \ && apt-get install --no-install-recommends -y docker-ce-cli \ + && apt-get install --no-install-recommends -y python3-markupsafe \ + && apt-get install --no-install-recommends -y python3-bcrypt \ && apt-get purge -y --auto-remove $buildDeps \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* @@ -39,13 +41,13 @@ ENV BIOMAJ_CONFIG=/etc/biomaj/config.yml RUN mkdir -p /var/log/biomaj RUN pip3 install --no-cache-dir setuptools --upgrade && \ - pip3 install --no-cache-dir greenlet==0.4.17 && \ - pip3 install --no-cache-dir gevent==1.4.0 && \ - pip3 install --no-cache-dir graypy && \ - pip3 install --no-cache-dir pymongo==3.12.3 && \ - pip3 install --no-cache-dir redis==3.5.3 && \ - pip3 install --no-cache-dir wheel && \ - pip3 install --no-cache-dir PyYAML==5.3.1 +pip3 install --no-cache-dir greenlet==0.4.17 && \ +pip3 install --no-cache-dir gevent==1.4.0 && \ +pip3 install --no-cache-dir graypy && \ +pip3 install --no-cache-dir pymongo==3.12.3 && \ +pip3 install --no-cache-dir redis==3.5.3 && \ +pip3 install --no-cache-dir wheel && \ +pip3 install --no-cache-dir PyYAML==5.4.1 ENV SUDO_FORCE_REMOVE=yes RUN buildDeps="gcc python3-dev protobuf-compiler" \ From 7dc1f21389f1bdd63ac0499272bdc559a22c956e Mon Sep 17 00:00:00 2001 From: remy siminel Date: Fri, 12 Apr 2024 13:47:26 +0200 Subject: [PATCH 28/74] fix05 --- docker-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-test.sh b/docker-test.sh index d1f60a0..7f22721 100755 --- a/docker-test.sh +++ b/docker-test.sh @@ -6,7 +6,7 @@ BANK=$1 TS=$(date +%s) USER="biomaj$TS" docker-compose ps -APIKEY=`docker-compose exec biomaj-user-web biomaj-users.py -A add -E biomaj@fake.fr -U $USER -P biomaj --json | awk -F'"apikey":' '{print $2}' | awk -F'"' '{print $2}'` +APIKEY=`docker-compose exec biomaj-user-web biomaj-users.py -A add -E biomaj@fake.fr -U $USER -P biomaj --json | jq -r '.apikey'` echo "APIKEY=$APIKEY" echo "update $BANK" From b7e7176393554ce8ec9e11bfb1f71cc902a1e2aa Mon Sep 17 00:00:00 2001 From: remy siminel Date: Mon, 15 Apr 2024 15:47:49 +0200 Subject: [PATCH 29/74] fix06 --- docker-test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-test.sh b/docker-test.sh index 7f22721..117e5a9 100755 --- a/docker-test.sh +++ b/docker-test.sh @@ -6,11 +6,11 @@ BANK=$1 TS=$(date +%s) USER="biomaj$TS" docker-compose ps -APIKEY=`docker-compose exec biomaj-user-web biomaj-users.py -A add -E biomaj@fake.fr -U $USER -P biomaj --json | jq -r '.apikey'` +APIKEY=`docker-compose exec biomaj-user-web python3 /usr/local/bin/biomaj-users.py -A add -E biomaj@fake.fr -U $USER -P biomaj --json | jq -r '.apikey'` echo "APIKEY=$APIKEY" echo "update $BANK" -docker-compose exec biomaj-user-web biomaj-cli.py --proxy http://biomaj-public-proxy --api-key $APIKEY --update --bank $BANK +docker-compose exec biomaj-user-web python3 /usr/local/bin/biomaj-cli.py --proxy http://biomaj-public-proxy --api-key $APIKEY --update --bank $BANK count=0 From 54c5a841804c21f1e338936062a5a86cc272204c Mon Sep 17 00:00:00 2001 From: remy siminel Date: Mon, 15 Apr 2024 16:56:58 +0200 Subject: [PATCH 30/74] fix07 --- Dockerfile | 4 ++-- docker-test.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index bb3e454..843ccaa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,7 +25,7 @@ ENV BIOMAJ_RELEASE="shahikorma-v13" RUN git clone https://github.com/genouest/biomaj-core.git && \ git clone https://github.com/genouest/biomaj-zipkin.git && \ - git clone https://github.com/genouest/biomaj-user.git && \ + git clone https://github.com/rsiminel/biomaj-user-shebanger.git && \ git clone https://github.com/genouest/biomaj-cli.git && \ git clone https://github.com/genouest/biomaj-process.git && \ git clone https://github.com/genouest/biomaj-download.git && \ @@ -56,7 +56,7 @@ RUN buildDeps="gcc python3-dev protobuf-compiler" \ && apt-get install -y $buildDeps --no-install-recommends \ && cd /root/biomaj-core && python3 setup.py build && pip3 install --no-cache-dir . \ && cd /root/biomaj-zipkin && python3 setup.py build && pip3 install --no-cache-dir . \ - && cd /root/biomaj-user && python3 setup.py build && pip3 install --no-cache-dir . \ + && cd /root/biomaj-user-shebanger && python3 setup.py build && pip3 install --no-cache-dir . \ && cd /root/biomaj-cli && python3 setup.py build && pip3 install --no-cache-dir . \ && cd /root/biomaj-process/biomaj_process/message && protoc --python_out=. procmessage.proto \ && cd /root/biomaj-process && python3 setup.py build && pip3 install --no-cache-dir . \ diff --git a/docker-test.sh b/docker-test.sh index 117e5a9..0aee060 100755 --- a/docker-test.sh +++ b/docker-test.sh @@ -10,7 +10,7 @@ APIKEY=`docker-compose exec biomaj-user-web python3 /usr/local/bin/biomaj-users. echo "APIKEY=$APIKEY" echo "update $BANK" -docker-compose exec biomaj-user-web python3 /usr/local/bin/biomaj-cli.py --proxy http://biomaj-public-proxy --api-key $APIKEY --update --bank $BANK +docker-compose exec biomaj-user-web biomaj-cli.py --proxy http://biomaj-public-proxy --api-key $APIKEY --update --bank $BANK count=0 From b67013e5e28b809774ffd861ebfe1f0e0333931a Mon Sep 17 00:00:00 2001 From: remy siminel Date: Mon, 15 Apr 2024 17:03:17 +0200 Subject: [PATCH 31/74] replace docker-compose --- docker-test.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docker-test.sh b/docker-test.sh index 0aee060..009f6ed 100755 --- a/docker-test.sh +++ b/docker-test.sh @@ -5,20 +5,20 @@ set -e BANK=$1 TS=$(date +%s) USER="biomaj$TS" -docker-compose ps -APIKEY=`docker-compose exec biomaj-user-web python3 /usr/local/bin/biomaj-users.py -A add -E biomaj@fake.fr -U $USER -P biomaj --json | jq -r '.apikey'` +docker compose ps +APIKEY=`docker compose exec biomaj-user-web python3 /usr/local/bin/biomaj-users.py -A add -E biomaj@fake.fr -U $USER -P biomaj --json | jq -r '.apikey'` echo "APIKEY=$APIKEY" echo "update $BANK" -docker-compose exec biomaj-user-web biomaj-cli.py --proxy http://biomaj-public-proxy --api-key $APIKEY --update --bank $BANK +docker compose exec biomaj-user-web biomaj-cli.py --proxy http://biomaj-public-proxy --api-key $APIKEY --update --bank $BANK count=0 while true; do - docker-compose exec biomaj-user-web biomaj-cli.py --proxy http://biomaj-public-proxy --api-key $APIKEY --status --bank $BANK + docker compose exec biomaj-user-web biomaj-cli.py --proxy http://biomaj-public-proxy --api-key $APIKEY --status --bank $BANK sleep 60 echo "Check $BANK update status" - PROD=`docker-compose exec biomaj-user-web biomaj-cli.py --proxy http://biomaj-public-proxy --api-key $APIKEY --status --bank $BANK --json | jq '.bank.production.details[0]'` + PROD=`docker compose exec biomaj-user-web biomaj-cli.py --proxy http://biomaj-public-proxy --api-key $APIKEY --status --bank $BANK --json | jq '.bank.production.details[0]'` echo "PROD = $PROD" if [ "$PROD" == "null" ]; then echo "Not updated yet, trying again..." @@ -26,9 +26,9 @@ while true; do if test $count -eq 5 then echo "Still failing after 5 minutes" - docker-compose logs biomaj-daemon-message - docker-compose logs biomaj-download-web - docker-compose logs biomaj-download-message + docker compose logs biomaj-daemon-message + docker compose logs biomaj-download-web + docker compose logs biomaj-download-message exit 1 fi else @@ -36,5 +36,5 @@ while true; do break fi done -docker-compose exec biomaj-user-web biomaj-cli.py --proxy http://biomaj-public-proxy --api-key $APIKEY --status --bank $BANK +docker compose exec biomaj-user-web biomaj-cli.py --proxy http://biomaj-public-proxy --api-key $APIKEY --status --bank $BANK From 17e292a6874b203d84c0518fce263809d130c13f Mon Sep 17 00:00:00 2001 From: remy siminel Date: Tue, 16 Apr 2024 13:44:06 +0200 Subject: [PATCH 32/74] fix08 --- .github/workflows/dockerimage.yml | 4 +++- Dockerfile | 4 +--- docker-test.sh | 10 ++++++---- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml index e722fd1..eef2ea2 100644 --- a/.github/workflows/dockerimage.yml +++ b/.github/workflows/dockerimage.yml @@ -21,6 +21,8 @@ jobs: run: | echo "BIOMAJ_DIR=$(pwd)" > .env echo "BIOMAJ_DATA_DIR=$(pwd)/biomaj" >> .env + echo "BIOMAJ_USER_PASSWORD=biomaj_user_default_password" >>> .env + echo "DOCKER_URL=tcp://x.y.z:2375" >>>> .env mkdir -p biomaj biomaj/conf biomaj/log biomaj/lock biomaj/db biomaj/process biomaj/cache cp test-local/etc/biomaj/conf.d/alu.properties biomaj/conf/ cp test-local/etc/biomaj/conf.d/local.properties biomaj/conf/ @@ -29,7 +31,7 @@ jobs: docker compose config -q - name: Build Docker image run: | - docker compose build -q + docker compose build -q --no-cache - name: Start Docker containers run: | docker compose up -d diff --git a/Dockerfile b/Dockerfile index 843ccaa..3e2887a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,8 +15,6 @@ RUN buildDeps='gnupg2 dirmngr' \ && curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - \ && apt-get update \ && apt-get install --no-install-recommends -y docker-ce-cli \ - && apt-get install --no-install-recommends -y python3-markupsafe \ - && apt-get install --no-install-recommends -y python3-bcrypt \ && apt-get purge -y --auto-remove $buildDeps \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* @@ -68,7 +66,7 @@ RUN buildDeps="gcc python3-dev protobuf-compiler" \ && cd /root/biomaj-ftp && python3 setup.py build && pip3 install --no-cache-dir . \ && cd /root/biomaj-release && python3 setup.py build && pip3 install --no-cache-dir . \ && cd /root/biomaj-data && python3 setup.py build && pip3 install --no-cache-dir . \ - && apt-get install --no-install-recommends -y wget bzip2 ca-certificates curl git \ + && apt-get install --no-install-recommends -y wget bzip2 ca-certificates curl git nano python3-markupsafe python3-bcrypt \ && apt-get purge -y --auto-remove $buildDeps \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* diff --git a/docker-test.sh b/docker-test.sh index 009f6ed..910bb1c 100755 --- a/docker-test.sh +++ b/docker-test.sh @@ -10,7 +10,9 @@ APIKEY=`docker compose exec biomaj-user-web python3 /usr/local/bin/biomaj-users. echo "APIKEY=$APIKEY" echo "update $BANK" +echo "biomaj-cli begin" docker compose exec biomaj-user-web biomaj-cli.py --proxy http://biomaj-public-proxy --api-key $APIKEY --update --bank $BANK +echo "biomaj-cli end" count=0 @@ -26,10 +28,10 @@ while true; do if test $count -eq 5 then echo "Still failing after 5 minutes" - docker compose logs biomaj-daemon-message - docker compose logs biomaj-download-web - docker compose logs biomaj-download-message - exit 1 + #docker compose logs biomaj-daemon-message + #docker compose logs biomaj-download-web + #docker compose logs biomaj-download-message + exit 1 fi else echo "Success" From d6b297b1dd284b7e78eb43efe13d6fb37b74765b Mon Sep 17 00:00:00 2001 From: remy siminel Date: Tue, 16 Apr 2024 14:27:52 +0200 Subject: [PATCH 33/74] fix09 --- .github/workflows/dockerimage.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml index eef2ea2..6595391 100644 --- a/.github/workflows/dockerimage.yml +++ b/.github/workflows/dockerimage.yml @@ -21,8 +21,6 @@ jobs: run: | echo "BIOMAJ_DIR=$(pwd)" > .env echo "BIOMAJ_DATA_DIR=$(pwd)/biomaj" >> .env - echo "BIOMAJ_USER_PASSWORD=biomaj_user_default_password" >>> .env - echo "DOCKER_URL=tcp://x.y.z:2375" >>>> .env mkdir -p biomaj biomaj/conf biomaj/log biomaj/lock biomaj/db biomaj/process biomaj/cache cp test-local/etc/biomaj/conf.d/alu.properties biomaj/conf/ cp test-local/etc/biomaj/conf.d/local.properties biomaj/conf/ From 51d590f90503899d1211988563e16895e4d4e715 Mon Sep 17 00:00:00 2001 From: remy siminel Date: Tue, 16 Apr 2024 14:28:30 +0200 Subject: [PATCH 34/74] fix10 --- .github/workflows/dockerimage.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml index 6595391..d7fb1fd 100644 --- a/.github/workflows/dockerimage.yml +++ b/.github/workflows/dockerimage.yml @@ -35,7 +35,6 @@ jobs: docker compose up -d - name: Execute test script run: | - sleep 20 ./docker-test.sh local - name: Shut down Docker containers run: | From ba25b60bc3f247395cb3bc17bdbb914404366a6d Mon Sep 17 00:00:00 2001 From: remy siminel Date: Tue, 16 Apr 2024 14:33:54 +0200 Subject: [PATCH 35/74] fix publish test password --- .github/workflows/dockerimage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml index d7fb1fd..94e062d 100644 --- a/.github/workflows/dockerimage.yml +++ b/.github/workflows/dockerimage.yml @@ -49,7 +49,7 @@ jobs: uses: docker/login-action@v3 with: username: ${{ github.actor }} - password: ${{ secrets.GHR_TOKEN }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Extract metadata (tags, labels) for Docker id: meta From 6f398beffca751ed8e7e96d938dd80dd5dc59d46 Mon Sep 17 00:00:00 2001 From: remy siminel Date: Tue, 16 Apr 2024 15:10:42 +0200 Subject: [PATCH 36/74] fix publish test username --- .github/workflows/dockerimage.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml index 94e062d..9b0d283 100644 --- a/.github/workflows/dockerimage.yml +++ b/.github/workflows/dockerimage.yml @@ -48,8 +48,8 @@ jobs: - name: Log in to Docker Hub uses: docker/login-action@v3 with: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + username: ${{ secrets.GHR_USRNAME }} + password: ${{ secrets.GHR_TOKEN }} - name: Extract metadata (tags, labels) for Docker id: meta From a7160d969e807da22ee051157fc1aa48b6bf51a1 Mon Sep 17 00:00:00 2001 From: remy siminel Date: Tue, 16 Apr 2024 17:31:12 +0200 Subject: [PATCH 37/74] fix11 --- Dockerfile | 3 ++- docker-test.sh | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3e2887a..8301be5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,7 +45,8 @@ pip3 install --no-cache-dir graypy && \ pip3 install --no-cache-dir pymongo==3.12.3 && \ pip3 install --no-cache-dir redis==3.5.3 && \ pip3 install --no-cache-dir wheel && \ -pip3 install --no-cache-dir PyYAML==5.4.1 +pip3 install --no-cache-dir PyYAML==5.4.1 && \ +python3 -m pip install ftputil ENV SUDO_FORCE_REMOVE=yes RUN buildDeps="gcc python3-dev protobuf-compiler" \ diff --git a/docker-test.sh b/docker-test.sh index 910bb1c..2ddb261 100755 --- a/docker-test.sh +++ b/docker-test.sh @@ -28,10 +28,10 @@ while true; do if test $count -eq 5 then echo "Still failing after 5 minutes" - #docker compose logs biomaj-daemon-message - #docker compose logs biomaj-download-web - #docker compose logs biomaj-download-message - exit 1 + docker compose logs biomaj-daemon-message + docker compose logs biomaj-download-web + docker compose logs biomaj-download-message + exit 1 fi else echo "Success" From 77140501538a52bd75fa73f144832d1478537d86 Mon Sep 17 00:00:00 2001 From: remy siminel Date: Wed, 17 Apr 2024 09:31:28 +0200 Subject: [PATCH 38/74] update dockerfile dependancies --- Dockerfile | 2 +- biomaj/cache/files_1713339021.1611493 | 1 + biomaj/cache/local_files_1713339021.1611493 | 1 + .../local_2024-4-9/flat/global.properties | 185 ++++++++++++++++++ .../flat/global_local.properties | 171 ++++++++++++++++ .../local/1713339020.9425752/TEST1_echo.err | 0 .../local/1713339020.9425752/TEST1_echo.out | 1 + biomaj/log/local/1713339020.9425752/local.log | 62 ++++++ docker-test.sh | 2 - 9 files changed, 422 insertions(+), 3 deletions(-) create mode 100644 biomaj/cache/files_1713339021.1611493 create mode 100644 biomaj/cache/local_files_1713339021.1611493 create mode 100644 biomaj/db/local/local_2024-4-9/flat/global.properties create mode 100644 biomaj/db/local/local_2024-4-9/flat/global_local.properties create mode 100644 biomaj/log/local/1713339020.9425752/TEST1_echo.err create mode 100644 biomaj/log/local/1713339020.9425752/TEST1_echo.out create mode 100644 biomaj/log/local/1713339020.9425752/local.log diff --git a/Dockerfile b/Dockerfile index 8301be5..3666fd2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -67,7 +67,7 @@ RUN buildDeps="gcc python3-dev protobuf-compiler" \ && cd /root/biomaj-ftp && python3 setup.py build && pip3 install --no-cache-dir . \ && cd /root/biomaj-release && python3 setup.py build && pip3 install --no-cache-dir . \ && cd /root/biomaj-data && python3 setup.py build && pip3 install --no-cache-dir . \ - && apt-get install --no-install-recommends -y wget bzip2 ca-certificates curl git nano python3-markupsafe python3-bcrypt \ + && apt-get install --no-install-recommends -y wget bzip2 ca-certificates curl git nano python3-markupsafe python3-bcrypt python3-yapsy \ && apt-get purge -y --auto-remove $buildDeps \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* diff --git a/biomaj/cache/files_1713339021.1611493 b/biomaj/cache/files_1713339021.1611493 new file mode 100644 index 0000000..088f2e1 --- /dev/null +++ b/biomaj/cache/files_1713339021.1611493 @@ -0,0 +1 @@ +[{"permissions": "33188", "group": "0", "user": "0", "size": 3659, "month": 4, "day": 9, "year": 2024, "name": "global_local.properties", "hash": "d5a43c340037f49a5d160d33b913452f", "root": "/etc/biomaj/", "save_as": "global_local.properties"}, {"permissions": "33188", "group": "0", "user": "0", "size": 4157, "month": 4, "day": 9, "year": 2024, "name": "global.properties", "hash": "547be9f51095593544aeb968cd1131de", "root": "/etc/biomaj/", "save_as": "global.properties"}] \ No newline at end of file diff --git a/biomaj/cache/local_files_1713339021.1611493 b/biomaj/cache/local_files_1713339021.1611493 new file mode 100644 index 0000000..088f2e1 --- /dev/null +++ b/biomaj/cache/local_files_1713339021.1611493 @@ -0,0 +1 @@ +[{"permissions": "33188", "group": "0", "user": "0", "size": 3659, "month": 4, "day": 9, "year": 2024, "name": "global_local.properties", "hash": "d5a43c340037f49a5d160d33b913452f", "root": "/etc/biomaj/", "save_as": "global_local.properties"}, {"permissions": "33188", "group": "0", "user": "0", "size": 4157, "month": 4, "day": 9, "year": 2024, "name": "global.properties", "hash": "547be9f51095593544aeb968cd1131de", "root": "/etc/biomaj/", "save_as": "global.properties"}] \ No newline at end of file diff --git a/biomaj/db/local/local_2024-4-9/flat/global.properties b/biomaj/db/local/local_2024-4-9/flat/global.properties new file mode 100644 index 0000000..332fe6d --- /dev/null +++ b/biomaj/db/local/local_2024-4-9/flat/global.properties @@ -0,0 +1,185 @@ +[GENERAL] +root.dir=/var/lib/biomaj/data +conf.dir=%(root.dir)s/conf +log.dir=%(root.dir)s/log +process.dir=%(root.dir)s/process +cache.dir=%(root.dir)s/cache +lock.dir=%(root.dir)s/lock + +plugins_dir=/var/lib/biomaj/plugins + +#The root directory where all databases are stored. +#If your data is not stored under one directory hirearchy +#you can override this value in the database properties file. +data.dir=%(root.dir)s/db + +db.url=mongodb://biomaj-mongo:27017 +db.name=biomaj + +use_ldap=0 +ldap.host=localhost +ldap.port=389 +ldap.dn=nodomain + +use_elastic=0 +#Comma separated list of elasticsearch nodes host1,host2:port2 +elastic_nodes=biomaj-elasticsearch +elastic_index=biomaj +# Calculate data.dir size stats +data.stats=1 + +redis.host=biomaj-redis +redis.port=6379 +redis.db=0 +redis.prefix=biomaj + +# Influxdb configuration (optional) +# User and db must be manually created in influxdb before use +influxdb.host=biomaj-influxdb +influxdb.port=8086 +influxdb.user=biomaj +influxdb.password=biomaj +influxdb.db=biomaj + + +# Needed for download remote service (should be a load balancer to services +micro.biomaj.proxy=http://biomaj-internal-proxy +micro.biomaj.rabbit_mq=biomaj-rabbitmq +micro.biomaj.rabbit_mq_port=5672 +micro.biomaj.rabbit_mq_user=biomaj +micro.biomaj.rabbit_mq_password=biomaj +micro.biomaj.rabbit_mq_virtualhost=/ +micro.biomaj.service.download=1 +micro.biomaj.service.process=1 +micro.biomaj.service.user=1 +micro.biomaj.service.daemon=1 + +auto_publish=1 + +admin=biomaj + +schedule.delay=7 +schedule.execute=1 + +######################## +# Global properties file + + +#To override these settings for a specific database go to its +#properties file and uncomment or add the specific line you want +#to override. + +#---------------- +# Mail Configuration +#--------------- +#Uncomment thes lines if you want receive mail when the workflow is finished + +mail.smtp.host= +#mail.smtp.port=25 +mail.admin= +mail.from=biomaj@genouest.org +#mail.user= +#mail.password= +#mail.tls= +# tail last X bytes of log in mail body , 0 = no tail +# mail.body.tail=2000000 +# attach log file if size < X bytes, 0 for no attach +#mail.body.attach=4000000 +# path to jinja template for subject, leave empty for defaults +#mail.template.subject= +# path to jinja template for body, leave empty for default +#mail.template.body= + +#--------------------- +#Proxy authentification +#--------------------- +#proxyHost= +#proxyPort= +#proxyUser= +#proxyPassword= + +#--------------------- +# PROTOCOL +#------------------- +#possible values : ftp, http, rsync, local +port=21 +username=anonymous +password=anonymous@nowhere.com + +#access user for production directories +production.directory.chmod=775 + +#Number of thread during the download +bank.num.threads=4 + +#Number of threads to use for downloading and processing +files.num.threads=4 + +#to keep more than one release increase this value +keep.old.version=1 + +#Link copy property +do.link.copy=true + + +#The historic log file is generated in log/ +#define level information for output : DEBUG,INFO,WARN,ERR +historic.logfile.level=INFO + +http.parse.dir.line=.*([\\d]{2}-[\\w\\d]{2,5}-[\\d]{4}\\s[\\d]{2}:[\\d]{2}) +http.parse.file.line=.*([\\d]{2}-[\\w\\d]{2,5}-[\\d]{4}\\s[\\d]{2}:[\\d]{2}) +http.parse.file.line= Date: Wed, 17 Apr 2024 10:48:45 +0200 Subject: [PATCH 39/74] fix12 --- .github/workflows/dockerimage.yml | 3 ++- .gitignore | 5 +++-- biomaj/cache/files_1713342794.8999193 | 1 + biomaj/cache/local_files_1713342794.8999193 | 1 + biomaj/log/local/1713339020.9425752/local.log | 1 + 5 files changed, 8 insertions(+), 3 deletions(-) create mode 100644 biomaj/cache/files_1713342794.8999193 create mode 100644 biomaj/cache/local_files_1713342794.8999193 diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml index 9b0d283..5859ea8 100644 --- a/.github/workflows/dockerimage.yml +++ b/.github/workflows/dockerimage.yml @@ -29,12 +29,13 @@ jobs: docker compose config -q - name: Build Docker image run: | - docker compose build -q --no-cache + docker compose build --no-cache - name: Start Docker containers run: | docker compose up -d - name: Execute test script run: | + sleep 20 ./docker-test.sh local - name: Shut down Docker containers run: | diff --git a/.gitignore b/.gitignore index 90a36d9..9a692d2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,8 @@ .env biomaj -biomaj/conf/alu.properties -biomaj/conf/local.properties +biomaj/cash/ +biomaj/conf/ +biomaj/log/local/ !*biomaj/ mongo redis diff --git a/biomaj/cache/files_1713342794.8999193 b/biomaj/cache/files_1713342794.8999193 new file mode 100644 index 0000000..0637a08 --- /dev/null +++ b/biomaj/cache/files_1713342794.8999193 @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/biomaj/cache/local_files_1713342794.8999193 b/biomaj/cache/local_files_1713342794.8999193 new file mode 100644 index 0000000..0637a08 --- /dev/null +++ b/biomaj/cache/local_files_1713342794.8999193 @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/biomaj/log/local/1713339020.9425752/local.log b/biomaj/log/local/1713339020.9425752/local.log index de7d99b..afd2543 100644 --- a/biomaj/log/local/1713339020.9425752/local.log +++ b/biomaj/log/local/1713339020.9425752/local.log @@ -60,3 +60,4 @@ 2024-04-17 07:30:51,917 ERROR [root][MainThread] Cannot load influxdb libraryNo module named 'msgpack' 2024-04-17 07:30:51,942 INFO [root][MainThread] Notify:none 2024-04-17 07:30:51,962 INFO [root][MainThread] Biomaj:Daemon:local:over +2024-04-17 07:35:56,002 INFO [root][MainThread] Biomaj:Daemon:local:update From 50160867b4f20d516c4c7af40f2723c83abe1e8d Mon Sep 17 00:00:00 2001 From: remy siminel Date: Wed, 17 Apr 2024 10:58:07 +0200 Subject: [PATCH 40/74] Revert "update dockerfile dependancies" This reverts commit 77140501538a52bd75fa73f144832d1478537d86. --- Dockerfile | 2 +- biomaj/cache/files_1713339021.1611493 | 1 - biomaj/cache/files_1713342794.8999193 | 1 - biomaj/cache/local_files_1713339021.1611493 | 1 - biomaj/cache/local_files_1713342794.8999193 | 1 - .../local_2024-4-9/flat/global.properties | 185 ------------------ .../flat/global_local.properties | 171 ---------------- .../local/1713339020.9425752/TEST1_echo.err | 0 .../local/1713339020.9425752/TEST1_echo.out | 1 - biomaj/log/local/1713339020.9425752/local.log | 63 ------ 10 files changed, 1 insertion(+), 425 deletions(-) delete mode 100644 biomaj/cache/files_1713339021.1611493 delete mode 100644 biomaj/cache/files_1713342794.8999193 delete mode 100644 biomaj/cache/local_files_1713339021.1611493 delete mode 100644 biomaj/cache/local_files_1713342794.8999193 delete mode 100644 biomaj/db/local/local_2024-4-9/flat/global.properties delete mode 100644 biomaj/db/local/local_2024-4-9/flat/global_local.properties delete mode 100644 biomaj/log/local/1713339020.9425752/TEST1_echo.err delete mode 100644 biomaj/log/local/1713339020.9425752/TEST1_echo.out delete mode 100644 biomaj/log/local/1713339020.9425752/local.log diff --git a/Dockerfile b/Dockerfile index 3666fd2..c1e0f65 100644 --- a/Dockerfile +++ b/Dockerfile @@ -67,7 +67,7 @@ RUN buildDeps="gcc python3-dev protobuf-compiler" \ && cd /root/biomaj-ftp && python3 setup.py build && pip3 install --no-cache-dir . \ && cd /root/biomaj-release && python3 setup.py build && pip3 install --no-cache-dir . \ && cd /root/biomaj-data && python3 setup.py build && pip3 install --no-cache-dir . \ - && apt-get install --no-install-recommends -y wget bzip2 ca-certificates curl git nano python3-markupsafe python3-bcrypt python3-yapsy \ + && apt-get install --no-install-recommends -y wget bzip2 ca-certificates curl git nano python3-markupsafe python3-bcrypt python3-yapsy\ && apt-get purge -y --auto-remove $buildDeps \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* diff --git a/biomaj/cache/files_1713339021.1611493 b/biomaj/cache/files_1713339021.1611493 deleted file mode 100644 index 088f2e1..0000000 --- a/biomaj/cache/files_1713339021.1611493 +++ /dev/null @@ -1 +0,0 @@ -[{"permissions": "33188", "group": "0", "user": "0", "size": 3659, "month": 4, "day": 9, "year": 2024, "name": "global_local.properties", "hash": "d5a43c340037f49a5d160d33b913452f", "root": "/etc/biomaj/", "save_as": "global_local.properties"}, {"permissions": "33188", "group": "0", "user": "0", "size": 4157, "month": 4, "day": 9, "year": 2024, "name": "global.properties", "hash": "547be9f51095593544aeb968cd1131de", "root": "/etc/biomaj/", "save_as": "global.properties"}] \ No newline at end of file diff --git a/biomaj/cache/files_1713342794.8999193 b/biomaj/cache/files_1713342794.8999193 deleted file mode 100644 index 0637a08..0000000 --- a/biomaj/cache/files_1713342794.8999193 +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/biomaj/cache/local_files_1713339021.1611493 b/biomaj/cache/local_files_1713339021.1611493 deleted file mode 100644 index 088f2e1..0000000 --- a/biomaj/cache/local_files_1713339021.1611493 +++ /dev/null @@ -1 +0,0 @@ -[{"permissions": "33188", "group": "0", "user": "0", "size": 3659, "month": 4, "day": 9, "year": 2024, "name": "global_local.properties", "hash": "d5a43c340037f49a5d160d33b913452f", "root": "/etc/biomaj/", "save_as": "global_local.properties"}, {"permissions": "33188", "group": "0", "user": "0", "size": 4157, "month": 4, "day": 9, "year": 2024, "name": "global.properties", "hash": "547be9f51095593544aeb968cd1131de", "root": "/etc/biomaj/", "save_as": "global.properties"}] \ No newline at end of file diff --git a/biomaj/cache/local_files_1713342794.8999193 b/biomaj/cache/local_files_1713342794.8999193 deleted file mode 100644 index 0637a08..0000000 --- a/biomaj/cache/local_files_1713342794.8999193 +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/biomaj/db/local/local_2024-4-9/flat/global.properties b/biomaj/db/local/local_2024-4-9/flat/global.properties deleted file mode 100644 index 332fe6d..0000000 --- a/biomaj/db/local/local_2024-4-9/flat/global.properties +++ /dev/null @@ -1,185 +0,0 @@ -[GENERAL] -root.dir=/var/lib/biomaj/data -conf.dir=%(root.dir)s/conf -log.dir=%(root.dir)s/log -process.dir=%(root.dir)s/process -cache.dir=%(root.dir)s/cache -lock.dir=%(root.dir)s/lock - -plugins_dir=/var/lib/biomaj/plugins - -#The root directory where all databases are stored. -#If your data is not stored under one directory hirearchy -#you can override this value in the database properties file. -data.dir=%(root.dir)s/db - -db.url=mongodb://biomaj-mongo:27017 -db.name=biomaj - -use_ldap=0 -ldap.host=localhost -ldap.port=389 -ldap.dn=nodomain - -use_elastic=0 -#Comma separated list of elasticsearch nodes host1,host2:port2 -elastic_nodes=biomaj-elasticsearch -elastic_index=biomaj -# Calculate data.dir size stats -data.stats=1 - -redis.host=biomaj-redis -redis.port=6379 -redis.db=0 -redis.prefix=biomaj - -# Influxdb configuration (optional) -# User and db must be manually created in influxdb before use -influxdb.host=biomaj-influxdb -influxdb.port=8086 -influxdb.user=biomaj -influxdb.password=biomaj -influxdb.db=biomaj - - -# Needed for download remote service (should be a load balancer to services -micro.biomaj.proxy=http://biomaj-internal-proxy -micro.biomaj.rabbit_mq=biomaj-rabbitmq -micro.biomaj.rabbit_mq_port=5672 -micro.biomaj.rabbit_mq_user=biomaj -micro.biomaj.rabbit_mq_password=biomaj -micro.biomaj.rabbit_mq_virtualhost=/ -micro.biomaj.service.download=1 -micro.biomaj.service.process=1 -micro.biomaj.service.user=1 -micro.biomaj.service.daemon=1 - -auto_publish=1 - -admin=biomaj - -schedule.delay=7 -schedule.execute=1 - -######################## -# Global properties file - - -#To override these settings for a specific database go to its -#properties file and uncomment or add the specific line you want -#to override. - -#---------------- -# Mail Configuration -#--------------- -#Uncomment thes lines if you want receive mail when the workflow is finished - -mail.smtp.host= -#mail.smtp.port=25 -mail.admin= -mail.from=biomaj@genouest.org -#mail.user= -#mail.password= -#mail.tls= -# tail last X bytes of log in mail body , 0 = no tail -# mail.body.tail=2000000 -# attach log file if size < X bytes, 0 for no attach -#mail.body.attach=4000000 -# path to jinja template for subject, leave empty for defaults -#mail.template.subject= -# path to jinja template for body, leave empty for default -#mail.template.body= - -#--------------------- -#Proxy authentification -#--------------------- -#proxyHost= -#proxyPort= -#proxyUser= -#proxyPassword= - -#--------------------- -# PROTOCOL -#------------------- -#possible values : ftp, http, rsync, local -port=21 -username=anonymous -password=anonymous@nowhere.com - -#access user for production directories -production.directory.chmod=775 - -#Number of thread during the download -bank.num.threads=4 - -#Number of threads to use for downloading and processing -files.num.threads=4 - -#to keep more than one release increase this value -keep.old.version=1 - -#Link copy property -do.link.copy=true - - -#The historic log file is generated in log/ -#define level information for output : DEBUG,INFO,WARN,ERR -historic.logfile.level=INFO - -http.parse.dir.line=.*([\\d]{2}-[\\w\\d]{2,5}-[\\d]{4}\\s[\\d]{2}:[\\d]{2}) -http.parse.file.line=.*([\\d]{2}-[\\w\\d]{2,5}-[\\d]{4}\\s[\\d]{2}:[\\d]{2}) -http.parse.file.line= Date: Wed, 17 Apr 2024 11:51:42 +0200 Subject: [PATCH 41/74] fix .gitignore --- .gitignore | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.gitignore b/.gitignore index 9a692d2..f75526a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,5 @@ .env biomaj -biomaj/cash/ -biomaj/conf/ -biomaj/log/local/ -!*biomaj/ mongo redis rabbitmq From afb5dc37182ee49727f0f7fc1c7d7fcf08f10f26 Mon Sep 17 00:00:00 2001 From: remy siminel Date: Fri, 19 Apr 2024 09:50:46 +0200 Subject: [PATCH 42/74] liniting dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c1e0f65..8e97526 100644 --- a/Dockerfile +++ b/Dockerfile @@ -46,7 +46,7 @@ pip3 install --no-cache-dir pymongo==3.12.3 && \ pip3 install --no-cache-dir redis==3.5.3 && \ pip3 install --no-cache-dir wheel && \ pip3 install --no-cache-dir PyYAML==5.4.1 && \ -python3 -m pip install ftputil +python3 -m pip install --no-cache-dir ftputil ENV SUDO_FORCE_REMOVE=yes RUN buildDeps="gcc python3-dev protobuf-compiler" \ From ac6300f9f21310a6b27a524faddc8a1da15d5ba2 Mon Sep 17 00:00:00 2001 From: remy siminel Date: Fri, 19 Apr 2024 11:01:28 +0200 Subject: [PATCH 43/74] test download requirements --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8e97526..0dc7f2c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,9 +26,9 @@ RUN git clone https://github.com/genouest/biomaj-core.git && \ git clone https://github.com/rsiminel/biomaj-user-shebanger.git && \ git clone https://github.com/genouest/biomaj-cli.git && \ git clone https://github.com/genouest/biomaj-process.git && \ - git clone https://github.com/genouest/biomaj-download.git && \ + git clone https://github.com/rsiminel/biomaj-download.git && \ git clone https://github.com/genouest/biomaj.git && \ - git clone https://github.com/genouest/biomaj-daemon.git && \ + git clone https://github.com/rsiminel/biomaj-daemon.git && \ git clone https://github.com/genouest/biomaj-watcher.git && \ git clone https://github.com/genouest/biomaj-ftp.git && \ git clone https://github.com/genouest/biomaj-release.git && \ From 8eb67156cba0db3ac6a0f26066c33ea3f741f892 Mon Sep 17 00:00:00 2001 From: remy siminel Date: Fri, 19 Apr 2024 11:20:03 +0200 Subject: [PATCH 44/74] test process requirements --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0dc7f2c..304002e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,9 +25,9 @@ RUN git clone https://github.com/genouest/biomaj-core.git && \ git clone https://github.com/genouest/biomaj-zipkin.git && \ git clone https://github.com/rsiminel/biomaj-user-shebanger.git && \ git clone https://github.com/genouest/biomaj-cli.git && \ - git clone https://github.com/genouest/biomaj-process.git && \ + git clone https://github.com/rsiminel/biomaj-process.git && \ git clone https://github.com/rsiminel/biomaj-download.git && \ - git clone https://github.com/genouest/biomaj.git && \ + git clone https://github.com/rsiminel/biomaj.git && \ git clone https://github.com/rsiminel/biomaj-daemon.git && \ git clone https://github.com/genouest/biomaj-watcher.git && \ git clone https://github.com/genouest/biomaj-ftp.git && \ From 316daa63598a54c620908b8acfb3105ca3b3f53d Mon Sep 17 00:00:00 2001 From: remy siminel Date: Fri, 19 Apr 2024 11:32:44 +0200 Subject: [PATCH 45/74] test core and ftp requirements --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 304002e..b85c929 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ RUN buildDeps='gnupg2 dirmngr' \ ENV BIOMAJ_RELEASE="shahikorma-v13" -RUN git clone https://github.com/genouest/biomaj-core.git && \ +RUN git clone https://github.com/rsiminel/biomaj-core.git && \ git clone https://github.com/genouest/biomaj-zipkin.git && \ git clone https://github.com/rsiminel/biomaj-user-shebanger.git && \ git clone https://github.com/genouest/biomaj-cli.git && \ @@ -30,7 +30,7 @@ RUN git clone https://github.com/genouest/biomaj-core.git && \ git clone https://github.com/rsiminel/biomaj.git && \ git clone https://github.com/rsiminel/biomaj-daemon.git && \ git clone https://github.com/genouest/biomaj-watcher.git && \ - git clone https://github.com/genouest/biomaj-ftp.git && \ + git clone https://github.com/rsiminel/biomaj-ftp.git && \ git clone https://github.com/genouest/biomaj-release.git && \ git clone https://github.com/genouest/biomaj-data.git From 002d5c1e97fc8578c8811d2c6309c1dfdddef468 Mon Sep 17 00:00:00 2001 From: remy siminel Date: Fri, 19 Apr 2024 11:47:14 +0200 Subject: [PATCH 46/74] test release and watcher requirements --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index b85c929..822763a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,9 +29,9 @@ RUN git clone https://github.com/rsiminel/biomaj-core.git && \ git clone https://github.com/rsiminel/biomaj-download.git && \ git clone https://github.com/rsiminel/biomaj.git && \ git clone https://github.com/rsiminel/biomaj-daemon.git && \ - git clone https://github.com/genouest/biomaj-watcher.git && \ + git clone https://github.com/rsiminel/biomaj-watcher.git && \ git clone https://github.com/rsiminel/biomaj-ftp.git && \ - git clone https://github.com/genouest/biomaj-release.git && \ + git clone https://github.com/rsiminel/biomaj-release.git && \ git clone https://github.com/genouest/biomaj-data.git ENV BIOMAJ_CONFIG=/etc/biomaj/config.yml From 628af3fbd2abe5625e8446d040918576d45512d7 Mon Sep 17 00:00:00 2001 From: remy siminel Date: Fri, 19 Apr 2024 12:04:51 +0200 Subject: [PATCH 47/74] undo requirement tests --- Dockerfile | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 822763a..bfbd11b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,17 +21,17 @@ RUN buildDeps='gnupg2 dirmngr' \ ENV BIOMAJ_RELEASE="shahikorma-v13" -RUN git clone https://github.com/rsiminel/biomaj-core.git && \ +RUN git clone https://github.com/genouest/biomaj-core.git && \ git clone https://github.com/genouest/biomaj-zipkin.git && \ - git clone https://github.com/rsiminel/biomaj-user-shebanger.git && \ + git clone https://github.com/genouest/biomaj-user-shebanger.git && \ git clone https://github.com/genouest/biomaj-cli.git && \ - git clone https://github.com/rsiminel/biomaj-process.git && \ - git clone https://github.com/rsiminel/biomaj-download.git && \ - git clone https://github.com/rsiminel/biomaj.git && \ - git clone https://github.com/rsiminel/biomaj-daemon.git && \ - git clone https://github.com/rsiminel/biomaj-watcher.git && \ - git clone https://github.com/rsiminel/biomaj-ftp.git && \ - git clone https://github.com/rsiminel/biomaj-release.git && \ + git clone https://github.com/genouest/biomaj-process.git && \ + git clone https://github.com/genouest/biomaj-download.git && \ + git clone https://github.com/genouest/biomaj.git && \ + git clone https://github.com/genouest/biomaj-daemon.git && \ + git clone https://github.com/genouest/biomaj-watcher.git && \ + git clone https://github.com/genouest/biomaj-ftp.git && \ + git clone https://github.com/genouest/biomaj-release.git && \ git clone https://github.com/genouest/biomaj-data.git ENV BIOMAJ_CONFIG=/etc/biomaj/config.yml From 0105e34c9be02a15adf87c8a869943803807c6a9 Mon Sep 17 00:00:00 2001 From: remy siminel Date: Fri, 19 Apr 2024 14:09:37 +0200 Subject: [PATCH 48/74] fix13 --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index bfbd11b..dec485d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,7 @@ ENV BIOMAJ_RELEASE="shahikorma-v13" RUN git clone https://github.com/genouest/biomaj-core.git && \ git clone https://github.com/genouest/biomaj-zipkin.git && \ - git clone https://github.com/genouest/biomaj-user-shebanger.git && \ + git clone https://github.com/genouest/biomaj-user.git && \ git clone https://github.com/genouest/biomaj-cli.git && \ git clone https://github.com/genouest/biomaj-process.git && \ git clone https://github.com/genouest/biomaj-download.git && \ @@ -55,7 +55,7 @@ RUN buildDeps="gcc python3-dev protobuf-compiler" \ && apt-get install -y $buildDeps --no-install-recommends \ && cd /root/biomaj-core && python3 setup.py build && pip3 install --no-cache-dir . \ && cd /root/biomaj-zipkin && python3 setup.py build && pip3 install --no-cache-dir . \ - && cd /root/biomaj-user-shebanger && python3 setup.py build && pip3 install --no-cache-dir . \ + && cd /root/biomaj-user && python3 setup.py build && pip3 install --no-cache-dir . \ && cd /root/biomaj-cli && python3 setup.py build && pip3 install --no-cache-dir . \ && cd /root/biomaj-process/biomaj_process/message && protoc --python_out=. procmessage.proto \ && cd /root/biomaj-process && python3 setup.py build && pip3 install --no-cache-dir . \ From 27b0e321a5ddaf739fa35a17596fdb832055fb44 Mon Sep 17 00:00:00 2001 From: Remy Siminel <95074810+rsiminel@users.noreply.github.com> Date: Thu, 25 Apr 2024 14:07:58 +0200 Subject: [PATCH 49/74] lint --- Dockerfile | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index dec485d..38c7830 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,14 +39,14 @@ ENV BIOMAJ_CONFIG=/etc/biomaj/config.yml RUN mkdir -p /var/log/biomaj RUN pip3 install --no-cache-dir setuptools --upgrade && \ -pip3 install --no-cache-dir greenlet==0.4.17 && \ -pip3 install --no-cache-dir gevent==1.4.0 && \ -pip3 install --no-cache-dir graypy && \ -pip3 install --no-cache-dir pymongo==3.12.3 && \ -pip3 install --no-cache-dir redis==3.5.3 && \ -pip3 install --no-cache-dir wheel && \ -pip3 install --no-cache-dir PyYAML==5.4.1 && \ -python3 -m pip install --no-cache-dir ftputil + pip3 install --no-cache-dir greenlet==0.4.17 && \ + pip3 install --no-cache-dir gevent==1.4.0 && \ + pip3 install --no-cache-dir graypy && \ + pip3 install --no-cache-dir pymongo==3.12.3 && \ + pip3 install --no-cache-dir redis==3.5.3 && \ + pip3 install --no-cache-dir wheel && \ + pip3 install --no-cache-dir PyYAML==5.4.1 && \ + python3 -m pip install --no-cache-dir ftputil ENV SUDO_FORCE_REMOVE=yes RUN buildDeps="gcc python3-dev protobuf-compiler" \ From 5465e7797f6cbe779b18946135a6a607eb107e9a Mon Sep 17 00:00:00 2001 From: Remy Siminel <95074810+rsiminel@users.noreply.github.com> Date: Thu, 25 Apr 2024 14:10:50 +0200 Subject: [PATCH 50/74] update secrets --- .github/workflows/dockerimage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml index 5859ea8..f5a5971 100644 --- a/.github/workflows/dockerimage.yml +++ b/.github/workflows/dockerimage.yml @@ -49,7 +49,7 @@ jobs: - name: Log in to Docker Hub uses: docker/login-action@v3 with: - username: ${{ secrets.GHR_USRNAME }} + username: ${{ secrets.GHR_USER }} password: ${{ secrets.GHR_TOKEN }} - name: Extract metadata (tags, labels) for Docker From 570d535603e1b5e39f31fef01974219f35908227 Mon Sep 17 00:00:00 2001 From: mboudet Date: Thu, 25 Apr 2024 18:02:24 +0200 Subject: [PATCH 51/74] Update Dockerfile --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 38c7830..f3b5660 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,7 +38,8 @@ ENV BIOMAJ_CONFIG=/etc/biomaj/config.yml RUN mkdir -p /var/log/biomaj -RUN pip3 install --no-cache-dir setuptools --upgrade && \ +RUN pip3 install --no-cache-dir pip --upgrade && \ + pip3 install --no-cache-dir setuptools --upgrade && \ pip3 install --no-cache-dir greenlet==0.4.17 && \ pip3 install --no-cache-dir gevent==1.4.0 && \ pip3 install --no-cache-dir graypy && \ From 0807e07a969a8a994576511952ddd060f3a98367 Mon Sep 17 00:00:00 2001 From: mboudet Date: Fri, 26 Apr 2024 16:10:31 +0200 Subject: [PATCH 52/74] Testing more sleep --- .github/workflows/dockerimage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml index f5a5971..6e888cc 100644 --- a/.github/workflows/dockerimage.yml +++ b/.github/workflows/dockerimage.yml @@ -35,7 +35,7 @@ jobs: docker compose up -d - name: Execute test script run: | - sleep 20 + sleep 60 ./docker-test.sh local - name: Shut down Docker containers run: | From e4a4579a3f1559aeb5904bbcd7cb006c65e74d9e Mon Sep 17 00:00:00 2001 From: mboudet Date: Fri, 26 Apr 2024 16:29:21 +0200 Subject: [PATCH 53/74] Update dockerimage.yml logs? --- .github/workflows/dockerimage.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml index 6e888cc..a0e56e2 100644 --- a/.github/workflows/dockerimage.yml +++ b/.github/workflows/dockerimage.yml @@ -36,6 +36,7 @@ jobs: - name: Execute test script run: | sleep 60 + docker compose logs ./docker-test.sh local - name: Shut down Docker containers run: | From 387df45b4f57d91abb80bf17593d6e538262b699 Mon Sep 17 00:00:00 2001 From: mboudet Date: Fri, 26 Apr 2024 15:57:39 +0000 Subject: [PATCH 54/74] Fix protobug version --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index f3b5660..d1b5502 100644 --- a/Dockerfile +++ b/Dockerfile @@ -47,6 +47,7 @@ RUN pip3 install --no-cache-dir pip --upgrade && \ pip3 install --no-cache-dir redis==3.5.3 && \ pip3 install --no-cache-dir wheel && \ pip3 install --no-cache-dir PyYAML==5.4.1 && \ + pip3 install --no-cache-dir protobuf==3.20.3 && \ python3 -m pip install --no-cache-dir ftputil ENV SUDO_FORCE_REMOVE=yes From c542ddd9480d3700a79f365bac21daa6efa9d74f Mon Sep 17 00:00:00 2001 From: mboudet Date: Fri, 26 Apr 2024 16:10:01 +0000 Subject: [PATCH 55/74] Remove logging --- .github/workflows/dockerimage.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml index a0e56e2..f5a5971 100644 --- a/.github/workflows/dockerimage.yml +++ b/.github/workflows/dockerimage.yml @@ -35,8 +35,7 @@ jobs: docker compose up -d - name: Execute test script run: | - sleep 60 - docker compose logs + sleep 20 ./docker-test.sh local - name: Shut down Docker containers run: | From 7c44cd9553aa4cfb0a875e88bc43254b001dc0b2 Mon Sep 17 00:00:00 2001 From: remy siminel Date: Thu, 16 May 2024 11:38:58 +0200 Subject: [PATCH 56/74] restrict publish job to pushes to master branch --- .github/workflows/dockerimage.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml index f5a5971..f1b9e09 100644 --- a/.github/workflows/dockerimage.yml +++ b/.github/workflows/dockerimage.yml @@ -43,6 +43,7 @@ jobs: publish: runs-on: ubuntu-latest + if: github.ref == 'refs/heads/master' steps: - uses: actions/checkout@v4 From 350429eac9ce869c74b0752184620c0d7ccd1f69 Mon Sep 17 00:00:00 2001 From: remy siminel Date: Thu, 16 May 2024 11:46:49 +0200 Subject: [PATCH 57/74] pull from genouest/biomaj-docker/renew --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 38c7830..d1b5502 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,7 +38,8 @@ ENV BIOMAJ_CONFIG=/etc/biomaj/config.yml RUN mkdir -p /var/log/biomaj -RUN pip3 install --no-cache-dir setuptools --upgrade && \ +RUN pip3 install --no-cache-dir pip --upgrade && \ + pip3 install --no-cache-dir setuptools --upgrade && \ pip3 install --no-cache-dir greenlet==0.4.17 && \ pip3 install --no-cache-dir gevent==1.4.0 && \ pip3 install --no-cache-dir graypy && \ @@ -46,6 +47,7 @@ RUN pip3 install --no-cache-dir setuptools --upgrade && \ pip3 install --no-cache-dir redis==3.5.3 && \ pip3 install --no-cache-dir wheel && \ pip3 install --no-cache-dir PyYAML==5.4.1 && \ + pip3 install --no-cache-dir protobuf==3.20.3 && \ python3 -m pip install --no-cache-dir ftputil ENV SUDO_FORCE_REMOVE=yes From be7e1ba1a0186bf05a80289699bc0734d10dd51a Mon Sep 17 00:00:00 2001 From: remy siminel Date: Thu, 16 May 2024 12:03:17 +0200 Subject: [PATCH 58/74] update debian to bullseye in dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d1b5502..29f7fdd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:buster +FROM debian:bullseye WORKDIR /root ENV BIOMAJ_CONFIG=/root/config.yml From ab5281036917215126a08a2b2a3d2e7e56cebc64 Mon Sep 17 00:00:00 2001 From: remy siminel Date: Tue, 21 May 2024 17:25:19 +0200 Subject: [PATCH 59/74] prometheus version update --- Dockerfile | 2 +- biomaj-config/prometheus_v2.yml | 22 ++++++++++++++++++++++ docker-compose.yml | 2 +- 3 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 biomaj-config/prometheus_v2.yml diff --git a/Dockerfile b/Dockerfile index 29f7fdd..d1b5502 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:bullseye +FROM debian:buster WORKDIR /root ENV BIOMAJ_CONFIG=/root/config.yml diff --git a/biomaj-config/prometheus_v2.yml b/biomaj-config/prometheus_v2.yml new file mode 100644 index 0000000..dd63979 --- /dev/null +++ b/biomaj-config/prometheus_v2.yml @@ -0,0 +1,22 @@ +global: + scrape_interval: 15s # By default, scrape targets every 15 seconds. + evaluation_interval: 15s # By default, scrape targets every 15 seconds. + # scrape_timeout is set to the global default (10s). + + # Attach these extra labels to all timeseries collected by this Prometheus + # instance. + external_labels: + monitor: 'biomaj' + +# A scrape configuration containing exactly one endpoint to scrape: +# Here it's Prometheus itself. +scrape_configs: + # The job name is added as a label `job=` to any timeseries scraped + # from this config. + - job_name: 'biomaj' + consul_sd_configs: + - server: 'biomaj-consul:8500' + services: + - biomaj-daemon + - biomaj-process + - biomaj-download diff --git a/docker-compose.yml b/docker-compose.yml index 538f848..4283754 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -38,7 +38,7 @@ biomaj-prometheus: - image: prom/prometheus:v1.8.2 + image: prom/prometheus:v2.3.0 ports: - "9090:9090" volumes: From 33cf22ba94dbf480ddafae540f97a2793135295a Mon Sep 17 00:00:00 2001 From: remy siminel Date: Wed, 22 May 2024 10:42:38 +0200 Subject: [PATCH 60/74] prometheus version update fix --- docker-compose.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4283754..b959964 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,9 +36,8 @@ - "8600:53" command: agent -dev -client 0.0.0.0 -log-level info - biomaj-prometheus: - image: prom/prometheus:v2.3.0 + image: prom/prometheus:v2.10.0 ports: - "9090:9090" volumes: @@ -46,8 +45,8 @@ depends_on: - biomaj-consul command: - - '-storage.local.retention=720h' - - '-config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.retention.time=720h' + - '--config.file=/etc/prometheus/prometheus.yml' biomaj-influxdb: image: influxdb @@ -98,7 +97,6 @@ - biomaj-redis command: ["/startup.sh", "gunicorn", "-b", "0.0.0.0:5000", "--log-config", "/etc/biomaj/production.ini biomaj_user.wsgi:app"] - biomaj-download-web: image: quay.io/genouest/biomaj build: From 700e9826bf415c60fd4109b301037da5da0a49b6 Mon Sep 17 00:00:00 2001 From: remy siminel Date: Wed, 22 May 2024 11:55:13 +0200 Subject: [PATCH 61/74] debian bullseye update fix --- Dockerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index d1b5502..f29de0b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:buster +FROM debian:bullseye WORKDIR /root ENV BIOMAJ_CONFIG=/root/config.yml @@ -41,7 +41,7 @@ RUN mkdir -p /var/log/biomaj RUN pip3 install --no-cache-dir pip --upgrade && \ pip3 install --no-cache-dir setuptools --upgrade && \ pip3 install --no-cache-dir greenlet==0.4.17 && \ - pip3 install --no-cache-dir gevent==1.4.0 && \ + # pip3 install --no-cache-dir gevent==1.4.0 && \ pip3 install --no-cache-dir graypy && \ pip3 install --no-cache-dir pymongo==3.12.3 && \ pip3 install --no-cache-dir redis==3.5.3 && \ @@ -75,9 +75,9 @@ RUN buildDeps="gcc python3-dev protobuf-compiler" \ && rm -rf /var/lib/apt/lists/* -RUN pip3 uninstall -y gunicorn && pip3 install --no-cache-dir gunicorn==19.9.0 && \ - pip3 uninstall -y greenlet && pip3 install --no-cache-dir greenlet==0.4.15 && \ - pip3 uninstall -y gevent && pip3 install --no-cache-dir gevent==1.3.7 +RUN pip3 uninstall -y gunicorn && pip3 install --no-cache-dir gunicorn==19.9.0 +# RUN pip3 uninstall -y greenlet && pip3 install --no-cache-dir greenlet==0.4.15 && \ +# pip3 uninstall -y gevent && pip3 install --no-cache-dir gevent==1.3.7 From d92ddf9a62d4008897189c34116909c3937cff62 Mon Sep 17 00:00:00 2001 From: remy siminel Date: Wed, 22 May 2024 15:14:42 +0200 Subject: [PATCH 62/74] implement [this](https://github.com/genouest/biomaj-docker/pull/3#discussion_r1603586069) --- Dockerfile | 28 +++++----- Dockerfile-deb11 | 141 ----------------------------------------------- 2 files changed, 14 insertions(+), 155 deletions(-) delete mode 100644 Dockerfile-deb11 diff --git a/Dockerfile b/Dockerfile index f29de0b..cd50a0b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -55,20 +55,20 @@ RUN buildDeps="gcc python3-dev protobuf-compiler" \ && set -x \ && apt-get update \ && apt-get install -y $buildDeps --no-install-recommends \ - && cd /root/biomaj-core && python3 setup.py build && pip3 install --no-cache-dir . \ - && cd /root/biomaj-zipkin && python3 setup.py build && pip3 install --no-cache-dir . \ - && cd /root/biomaj-user && python3 setup.py build && pip3 install --no-cache-dir . \ - && cd /root/biomaj-cli && python3 setup.py build && pip3 install --no-cache-dir . \ - && cd /root/biomaj-process/biomaj_process/message && protoc --python_out=. procmessage.proto \ - && cd /root/biomaj-process && python3 setup.py build && pip3 install --no-cache-dir . \ - && cd /root/biomaj-download/biomaj_download/message && protoc --python_out=. downmessage.proto \ - && cd /root/biomaj-download && python3 setup.py build && pip3 install --no-cache-dir . \ - && cd /root/biomaj && python3 setup.py build && pip3 install --no-cache-dir . \ - && cd /root/biomaj-daemon && python3 setup.py build && pip3 install --no-cache-dir . \ - && cd /root/biomaj-watcher && pip3 install --no-cache-dir -r requirements.txt && pip3 install --no-cache-dir . \ - && cd /root/biomaj-ftp && python3 setup.py build && pip3 install --no-cache-dir . \ - && cd /root/biomaj-release && python3 setup.py build && pip3 install --no-cache-dir . \ - && cd /root/biomaj-data && python3 setup.py build && pip3 install --no-cache-dir . \ + && pip install git+https://github.com/genouest/biomaj-core.git \ + && pip install git+https://github.com/genouest/biomaj-zipkin.git \ + && pip install git+https://github.com/genouest/biomaj-user.git \ + && pip install git+https://github.com/genouest/biomaj-cli.git \ + # && cd /root/biomaj-process/biomaj_process/message && protoc --python_out=. procmessage.proto \ + && pip install git+https://github.com/genouest/biomaj-process.git \ + # && cd /root/biomaj-download/biomaj_download/message && protoc --python_out=. downmessage.proto \ + && pip install git+https://github.com/genouest/biomaj-download.git \ + && pip install git+https://github.com/genouest/biomaj.git \ + && pip install git+https://github.com/genouest/biomaj-daemon.git \ + && pip install git+https://github.com/genouest/biomaj-watcher.git \ + && pip install git+https://github.com/genouest/biomaj-ftp.git \ + && pip install git+https://github.com/genouest/biomaj-release.git \ + && pip install git+https://github.com/genouest/biomaj-data.git \ && apt-get install --no-install-recommends -y wget bzip2 ca-certificates curl git nano python3-markupsafe python3-bcrypt python3-yapsy\ && apt-get purge -y --auto-remove $buildDeps \ && apt-get clean \ diff --git a/Dockerfile-deb11 b/Dockerfile-deb11 deleted file mode 100644 index 434ade0..0000000 --- a/Dockerfile-deb11 +++ /dev/null @@ -1,141 +0,0 @@ -FROM debian:bullseye - -WORKDIR /root -ENV BIOMAJ_CONFIG=/root/config.yml -ENV prometheus_multiproc_dir=/tmp/biomaj-prometheus-multiproc - -RUN rm -rf /tmp/biomaj-prometheus-multiproc -RUN mkdir -p /tmp/biomaj-prometheus-multiproc - -RUN apt-get update && apt-get install -y apt-transport-https curl libcurl4-openssl-dev python3-pycurl python3-setuptools python3-pip git unzip bzip2 ca-certificates jq --no-install-recommends - -# Install docker to allow docker execution from process-message -RUN buildDeps='gnupg2 dirmngr software-properties-common' \ - && set -x \ - && apt-get install -y $buildDeps --no-install-recommends \ - && curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - \ - && add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable" \ - && apt-get update \ - && apt-get install -y docker-ce-cli \ - && apt-get purge -y --auto-remove $buildDeps - -ENV BIOMAJ_RELEASE="shahikorma-v14-deb11" - -RUN git clone https://github.com/genouest/biomaj-core.git -#RUN easy_install3 pip -RUN pip3 install setuptools --upgrade - -RUN git clone https://github.com/genouest/biomaj-zipkin.git - -RUN git clone https://github.com/genouest/biomaj-user.git - -RUN git clone https://github.com/genouest/biomaj-cli.git - -RUN git clone https://github.com/genouest/biomaj-process.git - -RUN git clone https://github.com/genouest/biomaj-download.git - -RUN git clone https://github.com/genouest/biomaj.git && echo "Install biomaj" - -RUN git clone https://github.com/genouest/biomaj-daemon.git && echo "Install daemon" - -RUN git clone https://github.com/genouest/biomaj-watcher.git && echo "Install biomaj-watcher" - -RUN git clone https://github.com/genouest/biomaj-ftp.git - -RUN git clone https://github.com/genouest/biomaj-release.git - -RUN git clone https://github.com/genouest/biomaj-data.git - -ENV BIOMAJ_CONFIG=/etc/biomaj/config.yml - -RUN mkdir -p /var/log/biomaj - -#RUN pip3 install greenlet==0.4.17 -#RUN pip3 install gevent==1.4.0 -RUN pip3 install graypy -#RUN pip3 install pymongo==3.12.3 -#RUN pip3 install redis==3.5.3 -RUN pip3 install wheel -#RUN pip3 install PyYAML==5.3.1 - -ENV SUDO_FORCE_REMOVE=yes -RUN buildDeps='gcc python3-dev protobuf-compiler' \ - && set -x \ - && apt-get install -y $buildDeps --no-install-recommends \ - && cd /root/biomaj-core && python3 setup.py build && pip3 install . \ - && cd /root/biomaj-zipkin && python3 setup.py build && pip3 install . \ - && cd /root/biomaj-user && python3 setup.py build && pip3 install . \ - && cd /root/biomaj-cli && python3 setup.py build && pip3 install . \ - && cd /root/biomaj-process/biomaj_process/message && protoc --python_out=. procmessage.proto \ - && cd /root/biomaj-process && python3 setup.py build && pip3 install . \ - && cd /root/biomaj-download/biomaj_download/message && protoc --python_out=. downmessage.proto \ - && cd /root/biomaj-download && python3 setup.py build && pip3 install . \ - && cd /root/biomaj && python3 setup.py build && pip3 install . \ - && cd /root/biomaj-daemon && python3 setup.py build && pip3 install . \ - && cd /root/biomaj-watcher && pip3 install -r requirements.txt && pip3 install . \ - && cd /root/biomaj-ftp && python3 setup.py build && pip3 install . \ - && cd /root/biomaj-release && python3 setup.py build && pip3 install . \ - && cd /root/biomaj-data && python3 setup.py build && pip3 install . \ - && apt-get install -y wget bzip2 ca-certificates curl git \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* \ - && apt-get purge -y --auto-remove $buildDeps - - - -#RUN apt-get update --fix-missing && \ -# apt-get install -y wget bzip2 ca-certificates curl git && \ -# apt-get clean && \ -# rm -rf /var/lib/apt/lists/* -#RUN pip3 uninstall -y gunicorn && pip3 install gunicorn==19.9.0 -#RUN pip3 uninstall -y greenlet && pip3 install greenlet==0.4.15 -#RUN pip3 uninstall -y gevent && pip3 install gevent==1.3.7 - - - -#Conda installation and give write permissions to conda folder -RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh && \ - /bin/bash ~/miniconda.sh -b -p /opt/conda && \ - rm ~/miniconda.sh && \ - /opt/conda/bin/conda config --add channels r && \ - /opt/conda/bin/conda config --add channels bioconda && \ - /opt/conda/bin/conda upgrade -y conda && \ - chmod 777 -R /opt/conda/ - - - - -RUN mkdir /data /config - -ENV PATH=$PATH:/opt/conda/bin - -#RUN conda config --add channels r -#RUN conda config --add channels bioconda -#RUN conda upgrade -y conda - -VOLUME ["/data", "/config"] - -RUN mkdir -p /var/lib/biomaj/data - -COPY biomaj-config/config.yml /etc/biomaj/config.yml -COPY biomaj-config/global.properties /etc/biomaj/global.properties -COPY biomaj-config/production.ini /etc/biomaj/production.ini -COPY biomaj-config/gunicorn_conf.py /etc/biomaj/gunicorn_conf.py -COPY watcher.sh /root/watcher.sh - -# Local test configuration -RUN mkdir -p /etc/biomaj/conf.d -RUN mkdir -p /var/log/biomaj -RUN mkdir -p /etc/biomaj/process.d -RUN mkdir -p /var/cache/biomaj -RUN mkdir -p /var/run/biomaj -COPY test-local/etc/biomaj/global_local.properties /etc/biomaj/global_local.properties -COPY test-local/etc/biomaj/conf.d/alu.properties /etc/biomaj/conf.d/alu.properties - -# Plugins -RUN cd /var/lib/biomaj && git clone https://github.com/genouest/biomaj-plugins.git plugins - -ADD https://github.com/ufoscout/docker-compose-wait/releases/download/2.6.0/wait /wait -RUN chmod +x /wait -COPY startup.sh /startup.sh From 85130697270f26ee73dc7c94ad9ff7061d039957 Mon Sep 17 00:00:00 2001 From: remy siminel Date: Wed, 22 May 2024 15:37:57 +0200 Subject: [PATCH 63/74] remove prometheus_v2.yml --- biomaj-config/prometheus_v2.yml | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 biomaj-config/prometheus_v2.yml diff --git a/biomaj-config/prometheus_v2.yml b/biomaj-config/prometheus_v2.yml deleted file mode 100644 index dd63979..0000000 --- a/biomaj-config/prometheus_v2.yml +++ /dev/null @@ -1,22 +0,0 @@ -global: - scrape_interval: 15s # By default, scrape targets every 15 seconds. - evaluation_interval: 15s # By default, scrape targets every 15 seconds. - # scrape_timeout is set to the global default (10s). - - # Attach these extra labels to all timeseries collected by this Prometheus - # instance. - external_labels: - monitor: 'biomaj' - -# A scrape configuration containing exactly one endpoint to scrape: -# Here it's Prometheus itself. -scrape_configs: - # The job name is added as a label `job=` to any timeseries scraped - # from this config. - - job_name: 'biomaj' - consul_sd_configs: - - server: 'biomaj-consul:8500' - services: - - biomaj-daemon - - biomaj-process - - biomaj-download From 0e31d03c7a07dcb6226a23258305bce83082417f Mon Sep 17 00:00:00 2001 From: remy siminel Date: Wed, 22 May 2024 15:50:17 +0200 Subject: [PATCH 64/74] fix --- Dockerfile | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index cd50a0b..50780df 100644 --- a/Dockerfile +++ b/Dockerfile @@ -55,20 +55,20 @@ RUN buildDeps="gcc python3-dev protobuf-compiler" \ && set -x \ && apt-get update \ && apt-get install -y $buildDeps --no-install-recommends \ - && pip install git+https://github.com/genouest/biomaj-core.git \ - && pip install git+https://github.com/genouest/biomaj-zipkin.git \ - && pip install git+https://github.com/genouest/biomaj-user.git \ - && pip install git+https://github.com/genouest/biomaj-cli.git \ + && pip install git+https://github.com/genouest/biomaj-core.git --no-cache-dir \ + && pip install git+https://github.com/genouest/biomaj-zipkin.git --no-cache-dir \ + && pip install git+https://github.com/genouest/biomaj-user.git --no-cache-dir \ + && pip install git+https://github.com/genouest/biomaj-cli.git --no-cache-dir \ # && cd /root/biomaj-process/biomaj_process/message && protoc --python_out=. procmessage.proto \ - && pip install git+https://github.com/genouest/biomaj-process.git \ + && pip install git+https://github.com/genouest/biomaj-process.git --no-cache-dir \ # && cd /root/biomaj-download/biomaj_download/message && protoc --python_out=. downmessage.proto \ - && pip install git+https://github.com/genouest/biomaj-download.git \ - && pip install git+https://github.com/genouest/biomaj.git \ - && pip install git+https://github.com/genouest/biomaj-daemon.git \ - && pip install git+https://github.com/genouest/biomaj-watcher.git \ - && pip install git+https://github.com/genouest/biomaj-ftp.git \ - && pip install git+https://github.com/genouest/biomaj-release.git \ - && pip install git+https://github.com/genouest/biomaj-data.git \ + && pip install git+https://github.com/genouest/biomaj-download.git --no-cache-dir \ + && pip install git+https://github.com/genouest/biomaj.git --no-cache-dir \ + && pip install git+https://github.com/genouest/biomaj-daemon.git --no-cache-dir \ + && pip install git+https://github.com/genouest/biomaj-watcher.git --no-cache-dir \ + && pip install git+https://github.com/genouest/biomaj-ftp.git --no-cache-dir \ + && pip install git+https://github.com/genouest/biomaj-release.git --no-cache-dir \ + && pip install git+https://github.com/genouest/biomaj-data.git --no-cache-dir \ && apt-get install --no-install-recommends -y wget bzip2 ca-certificates curl git nano python3-markupsafe python3-bcrypt python3-yapsy\ && apt-get purge -y --auto-remove $buildDeps \ && apt-get clean \ From 6cc3488865e3eaf45b0a23ddb16f18a81ae81dde Mon Sep 17 00:00:00 2001 From: remy siminel Date: Thu, 23 May 2024 17:35:32 +0200 Subject: [PATCH 65/74] remove redundant dockerfile instructions --- Dockerfile | 23 ++--------------------- 1 file changed, 2 insertions(+), 21 deletions(-) diff --git a/Dockerfile b/Dockerfile index 50780df..7a5d9e6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,19 +21,6 @@ RUN buildDeps='gnupg2 dirmngr' \ ENV BIOMAJ_RELEASE="shahikorma-v13" -RUN git clone https://github.com/genouest/biomaj-core.git && \ - git clone https://github.com/genouest/biomaj-zipkin.git && \ - git clone https://github.com/genouest/biomaj-user.git && \ - git clone https://github.com/genouest/biomaj-cli.git && \ - git clone https://github.com/genouest/biomaj-process.git && \ - git clone https://github.com/genouest/biomaj-download.git && \ - git clone https://github.com/genouest/biomaj.git && \ - git clone https://github.com/genouest/biomaj-daemon.git && \ - git clone https://github.com/genouest/biomaj-watcher.git && \ - git clone https://github.com/genouest/biomaj-ftp.git && \ - git clone https://github.com/genouest/biomaj-release.git && \ - git clone https://github.com/genouest/biomaj-data.git - ENV BIOMAJ_CONFIG=/etc/biomaj/config.yml RUN mkdir -p /var/log/biomaj @@ -41,7 +28,6 @@ RUN mkdir -p /var/log/biomaj RUN pip3 install --no-cache-dir pip --upgrade && \ pip3 install --no-cache-dir setuptools --upgrade && \ pip3 install --no-cache-dir greenlet==0.4.17 && \ - # pip3 install --no-cache-dir gevent==1.4.0 && \ pip3 install --no-cache-dir graypy && \ pip3 install --no-cache-dir pymongo==3.12.3 && \ pip3 install --no-cache-dir redis==3.5.3 && \ @@ -59,9 +45,9 @@ RUN buildDeps="gcc python3-dev protobuf-compiler" \ && pip install git+https://github.com/genouest/biomaj-zipkin.git --no-cache-dir \ && pip install git+https://github.com/genouest/biomaj-user.git --no-cache-dir \ && pip install git+https://github.com/genouest/biomaj-cli.git --no-cache-dir \ - # && cd /root/biomaj-process/biomaj_process/message && protoc --python_out=. procmessage.proto \ + && cd /root/biomaj-process/biomaj_process/message && protoc --python_out=. procmessage.proto \ && pip install git+https://github.com/genouest/biomaj-process.git --no-cache-dir \ - # && cd /root/biomaj-download/biomaj_download/message && protoc --python_out=. downmessage.proto \ + && cd /root/biomaj-download/biomaj_download/message && protoc --python_out=. downmessage.proto \ && pip install git+https://github.com/genouest/biomaj-download.git --no-cache-dir \ && pip install git+https://github.com/genouest/biomaj.git --no-cache-dir \ && pip install git+https://github.com/genouest/biomaj-daemon.git --no-cache-dir \ @@ -76,9 +62,6 @@ RUN buildDeps="gcc python3-dev protobuf-compiler" \ RUN pip3 uninstall -y gunicorn && pip3 install --no-cache-dir gunicorn==19.9.0 -# RUN pip3 uninstall -y greenlet && pip3 install --no-cache-dir greenlet==0.4.15 && \ -# pip3 uninstall -y gevent && pip3 install --no-cache-dir gevent==1.3.7 - #Conda installation and give write permissions to conda folder @@ -91,8 +74,6 @@ RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86 chmod 777 -R /opt/conda/ - - RUN mkdir /data /config ENV PATH=$PATH:/opt/conda/bin From ac2114c5bd8722adef495bb8d446893d95b94628 Mon Sep 17 00:00:00 2001 From: remy siminel Date: Thu, 23 May 2024 17:42:40 +0200 Subject: [PATCH 66/74] fix --- Dockerfile | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Dockerfile b/Dockerfile index 7a5d9e6..9390492 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,6 +21,19 @@ RUN buildDeps='gnupg2 dirmngr' \ ENV BIOMAJ_RELEASE="shahikorma-v13" +RUN git clone https://github.com/genouest/biomaj-core.git && \ + git clone https://github.com/genouest/biomaj-zipkin.git && \ + git clone https://github.com/genouest/biomaj-user.git && \ + git clone https://github.com/genouest/biomaj-cli.git && \ + git clone https://github.com/genouest/biomaj-process.git && \ + git clone https://github.com/genouest/biomaj-download.git && \ + git clone https://github.com/genouest/biomaj.git && \ + git clone https://github.com/genouest/biomaj-daemon.git && \ + git clone https://github.com/genouest/biomaj-watcher.git && \ + git clone https://github.com/genouest/biomaj-ftp.git && \ + git clone https://github.com/genouest/biomaj-release.git && \ + git clone https://github.com/genouest/biomaj-data.git + ENV BIOMAJ_CONFIG=/etc/biomaj/config.yml RUN mkdir -p /var/log/biomaj From 8d8d9e1f65a7e0f4170b4d30683af7d28edf18f3 Mon Sep 17 00:00:00 2001 From: remy siminel Date: Thu, 23 May 2024 17:47:35 +0200 Subject: [PATCH 67/74] revert prometheus version update --- docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index b959964..7c507fd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,7 +37,7 @@ command: agent -dev -client 0.0.0.0 -log-level info biomaj-prometheus: - image: prom/prometheus:v2.10.0 + image: prom/prometheus:v1.8.2 ports: - "9090:9090" volumes: @@ -45,8 +45,8 @@ depends_on: - biomaj-consul command: - - '--storage.tsdb.retention.time=720h' - - '--config.file=/etc/prometheus/prometheus.yml' + - '-storage.local.retention=720h' + - '-config.file=/etc/prometheus/prometheus.yml' biomaj-influxdb: image: influxdb From 3cc31f8e93b203ff6b54ca7805dd7727d4e565f0 Mon Sep 17 00:00:00 2001 From: remy siminel Date: Thu, 30 May 2024 15:04:56 +0200 Subject: [PATCH 68/74] remove dockerfile git clones --- Dockerfile | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9390492..e7dfcf7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,18 +21,8 @@ RUN buildDeps='gnupg2 dirmngr' \ ENV BIOMAJ_RELEASE="shahikorma-v13" -RUN git clone https://github.com/genouest/biomaj-core.git && \ - git clone https://github.com/genouest/biomaj-zipkin.git && \ - git clone https://github.com/genouest/biomaj-user.git && \ - git clone https://github.com/genouest/biomaj-cli.git && \ - git clone https://github.com/genouest/biomaj-process.git && \ - git clone https://github.com/genouest/biomaj-download.git && \ - git clone https://github.com/genouest/biomaj.git && \ - git clone https://github.com/genouest/biomaj-daemon.git && \ - git clone https://github.com/genouest/biomaj-watcher.git && \ - git clone https://github.com/genouest/biomaj-ftp.git && \ - git clone https://github.com/genouest/biomaj-release.git && \ - git clone https://github.com/genouest/biomaj-data.git +RUN git clone https://github.com/genouest/biomaj-process.git && \ + git clone https://github.com/genouest/biomaj-download.git ENV BIOMAJ_CONFIG=/etc/biomaj/config.yml From 4328a5b07f4c01c88e7823fe7153eeb206249712 Mon Sep 17 00:00:00 2001 From: mboudet Date: Mon, 22 Jul 2024 16:50:14 +0200 Subject: [PATCH 69/74] Update prometheus version --- CHANGES.md | 6 ++++-- docker-compose.yml | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 9612dd4..b346a9e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,7 +2,10 @@ Notes: requires mongo 3.x -## shahikorma +## v14 - TBR + +/!\ Breaking: Upgrading promoetheus version from 2.45.6. Data will be lost in the upgrade. +* Various fixes and cleanup ## v13 - 2022-04-28 @@ -84,4 +87,3 @@ Notes: requires mongo 3.x * Add naming in versions * Force pika library version 0.11.2 in biomaj-download and biomaj-process (latest not working) - diff --git a/docker-compose.yml b/docker-compose.yml index 7c507fd..4a9d474 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,7 +37,7 @@ command: agent -dev -client 0.0.0.0 -log-level info biomaj-prometheus: - image: prom/prometheus:v1.8.2 + image: prom/prometheus:v2.45.6 ports: - "9090:9090" volumes: @@ -45,8 +45,8 @@ depends_on: - biomaj-consul command: - - '-storage.local.retention=720h' - - '-config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.retention.time=720h' + - '--config.file=/etc/prometheus/prometheus.yml' biomaj-influxdb: image: influxdb From 72d90df6d74b3d32a6c3748fd444d16a748cd54b Mon Sep 17 00:00:00 2001 From: mboudet Date: Mon, 22 Jul 2024 17:59:31 +0200 Subject: [PATCH 70/74] Update docker-compose.yml --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4a9d474..4d2ea71 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -132,7 +132,7 @@ - biomaj-consul - biomaj-rabbitmq - biomaj-redis - command: ["/startup.sh", "python3", "/root/biomaj-download/bin/biomaj_download_consumer.py"] + command: ["/startup.sh", "biomaj_download_consumer.py"] biomaj-process-web: image: quay.io/genouest/biomaj From 8d4ceeda96f30e24c2be07821b01b54d33eebb7b Mon Sep 17 00:00:00 2001 From: mboudet Date: Mon, 22 Jul 2024 18:51:01 +0200 Subject: [PATCH 71/74] Update docker-compose.yml --- docker-compose.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4d2ea71..6d6a5c1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -173,7 +173,7 @@ - biomaj-rabbitmq - biomaj-redis - biomaj-elasticsearch - command: ["/startup.sh", "python3", "/root/biomaj-process/bin/biomaj_process_consumer.py"] + command: ["/startup.sh", "biomaj_process_consumer.py"] biomaj-daemon-web: image: quay.io/genouest/biomaj @@ -222,7 +222,7 @@ - biomaj-user-web - biomaj-download-web - biomaj-process-web - command: ["/startup.sh", "python3", "/root/biomaj-daemon/bin/biomaj_daemon_consumer.py"] + command: ["/startup.sh", "biomaj_daemon_consumer.py"] biomaj-watcher-web: image: quay.io/genouest/biomaj @@ -271,7 +271,7 @@ - biomaj-elasticsearch ports: - "2121:21" - command: ["/startup.sh", "python3", "/root/biomaj-ftp/bin/biomaj_ftp_service.py"] + command: ["/startup.sh", "biomaj_ftp_service.py"] biomaj-release-web: image: quay.io/genouest/biomaj @@ -289,7 +289,7 @@ - biomaj-redis - biomaj-daemon-web - biomaj-daemon-message - command: ["/startup.sh", "python3", "/root/biomaj-release/bin/biomaj_release.py"] + command: ["/startup.sh", "biomaj_release.py"] biomaj-expose: image: quay.io/genouest/biomaj-expose From a37257b9023f0c81856b5c9e7c383236e8b0f48f Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Mon, 19 Aug 2024 12:00:35 +0200 Subject: [PATCH 72/74] Update CHANGES.md --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index b346a9e..b2a2249 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,7 +4,7 @@ Notes: requires mongo 3.x ## v14 - TBR -/!\ Breaking: Upgrading promoetheus version from 2.45.6. Data will be lost in the upgrade. +/!\ Breaking: Upgrading prometheus version from 2.45.6. Data will be lost in the upgrade. * Various fixes and cleanup ## v13 - 2022-04-28 From 374664efede39553b57121bc1890c608f3196740 Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Mon, 19 Aug 2024 13:45:20 +0200 Subject: [PATCH 73/74] Update Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e7dfcf7..6c66243 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,7 +19,7 @@ RUN buildDeps='gnupg2 dirmngr' \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -ENV BIOMAJ_RELEASE="shahikorma-v13" +ENV BIOMAJ_RELEASE="14" RUN git clone https://github.com/genouest/biomaj-process.git && \ git clone https://github.com/genouest/biomaj-download.git From 99f5ebcb954fbf55dd4e1652e117d85bd84dd7fc Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Mon, 19 Aug 2024 14:37:43 +0200 Subject: [PATCH 74/74] change secret --- .github/workflows/dockerimage.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml index f1b9e09..cf024d7 100644 --- a/.github/workflows/dockerimage.yml +++ b/.github/workflows/dockerimage.yml @@ -50,8 +50,8 @@ jobs: - name: Log in to Docker Hub uses: docker/login-action@v3 with: - username: ${{ secrets.GHR_USER }} - password: ${{ secrets.GHR_TOKEN }} + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Extract metadata (tags, labels) for Docker id: meta