feat: add webserver

.github/workflows/dockerimgs.yml

@@ -0,0 +1,90 @@
+name: '[docker image]'
+
+on:
+  push:
+    branches:
+    - feat_workerFrontend
+  # release:
+  #   types:
+  #   - published
+
+jobs:
+  build-docker-img:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        dockerfile: ['./http.server.dockerfile', './http.worker.dockerfile']
+        include:
+        - dockerfile: ./http.server.dockerfile
+          dockerimg: siibra-jugex-server
+        - dockerfile: ./http.worker.dockerfile
+          dockerimg: siibra-jugex-worker
+    steps:
+    - uses: actions/checkout@v2
+    - name: 'Setup tags'
+      run: |
+        VERSION=$(grep -Po '^__version__.*?"\K[\w.]+' http_wrapper/__init__.py)
+        echo "Setting version to $VERSION"
+        echo "VERSION=$VERSION" >> $GITHUB_ENV
+
+        DOCKER_IMGSTREAM=${{ secrets.EBRAINS_DOCKER_REG_SIIBRA_TOOLBOX_NSP }}/${{ matrix.dockerimg }}
+        echo "Setting docker image stream to $DOCKER_IMGSTREAM"
+        echo "DOCKER_IMGSTREAM=$DOCKER_IMGSTREAM" >> $GITHUB_ENV
+
+    - name: 'Build docker image'
+      run: |
+        DOCKER_BUILT_TAG=$DOCKER_IMGSTREAM:$VERSION
+        echo "Building $DOCKER_BUILT_TAG"
+        docker build \
+          -t $DOCKER_BUILT_TAG \
+          -f ${{ matrix.dockerfile }} \
+          .
+        echo "Successfully built $DOCKER_BUILT_TAG"
+        echo "DOCKER_BUILT_TAG=$DOCKER_BUILT_TAG" >> $GITHUB_ENV
+
+    - name: 'Push to docker registry'
+      run: |
+        echo "Login to docker registry"
+        docker login \
+          -u '${{ secrets.EBRAINS_DOCKER_REG_SIIBRA_TOOLBOX_PUSHER_USER }}' \
+          -p '${{ secrets.EBRAINS_DOCKER_REG_SIIBRA_TOOLBOX_PUSHER_PWSD }}' \
+          ${{ secrets.EBRAINS_DOCKER_REG }}
+        echo "Pushing $DOCKER_BUILT_TAG"
+        docker push $DOCKER_BUILT_TAG
+
+    - name: 'Tag versioned tag to registry'
+      run: |
+
+        echo "Login to docker registry"
+        docker login \
+          -u '${{ secrets.EBRAINS_DOCKER_REG_SIIBRA_TOOLBOX_PUSHER_USER }}' \
+          -p '${{ secrets.EBRAINS_DOCKER_REG_SIIBRA_TOOLBOX_PUSHER_PWSD }}' \
+          ${{ secrets.EBRAINS_DOCKER_REG }}
+
+        BREAK=5
+        while [[ "$VERSION" == *"."* ]]
+        do
+          if [[ "$BREAK" == "0" ]]
+          then
+            echo "Fuse broke!"
+            exit 1
+          fi
+          VERSIONED_DOCKERTAG=$DOCKER_IMGSTREAM:$VERSION
+          echo "tagging and pushing $VERSIONED_DOCKERTAG"
+          docker tag $DOCKER_BUILT_TAG $VERSIONED_DOCKERTAG
+          docker push $VERSIONED_DOCKERTAG
+
+          echo "Push successful... Incrementing version & break"
+          VERSION=$(echo $VERSION | sed -e 's/\.\w*$//g')
+          BREAK=$(( "$BREAK" - 1 ))
+        done
+        echo "Done"
+
+
+  deploy-on-ebrains:
+    runs-on: ubuntu-latest
+    needs: build-docker-img
+    steps:
+    - name: "skip for now"
+      run: "echo skip for now"

.github/workflows/test.yml

@@ -0,0 +1,38 @@
+name: '[unit test]'
+
+on: [ push ]
+
+jobs:
+  unit-tests:
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        test_path: ['siibra_jugex', 'http_wrapper']
+        include:
+        - test_path: siibra_jugex
+          pip_install: '.'
+        - test_path: http_wrapper
+          pip_install: '-r ./http_wrapper/requirements-server.txt'
+          pip_install_extra: 'requests .'
+
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-python@v3
+      with:
+        python-version: '3.10'
+
+    - name: "Install testing libraries"
+      run: pip install pytest
+
+    - name: "Install dependencies defined in matrix.pip_install"
+      if: ${{ matrix.pip_install }}
+      run: pip install ${{ matrix.pip_install }}
+
+    - name: "Install dependencies defined in matrix.pip_install_extra"
+      if: ${{ matrix.pip_install_extra }}
+      run: pip install ${{ matrix.pip_install_extra }}
+
+    - name: "Run tests at test/${{ matrix.test_path }}"
+      run: pytest test/${{ matrix.test_path }}

.gitignore

@@ -8,3 +8,6 @@ dist
 examples/*.json
 upload_to_pypi.sh
 venv
+.vscode
+.env
+secret*

.helm/adhoc/cert.yaml

@@ -0,0 +1,42 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: siibra-jugex-certificate
+spec:
+  isCA: false
+  commonName: siibra-jugex.apps.tc.humanbrainproject.eu
+  dnsNames:
+  - siibra-jugex.apps.tc.humanbrainproject.eu
+  issuerRef:
+    kind: ClusterIssuer
+    name: letsencrypt-production-issuer-1
+  privateKey:
+    algorithm: RSA
+    encoding: PKCS1
+    size: 2048
+  renewBefore: 120h0m0s
+  secretName: siibra-jugex-prod-secret
+  usages:
+  - server auth
+---
+
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: neurogenpy-certificate
+spec:
+  isCA: false
+  commonName: ngpy.apps.hbp.eu
+  dnsNames:
+  - ngpy.apps.hbp.eu
+  issuerRef:
+    kind: ClusterIssuer
+    name: letsencrypt-production-issuer-1
+  privateKey:
+    algorithm: RSA
+    encoding: PKCS1
+    size: 2048
+  renewBefore: 120h0m0s
+  secretName: neurogenpy-prod-secret
+  usages:
+  - server auth

.helm/adhoc/configmap.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: jugex-toolbox
+data:
+  SIIBRA_TOOLBOX_LOG_DIR: /siibra-tool-log
+  SIIBRA_TOOLBOX_DATA_DIR: /siibra-tool-data
+  SIIBRA_JURGEX_CELERY_BROKER: redis://redis:6379 
+  SIIBRA_JURGEX_CELERY_RESULT: redis://redis:6379
+  SIIBRA_CACHEDIR: /siibra-tool-data/tmp

.helm/adhoc/cron.yaml

@@ -0,0 +1,30 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: cronjob-trim-old-log-usage
+spec:
+  # run two month
+  schedule: "41 1 */5 */2 *"
+  concurrencyPolicy: Replace
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+          - name: cronjob-trim-old-log-usage
+            image: docker-registry.ebrains.eu/monitoring/kubectl:v1.29.2
+            imagePullPolicy: Always
+            command:
+            - /bin/ash
+            - -c
+            # delete all files over 30 days of modification date
+            - "find /siibra-tool-log -mindepth 1 -maxdepth 1 -mtime +30 -delete"
+            volumeMounts:
+              - mountPath: /siibra-tool-log
+                name: log-volume
+
+          restartPolicy: OnFailure
+          volumes:
+          - name: log-volume
+            persistentVolumeClaim:
+              claimName: log-volume-claim-2

.helm/adhoc/deployment.yaml

@@ -0,0 +1,60 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: redis
+spec:
+  progressDeadlineSeconds: 600
+  replicas: 1
+  revisionHistoryLimit: 10
+  selector:
+    matchLabels:
+      app: redis
+  strategy:
+    rollingUpdate:
+      maxSurge: 25%
+      maxUnavailable: 25%
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: redis
+    spec:
+      containers:
+      - image: docker-registry.ebrains.eu/monitoring/redis:alpine3.17
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          exec:
+            command:
+            - redis-cli
+            - ping
+          failureThreshold: 3
+          periodSeconds: 30
+          successThreshold: 1
+          timeoutSeconds: 1
+        name: redis
+        ports:
+        - containerPort: 6379
+          protocol: TCP
+        readinessProbe:
+          exec:
+            command:
+            - redis-cli
+            - ping
+          failureThreshold: 3
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 1
+        resources:
+          limits:
+            cpu: 200m
+            memory: 1Gi
+          requests:
+            cpu: 100m
+            memory: 128Mi
+        terminationMessagePath: /dev/termination-log
+        terminationMessagePolicy: File
+      dnsPolicy: ClusterFirst
+      restartPolicy: Always
+      schedulerName: default-scheduler
+      securityContext: {}
+      terminationGracePeriodSeconds: 30

.helm/adhoc/ingress.yaml

@@ -0,0 +1,33 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: jugex-toolbox
+spec:
+  rules:
+  - host: siibra-jugex.apps.tc.humanbrainproject.eu
+    http:
+      paths:
+      - backend:
+          service:
+            name: jugex-toolbox
+            port:
+              number: 6001
+        path: /
+        pathType: ImplementationSpecific
+  - host: ngpy.apps.hbp.eu
+    http:
+      paths:
+      - backend:
+          service:
+            name: neurogenpy-toolbox
+            port:
+              number: 6001
+        path: /
+        pathType: ImplementationSpecific
+  tls:
+  - hosts:
+    - siibra-jugex.apps.tc.humanbrainproject.eu
+    secretName: siibra-jugex-prod-secret
+  - hosts:
+    - ngpy.apps.hbp.eu
+    secretName: neurogenpy-prod-secret

.helm/adhoc/pvc.yaml

@@ -0,0 +1,27 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  labels:
+    type: longhorn-pvc
+  name: data-volume-claim
+spec:
+  accessModes:
+  - ReadWriteMany
+  resources:
+    requests:
+      storage: 16Gi
+  storageClassName: longhorn-1
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  labels:
+    type: longhorn-pvc
+  name: log-volume-claim-2
+spec:
+  accessModes:
+  - ReadWriteMany
+  resources:
+    requests:
+      storage: 4Gi
+  storageClassName: longhorn-1

.helm/adhoc/service.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: redis
+  name: redis
+spec:
+  ports:
+  - port: 6379
+    protocol: TCP
+    targetPort: 6379
+  selector:
+    app: redis
+  type: ClusterIP

.helm/deploy_neurogenpy.sh

@@ -0,0 +1,16 @@
+#! /bin/bash
+
+SERVER_IMG="docker-registry.ebrains.eu/siibra-toolbox/neurogenpy"
+WORKER_IMG="docker-registry.ebrains.eu/siibra-toolbox/neurogenpy"
+SERVER_TAG="server"
+WORKER_TAG="worker"
+
+helm install neurogenpy ./toolbox \
+    --set image.serverRepository=$SERVER_IMG \
+    --set image.workerRepository=$WORKER_IMG \
+    --set image.serverTag=$SERVER_TAG \
+    --set image.workerTag=$WORKER_TAG \
+    --set workerResources.limits.memory=4Gi \
+    --set envObj.NEUROGENPY_CELERY_BROKER=redis://redis:6379 \
+    --set envObj.NEUROGENPY_CELERY_RESULT=redis://redis:6379
+