Merge branch 'main' into 200-provide-info-for-cli-login

.github/dependabot.yaml

@@ -17,7 +17,7 @@ updates:
     reviewers:
       - "neicnordic/sensitive-data-development-collaboration"
   - package-ecosystem: docker
-    directory: "/postgres"
+    directory: "/postgresql"
     schedule:
       interval: weekly
     open-pull-requests-limit: 10
@@ -52,7 +52,7 @@ updates:
     reviewers:
       - "neicnordic/sensitive-data-development-collaboration"
   - package-ecosystem: docker
-    directory: "/sda-sftp-iinbox"
+    directory: "/sda-sftp-inbox"
     schedule:
       interval: weekly
     open-pull-requests-limit: 10

.github/integration/scripts/charts/dependencies.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+set -ex
+
+YQ_VERSION="v4.20.1"
+C4GH_VERSION="$(curl -sL https://api.github.com/repos/neicnordic/crypt4gh/releases/latest | jq -r '.name')"
+
+random-string() {
+        head -c 32 /dev/urandom | base64 -w0 | tr -d '/+' | fold -w 32 | head -n 1
+}
+
+sudo curl -sLO "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64" -O /usr/bin/yq &&
+        sudo chmod +x /usr/bin/yq
+
+curl -sL https://github.com/neicnordic/crypt4gh/releases/download/"${C4GH_VERSION}"/crypt4gh_linux_x86_64.tar.gz | sudo tar -xz -C /usr/bin/ &&
+        sudo chmod +x /usr/bin/crypt4gh
+
+# secret for the crypt4gh keypair
+C4GHPASSPHRASE="$(random-string)"
+export C4GHPASSPHRASE
+crypt4gh generate -n c4gh -p "$C4GHPASSPHRASE"
+kubectl create secret generic c4gh --from-file="c4gh.sec.pem" --from-file="c4gh.pub.pem" --from-literal=passphrase="${C4GHPASSPHRASE}"
+
+# secret for the OIDC keypair
+openssl ecparam -name prime256v1 -genkey -noout -out "jwt.key"
+openssl ec -in "jwt.key" -pubout -out "jwt.pub"
+kubectl create secret generic jwk --from-file="jwt.key" --from-file="jwt.pub"
+
+## OIDC
+SELF=$(dirname "$0")
+kubectl create configmap oidc --from-file="$SELF/../../sda/oidc.py"
+
+helm repo add jetstack https://charts.jetstack.io
+helm repo add minio https://charts.min.io/
+
+helm repo update
+
+helm install \
+        cert-manager jetstack/cert-manager \
+        --namespace cert-manager \
+        --create-namespace \
+        --set installCRDs=true
+
+kubectl create namespace minio
+kubectl apply -f .github/integration/scripts/charts/dependencies.yaml
+
+## S3 storage backend
+MINIO_ACCESS="$(random-string)"
+export MINIO_ACCESS
+MINIO_SECRET="$(random-string)"
+export MINIO_SECRET
+helm install minio minio/minio \
+        --namespace minio \
+        --set rootUser="$MINIO_ACCESS",rootPassword="$MINIO_SECRET",persistence.enabled=false,mode=standalone,resources.requests.memory=128Mi
+
+PGPASSWORD="$(random-string)"
+export PGPASSWORD
+
+MQPASSWORD="$(random-string)"
+export MQPASSWORD
+
+TEST_TOKEN="$(bash .github/integration/scripts/sign_jwt.sh ES256 jwt.key)"
+export TEST_TOKEN
+
+## update values file with all credentials
+yq -i '
+.global.archive.s3AccessKey = strenv(MINIO_ACCESS) |
+.global.archive.s3SecretKey = strenv(MINIO_SECRET) |
+.global.backupArchive.s3AccessKey = strenv(MINIO_ACCESS) |
+.global.backupArchive.s3SecretKey = strenv(MINIO_SECRET) |
+.global.broker.password = strenv(MQPASSWORD) |
+.global.c4gh.passphrase = strenv(C4GHPASSPHRASE) |
+.global.db.password = strenv(PGPASSWORD) |
+.global.inbox.s3AccessKey = strenv(MINIO_ACCESS) |
+.global.inbox.s3SecretKey = strenv(MINIO_SECRET) |
+.releasetest.secrets.accessToken = strenv(TEST_TOKEN)
+' .github/integration/scripts/charts/values.yaml

.github/integration/scripts/charts/dependencies.yaml

@@ -0,0 +1,118 @@
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: selfsigned-issuer
+  namespace: cert-manager
+spec:
+  selfSigned: {}
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: selfsigned-ca
+  namespace: cert-manager
+spec:
+  isCA: true
+  commonName: selfsigned-ca
+  secretName: root-secret
+  privateKey:
+    algorithm: ECDSA
+    size: 256
+  issuerRef:
+    name: selfsigned-issuer
+    kind: Issuer
+    group: cert-manager.io
+---
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: cert-issuer
+  namespace: cert-manager
+spec:
+  ca:
+    secretName: root-secret
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: minio-cert
+  namespace: minio
+spec:
+  secretName: minio-cert
+  duration: 24h
+  commonName: minio
+  isCA: false
+  privateKey:
+    algorithm: ECDSA
+    size: 256
+  usages:
+    - server auth
+  dnsNames:
+    - localhost
+    - minio
+    - minio.minio.svc
+    - minio.minio.svc.cluster.local
+  ipAddresses:
+    - 127.0.0.1
+  issuerRef:
+    name: cert-issuer
+    # We can reference ClusterIssuers by changing the kind here.
+    # The default value is Issuer (i.e. a locally namespaced Issuer)
+    kind: ClusterIssuer
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: oidc-server
+spec:
+  selector:
+    matchLabels:
+      app: oidc-server
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: oidc-server
+    spec:
+      securityContext:
+        runAsNonRoot: true
+        runAsGroup: 1000
+        runAsUser: 1000
+        fsGroup: 1000
+      containers:
+      - name: oidc-server
+        image: neicnordic/mock-oidc:latest
+        ports:
+        - containerPort: 8080
+        env:
+          - name: PORT
+            value: "8080"
+          - name: HOST
+            value: oidc-server
+          - name: CLIENT_ID
+            value: DfCieZLuBU
+          - name: CLIENT_SECRET
+            value: DfCieZLuBU
+          - name: CLIENT_REDIRECT_URI
+            value: https://sda-auth/elixir/login
+        resources:
+          limits:
+            cpu: 250m
+            memory: 256Mi
+          requests:
+            cpu: 100m
+            memory: 128Mi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: oidc-server
+  labels:
+    app: oidc-server
+spec:
+  type: ClusterIP
+  ports:
+  - port: 8080
+    targetPort: 8080
+  selector:
+    app: oidc-server

.github/integration/scripts/charts/deploy_charts.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+set -ex
+
+if [ -z "$2" ];then
+    echo "PR number missing"
+    exit 1
+fi
+
+if [ "$1" == "sda-db" ]; then
+    ROOTPASS=$(yq e '.global.db.password' .github/integration/scripts/charts/values.yaml)
+    helm install postgres charts/sda-db \
+        --set image.tag="PR$2-postgres" \
+        --set image.pullPolicy=IfNotPresent \
+        --set global.postgresAdminPassword="$ROOTPASS" \
+        --set global.tls.enabled=false \
+        --set persistence.enabled=false \
+        --set resources=null \
+        --wait
+fi
+
+if [ "$1" == "sda-mq" ]; then
+    ADMINPASS=$(yq e '.global.broker.password' .github/integration/scripts/charts/values.yaml)
+    helm install broker charts/sda-mq \
+        --set image.tag="PR$2-rabbitmq" \
+        --set image.pullPolicy=IfNotPresent \
+        --set global.adminPassword="$ADMINPASS" \
+        --set global.adminUser=admin \
+        --set global.tls.enabled=false \
+        --set persistence.enabled=false \
+        --set resources=null \
+        --wait
+fi
+
+if [ "$1" == "sda-svc" ]; then
+    helm install pipeline charts/sda-svc \
+        --set image.tag="PR$2" \
+        --set image.pullPolicy=IfNotPresent \
+        -f .github/integration/scripts/charts/values.yaml \
+        --wait
+fi

.github/integration/scripts/charts/k3d.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+set -ex
+
+k8s="$(curl -L -s https://dl.k8s.io/release/stable.txt)"
+
+curl -s -L https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | sudo bash
+
+if [ -n "$1" ]; then
+    k8s=$(k3d version list k3s | grep "$1" | head -n 1 | cut -d '-' -f 1)
+fi
+
+curl -sLO https://dl.k8s.io/release/"$k8s"/bin/linux/amd64/kubectl
+chmod +x ./kubectl
+sudo mv ./kubectl /usr/local/bin/kubectl
+
+k3d cluster create sda --image=rancher/k3s:"$k8s"-k3s1 --wait --timeout 10m
+k3d kubeconfig merge sda --kubeconfig-switch-context
+mkdir -p ~/.kube/ && cp ~/.config/kubeconfig-sda.yaml ~/.kube/config

.github/integration/scripts/charts/values.yaml

@@ -0,0 +1,92 @@
+global:
+  schemaType: "isolated"
+  ingress:
+    deploy: false
+    hostName:
+      auth: pipeline-sda-svc-auth
+      download: pipeline-sda-svc-download
+      s3Inbox: pipeline-sda-svc-inbox
+  log:
+    level: "debug"
+  tls:
+    enabled: false
+    issuer: "cert-issuer"
+    clusterIssuer: ""
+  archive:
+    s3Url: "http://minio.minio"
+    s3Bucket: "archive"
+    s3Port: 9000
+  backupArchive:
+    storageType: "s3"
+    s3Url: "http://minio.minio"
+    s3Bucket: "backup"
+    s3Port: 9000
+  auth:
+    jwtSecret: jwk
+    jwtAlg: ES256
+    jwtKey: jwt.key
+    jwtPub: jwt.pub
+    resignJwt: true
+  broker:
+    durable: true
+    host: "broker-sda-mq"
+    port: 5672
+    routingError: "error"
+    backupRoutingKey: "backup"
+    ssl: false
+    username: "admin"
+  c4gh:
+    secretName: c4gh
+    keyFile: c4gh.sec.pem
+    publicFile: c4gh.pub.pem
+  db:
+    host: "postgres-sda-db"
+    user: "postgres"
+  doa:
+    enabled: false
+  download:
+    enabled: true
+    trusted:
+      configPath: "iss"
+      configFile: "iss.json"
+      iss:
+        - iss: "http://oidc-server:8080"
+          jku: "http://oidc-server:8080/jwks"
+  oidc:
+    provider: "http://oidc-server:8080"
+    jwkPath: "/jwks"
+    id: DfCieZLuBU
+    secret: DfCieZLuBU
+  inbox:
+    storageType: s3
+    s3Url: http://minio.minio
+    s3Port: 9000
+    s3Bucket: "inbox"
+    s3ReadyPath: "/minio/health/ready"
+
+auth:
+  replicaCount: 1
+  resources: null
+backup:
+  deploy: true
+  resources: null
+doa:
+  deploy: false
+download:
+  resources: null
+  replicaCount: 1
+finalize:
+  resources: null
+ingest:
+  resources: null
+intercept:
+  deploy: false
+mapper:
+  resources: null
+releasetest:
+  run: false
+s3Inbox:
+  resources: null
+  replicaCount: 1
+verify:
+  resources: null