dump

.github/integration/scripts/make_db_credentials.sh

@@ -7,14 +7,15 @@ apt-get -o DPkg::Lock::Timeout=60 install -y postgresql-client >/dev/null
 for n in download finalize inbox ingest mapper sync verify; do
     echo "creating credentials for: $n"
 
-    if [ "$n" = inbox ]; then
+    if [ "$n" = "inbox" ]; then
         psql -U postgres -h postgres -d sda -c "DROP ROLE IF EXISTS inbox;"
         psql -U postgres -h postgres -d sda -c "CREATE ROLE inbox;"
         psql -U postgres -h postgres -d sda -c "GRANT base, ingest TO inbox;"
     fi
 
-    if [ "$n" = ingest ]; then
+    if [ "$n" = "ingest" ]; then
         psql -U postgres -h postgres -d sda -c "GRANT UPDATE ON local_ega.main TO ingest;"
+        psql -U postgres -h postgres -d sda -c "GRANT base TO ingest;"
     fi
 
     psql -U postgres -h postgres -d sda -c "ALTER ROLE $n LOGIN PASSWORD '$n';"

.github/integration/scripts/make_sda_credentials.sh

@@ -15,6 +15,7 @@ for n in download finalize inbox ingest mapper sync verify; do
 
     if [ "$n" = ingest ]; then
         psql -U postgres -h postgres -d sda -c "GRANT UPDATE ON local_ega.main TO ingest;"
+        psql -U postgres -h postgres -d sda -c "GRANT base TO ingest;"
     fi
 
     psql -U postgres -h postgres -d sda -c "ALTER ROLE $n LOGIN PASSWORD '$n';"

.github/integration/sda-integration.yml

@@ -47,7 +47,7 @@ services:
           "rabbitmq-diagnostics -q check_running && rabbitmq-diagnostics -q check_local_alarms",
         ]
       interval: 10s
-      timeout: 2s
+      timeout: 5s
       retries: 6
     image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-rabbitmq
     ports:
@@ -98,6 +98,7 @@ services:
     environment:
       - BROKER_PASSWORD=inbox
       - BROKER_USER=inbox
+      - BROKER_ROUTINGKEY=inbox
       - DB_PASSWORD=inbox
       - DB_USER=inbox
     restart: always
@@ -108,6 +109,31 @@ services:
       - "18000:8000"
       - "18001:8001"
 
+  ingest:
+    image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}
+    command: [ sda-ingest ]
+    container_name: ingest
+    depends_on:
+      credentials:
+        condition: service_completed_successfully
+      minio:
+        condition: service_healthy
+      postgres:
+        condition: service_healthy
+      rabbitmq:
+        condition: service_healthy
+    environment:
+      - BROKER_PASSWORD=ingest
+      - BROKER_USER=ingest
+      - BROKER_QUEUE=ingest
+      - BROKER_ROUTINGKEY=archived
+      - DB_PASSWORD=ingest
+      - DB_USER=ingest
+    restart: always
+    volumes:
+      - ./sda/config.yaml:/config.yaml
+      - shared:/shared
+
   oidc:
     container_name: oidc
     command:
@@ -142,6 +168,8 @@ services:
     depends_on:
       credentials:
         condition: service_completed_successfully
+      ingest:
+        condition: service_started
       s3inbox:
         condition: service_started
     environment:

.github/integration/sda/config.yaml

@@ -1,6 +1,17 @@
 log:
   format: "json"
+  level: "debug"
+archive:
+  type: s3
+  url: "http://s3"
+  port: 9000
+  readypath: "/minio/health/ready"
+  accessKey: "access"
+  secretKey: "secretKey"
+  bucket: "archive"
+  region: "us-east-1"
 inbox:
+  type: s3
   url: "http://s3"
   port: 9000
   readypath: "/minio/health/ready"
@@ -16,7 +27,7 @@ broker:
   password: ""
   vhost: "/sda"
   exchange: "sda"
-  routingKey: "inbox"
+  routingKey: ""
   ssl: "false"
 
 db:
@@ -27,6 +38,9 @@ db:
   database: "sda"
   sslmode: "disable"
 
+c4gh:
+  filePath: /shared/c4gh.sec.pem
+  passphrase: "c4ghpass"
 
 server:
   cert: ""

.github/integration/tests/run_scripts.sh

@@ -4,7 +4,7 @@ set -e
 apt-get -o DPkg::Lock::Timeout=60 update > /dev/null
 apt-get -o DPkg::Lock::Timeout=60 install -y postgresql-client > /dev/null
 
-find "$1"/*.sh 2>/dev/null |  sort -t/ -k3 -n | while read -r runscript; do
+for runscript in "$1"/*.sh; do
     echo "Executing test script $runscript"
     bash -x "$runscript"
 done

.github/integration/tests/sda/10_upload_test.sh

@@ -1,5 +1,5 @@
 #!/bin/sh
-set -e
+set -eou pipefail
 
 # install tools if missing
 for t in curl jq postgresql-client; do
@@ -21,7 +21,7 @@ cd shared || true
 for file in NA12878.bam NA12878_20k_b37.bam; do
     curl -s -L -o /shared/$file "https://github.com/ga4gh/htsget-refserver/raw/main/data/gcp/gatk-test-data/wgs_bam/$file"
     if [ ! -f "$file.c4gh" ]; then
-        /shared/crypt4gh encrypt -p c4gh.pub.pem -f "$file"
+        yes| /shared/crypt4gh encrypt -p c4gh.pub.pem -f "$file"
     fi
     s3cmd -c s3cfg put "$file.c4gh" s3://test_dummy.org/
 done

.github/integration/tests/sda/20_ingest_test.sh

@@ -0,0 +1,58 @@
+#!/bin/sh
+set -eou pipefail
+
+cd shared || true
+
+for file in NA12878.bam NA12878_20k_b37.bam; do
+    ENC_SHA=$(sha256sum "$file.c4gh" | cut -d' ' -f 1)
+    ENC_MD5=$(md5sum "$file.c4gh" | cut -d' ' -f 1)
+
+    ## get correlation id from upload message
+    CORRID=$(
+        curl -s -X POST \
+            -H "content-type:application/json" \
+            -u guest:guest http://localhost:15672/api/queues/sda/inbox/get \
+            -d '{"count":1,"encoding":"auto","ackmode":"ack_requeue_false"}' | jq -r .[0].properties.correlation_id
+    )
+
+    ## publish message to trigger ingestion
+    properties=$(
+        jq -c -n \
+            --argjson delivery_mode 2 \
+            --arg correlation_id "$CORRID" \
+            --arg content_encoding UTF-8 \
+            --arg content_type application/json \
+            '$ARGS.named'
+    )
+
+    encrypted_checksums=$(
+        jq -c -n \
+            --arg sha256 "$ENC_SHA" \
+            --arg md5 "$ENC_MD5" \
+            '$ARGS.named|to_entries|map(with_entries(select(.key=="key").key="type"))'
+    )
+
+    ingest_payload=$(
+        jq -r -c -n \
+            --arg type ingest \
+            --arg user [email protected] \
+            --arg filepath test_dummy.org/"$file.c4gh" \
+            --argjson encrypted_checksums "$encrypted_checksums" \
+            '$ARGS.named|@base64'
+    )
+
+    ingest_body=$(
+        jq -c -n \
+            --arg vhost sda \
+            --arg name sda \
+            --argjson properties "$properties" \
+            --arg routing_key "ingest" \
+            --arg payload_encoding base64 \
+            --arg payload "$ingest_payload" \
+            '$ARGS.named'
+    )
+
+    curl -s -u guest:guest "http://localhost:15672/api/exchanges/sda/sda/publish" \
+        -H 'Content-Type: application/json;charset=UTF-8' \
+        -d "$ingest_body"
+done

.github/workflows/test.yml

@@ -136,7 +136,7 @@ jobs:
 
       - name: Get dependencies
         run: |
-          cd sda-download
+          cd sda
           go get -v -t -d ./...
           if [ -f Gopkg.toml ]; then
               curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh

go.work

@@ -0,0 +1,6 @@
+go 1.20
+
+use (
+    ./cmd/s3inbox
+    ./internal/database
+)