add test file

.github/integration/setup/setup.sh

@@ -86,7 +86,6 @@ do echo "waiting for buckets to be created"
     sleep 10
 done
 
-
 # Populate database with for testing the download service
 # Insert entry in sda.files
 file_ids=$(docker run --rm --name client --network testing_default \
@@ -118,17 +117,17 @@ hts_file_ids=$(docker run --rm --name client --network testing_default \
         submission_file_path, submission_file_size, archive_file_path, \
         archive_file_size, decrypted_file_size, backup_path, header, \
         encryption_method) VALUES ('FILE0000001', 'integration-test', 'dummy_gdi.eu/htsnexus_test_NA12878.bam.c4gh', \
-        NULL, 'abb0305b-861b-4983-84c2-c4ad9ab5ebe8', 2597919, 2596799, \
-        'NULL', '637279707434676801000000010000006c00000000000000ea3cb13f1decddb198de260f74e59d03a372fd2ae65509dea4c63f348a21a80ef1b9be145cd3943522a62bbc1992090f0c3a5ef6b4cd05c705333ca16f86818ab99e77e4695d0f45982c70f208cce240fb5e8928713d3aae5bc88206dcf795ce4fc2259a', 'CRYPT4GH'),
+        2597919, '1068a0b9-51c9-4bda-8fcf-14b3b884f7a4', 2597919, 2596799, \
+        '', '637279707434676801000000010000006c000000000000001b8aba36a673d374614406259c60b5c658492e05a771240f0ca5a407836bfc630ff275d8ae5e213a5853127611126e2fc79b4a16e8bbb95f3d9503902ac9ce59d0d58a15e9b8b364cb7d9c17ad1d863c89cd097667a1cc17b215095779886635bdd0ea70', 'CRYPT4GH'),
         ('FILE0000002', 'integration-test', 'dummy_gdi.eu/htsnexus_test_NA12878.bam.bai.c4gh', \
-        NULL, '4d6c1787-3641-4aba-8e3f-11f35b514418', 6756, 6728, \
-        'NULL', '637279707434676801000000010000006c000000000000002807585fbbafa584ef89bbd4140e41dc7afe35c6491787f9ea599704770d251ba8947577ade743f1f58539a6afb41022f2c38822befd16d20bc4e5b10fa582a8c676e6f27b9c804d4db0d225b64198bd5c69a2e2c87a79c3ef22d03b7a6a10771219ce85', 'CRYPT4GH'),
+        6756, '668330fb-bdd8-40b2-a88f-29e1e610fa4b', 6756, 6728, \
+        '', '637279707434676801000000010000006c00000000000000856951a851a8dd081cd3f17e8212c40caea530327704aaccfb852aaabf7c947ea9b541c32358151979451259b78bbaf5e824cf76d1758ec3a34bd845a2af389639be6adf46c77910861ca1236ed1f0d46a0e91dc31ebae28f5f75776d284d75ce2c0d444', 'CRYPT4GH'),
         ('FILE0000003', 'integration-test', 'dummy_gdi.eu/htsnexus_test_NA12878.bam.blocks.yaml.c4gh', \
-        NULL, 'bd0cc83e-86c9-41b4-b110-0089018895f4', 2061, 2033, \
-        'NULL', '637279707434676801000000010000006c00000000000000fe396aefdaeec924ec33ebb1a2de25613212cb8fe902939b28e5e11199885a11c1ba17cdcdbe13fc22fcc451a3010a6008e8f494d53fb8769d23c5ebb06f7ab75821121c634541d3366ae3325bf931e1206e66ce11091b279dd18cf5da5a50f8cf156896', 'CRYPT4GH'),
+        2061, 'ce796074-21d8-43ef-9b5e-848ce8b6e47b', 2061, 2033, \
+        '', '637279707434676801000000010000006c00000000000000d0a084f92c6ddd1193320d00e94be34d1e31d10654eae55d67fe53b165812c194a1eb18c09881adaf49a567281fceab23495ae925de7ea2027288dd1faa601e1303f87cb31968d12211825f0fa1fd6628ac33e3b629567e38ed9fe27cfefaa344b5bb523', 'CRYPT4GH'),
         ('FILE0000004', 'integration-test', 'dummy_gdi.eu/htsnexus_test_NA12878.bam.gzi.c4gh', \
-        NULL, '45a73706-7427-4839-9a89-ddb2998b091f', 2644, 2616, \
-        'NULL', '637279707434676801000000010000006c00000000000000c785c033c0ffb12575603cd3ff2e745e3f6552aed9959b942cdd572623dc8007c8437921c473ce5ebf50b050b714aa39b10e784d872adfd2fb27dc96744761feb14a212348664d7d14d459a8cab8ad8380830984a611393c506778656a4fc3e8d0daa0c7', 'CRYPT4GH') RETURNING id;" | xargs)
+        2644, '2c063d42-f57f-4e74-bb1c-f56af00825b7', 2644, 2616, \
+        '', '637279707434676801000000010000006c000000000000000818521c25f759f337e0560977cd36f29edd073ac05bc6efc0def114aa81c625c485461fae5c12c8967107b0edf8fd697bb00fa103592b0f4e161541fcbabb39e1c7cfc3ee84ebaa4085ecf802ae5e010ffafcc139a3dd5ed818c13192d44548718e9437', 'CRYPT4GH') RETURNING id;" | xargs)
 
 if [ -z "$hts_file_ids" ]; then
     echo "Failed to insert file entry into database"
@@ -194,25 +193,65 @@ docker run --rm --name client --network testing_default \
     neicnordic/pg-client:latest \
     postgresql://postgres:rootpasswd@postgres:5432/sda \
     -t -q -c "INSERT INTO sda.checksums (file_id, checksum, type, source) \
-        VALUES ('$hts_file_id1', '15d28f1f14764511402e8c0eaf68316a79d8d9e7a787594307ed339a2a411b98', 'SHA256', 'ARCHIVED');"
+        VALUES ('$hts_file_id1', 'e0ab4f1c82bcac6fa13040f4778a6642ba21dcf7c321bdacbe02c78dbdf8e2fb', 'SHA256', 'UPLOADED');"
+docker run --rm --name client --network testing_default \
+    neicnordic/pg-client:latest \
+    postgresql://postgres:rootpasswd@postgres:5432/sda \
+    -t -q -c "INSERT INTO sda.checksums (file_id, checksum, type, source) \
+        VALUES ('$hts_file_id1', 'a615ce3e5c8a59c646deb5b01cf088182a5102a02c1ce88b71d2461cfb621360', 'SHA256', 'ARCHIVED');"
+docker run --rm --name client --network testing_default \
+    neicnordic/pg-client:latest \
+    postgresql://postgres:rootpasswd@postgres:5432/sda \
+    -t -q -c "INSERT INTO sda.checksums (file_id, checksum, type, source) \
+        VALUES ('$hts_file_id1', '0424dfcc1255ff5f48ae39f9bf0129d1d01cb75fa6cf8d141670101860182060', 'SHA256', 'UNENCRYPTED');"
 
 docker run --rm --name client --network testing_default \
     neicnordic/pg-client:latest \
     postgresql://postgres:rootpasswd@postgres:5432/sda \
     -t -q -c "INSERT INTO sda.checksums (file_id, checksum, type, source) \
-        VALUES ('$hts_file_id2', 'bd0dc0bd81d7ef40ba57f8dcda80264b49e227da795dcde6588e6239c93f3af2', 'SHA256', 'ARCHIVED');"
+        VALUES ('$hts_file_id2', '0ea62b6fce887a61ecb03c2ebdadd96d398b3fd76e9e639064c19561ceb3c155', 'SHA256', 'UPLOADED');"
+docker run --rm --name client --network testing_default \
+    neicnordic/pg-client:latest \
+    postgresql://postgres:rootpasswd@postgres:5432/sda \
+    -t -q -c "INSERT INTO sda.checksums (file_id, checksum, type, source) \
+        VALUES ('$hts_file_id2', '136f509276413663780928f97f2c6dbf6fff0c290420a3144fb5af36b19d8d58', 'SHA256', 'ARCHIVED');"
+docker run --rm --name client --network testing_default \
+    neicnordic/pg-client:latest \
+    postgresql://postgres:rootpasswd@postgres:5432/sda \
+    -t -q -c "INSERT INTO sda.checksums (file_id, checksum, type, source) \
+        VALUES ('$hts_file_id2', '1da16bd728559f3089b87e563d3ca87e673b878cbbededbb9a33af81f386d530', 'SHA256', 'UNENCRYPTED');"
 
 docker run --rm --name client --network testing_default \
     neicnordic/pg-client:latest \
     postgresql://postgres:rootpasswd@postgres:5432/sda \
     -t -q -c "INSERT INTO sda.checksums (file_id, checksum, type, source) \
-        VALUES ('$hts_file_id3', 'b6c85131c91dc96a726e4a3dc9608c5640799b316654dadd14f7e1fd78bfd220', 'SHA256', 'ARCHIVED');"
+        VALUES ('$hts_file_id3', '77bfc5dfcbd3bad740a6337df39537a9a9687cf087aa409a059c380a7476380d', 'SHA256', 'UPLOADED');"
+docker run --rm --name client --network testing_default \
+    neicnordic/pg-client:latest \
+    postgresql://postgres:rootpasswd@postgres:5432/sda \
+    -t -q -c "INSERT INTO sda.checksums (file_id, checksum, type, source) \
+        VALUES ('$hts_file_id3', '5fcf7fbf9c9640e96e35fcc9feeb8d74399df02861e9368dce2d8229435df37e', 'SHA256', 'ARCHIVED');"
+docker run --rm --name client --network testing_default \
+    neicnordic/pg-client:latest \
+    postgresql://postgres:rootpasswd@postgres:5432/sda \
+    -t -q -c "INSERT INTO sda.checksums (file_id, checksum, type, source) \
+        VALUES ('$hts_file_id3', 'a4f22564184871ad9a75a5dd5e3c2ffbe7c9b48f0392bbd0c151aca95964b6cf', 'SHA256', 'UNENCRYPTED');"
 
 docker run --rm --name client --network testing_default \
     neicnordic/pg-client:latest \
     postgresql://postgres:rootpasswd@postgres:5432/sda \
     -t -q -c "INSERT INTO sda.checksums (file_id, checksum, type, source) \
-        VALUES ('$hts_file_id4', 'cdedc99765fd3d77bf6f12a89d4afc276d16155a08bab4ea251acaaca4cad344', 'SHA256', 'ARCHIVED');"
+        VALUES ('$hts_file_id4', 'f11a60bc3945676727af36e8f05ebede895db777addf69c8c7cabad7c5a681c7', 'SHA256', 'UPLOADED');"
+docker run --rm --name client --network testing_default \
+    neicnordic/pg-client:latest \
+    postgresql://postgres:rootpasswd@postgres:5432/sda \
+    -t -q -c "INSERT INTO sda.checksums (file_id, checksum, type, source) \
+        VALUES ('$hts_file_id4', '399602f732d35374033acc5dd51b5c2b658c491eabafa826f3f4b601fbf71067', 'SHA256', 'ARCHIVED');"
+docker run --rm --name client --network testing_default \
+    neicnordic/pg-client:latest \
+    postgresql://postgres:rootpasswd@postgres:5432/sda \
+    -t -q -c "INSERT INTO sda.checksums (file_id, checksum, type, source) \
+        VALUES ('$hts_file_id4', '55cc7a5adbc85096abefb83d99f784fd68adb8e2c83aa1a71c6b705bb95bf48c', 'SHA256', 'UNENCRYPTED');"
 
 for id in $hts_file_ids; do
     # Insert entry in sda.file_event_log

.github/integration/tests/tests.sh

@@ -409,6 +409,30 @@ fi
 
 rm -r test-download
 
+# check htsget endpoint works
+curl 'http://localhost:8089/s3-encrypted/https://doi.example/ty009.sfrrss/600.45asasga/htsnexus_test_NA12878.bam' -H "Authorization: Bearer $token"  -H "Client-Public-Key: $pubkey" -H "Range: bytes=16-123" -o p11-00.bam.c4gh
+
+# Download partial file by using htsget endpoint
+./sda-cli htsget -config testing/s3cmd-download.conf -dataset https://doi.example/ty009.sfrrss/600.45asasga -filename htsnexus_test_NA12878 -reference 11 -host http://localhost:8088 -pubkey user_key.pub.pem -output test-htsget/result.c4gh --force-overwrite
+
+# check if file exists in the path
+if [ ! -f "test-htsget/result.c4gh" ]; then
+    echo "Downloaded file not found"
+    exit 1
+fi
+
+# decrypt the downloaded file
+C4GH_PASSWORD="" ./sda-cli decrypt -key user_key.sec.pem test-htsget/result.c4gh
+
+if [ -f test-htsget/result  ]; then
+    echo "Decrypting downloaded file succeeded"
+else
+    echo "Failed to decrypt downloaded file"
+    exit 1
+fi
+
+rm -r test-htsget
+
 # Download recursively a folder
 echo "Downloading content of folder"
 ./sda-cli sda-download -config testing/s3cmd-download.conf -dataset-id https://doi.example/ty009.sfrrss/600.45asasga -url http://localhost:8080 -outdir download-folder --recursive main/subfolder2

testing/config-htsget-rs/download-config.toml

@@ -14,12 +14,12 @@ send_encrypted_to_client = true  # if true, htsget will calculate coordinates fo
 
 [resolvers.storage]
 # The url that will be returned in the response to the client
-response_url = "http://localhost:8443/s3-encrypted/"
+response_url = "http://localhost:8089/s3-encrypted/"
 # forward_headers must be true in order to pass the clients token and public key to the storage endpoint
 forward_headers = true
 
 [resolvers.storage.endpoints]
 # URL to retrieve an index file from the storage
-index = "http://dockerhost:8443/s3/"
+index = "http://dockerhost:8089/s3/"
 # URL to retrieve a data file from the storage
-file = "http://dockerhost:8443/s3-encrypted/"
+file = "http://dockerhost:8089/s3-encrypted/"

testing/docker-compose.yml

@@ -200,14 +200,19 @@ services:
       - GRPC_PORT=50051
       - GRPC_HOST=reencrypt
       - APP_SERVEUNENCRYPTEDDATA=true
+      - C4GH_PASSPHRASE=oaagCP1YgAZeEyl2eJAkHv9lkcWXWFgm
+      - C4GH_FILEPATH=/dev_utils/c4gh.sec.pem
     image: "ghcr.io/neicnordic/sensitive-data-archive:${TAG}-download"
     volumes:
       - ./archive_data/4293c9a7-dc50-46db-b79a-27ddc0dad1c6:/tmp/4293c9a7-dc50-46db-b79a-27ddc0dad1c6
-      - ./archive_data/abb0305b-861b-4983-84c2-c4ad9ab5ebe8:/tmp/abb0305b-861b-4983-84c2-c4ad9ab5ebe8
-      - ./archive_data/4d6c1787-3641-4aba-8e3f-11f35b514418:/tmp/4d6c1787-3641-4aba-8e3f-11f35b514418
-      - ./archive_data/bd0cc83e-86c9-41b4-b110-0089018895f4:/tmp/bd0cc83e-86c9-41b4-b110-0089018895f4
-      - ./archive_data/45a73706-7427-4839-9a89-ddb2998b091f:/tmp/45a73706-7427-4839-9a89-ddb2998b091f
+      - ./archive_data/1068a0b9-51c9-4bda-8fcf-14b3b884f7a4:/tmp/1068a0b9-51c9-4bda-8fcf-14b3b884f7a4
+      - ./archive_data/668330fb-bdd8-40b2-a88f-29e1e610fa4b:/tmp/668330fb-bdd8-40b2-a88f-29e1e610fa4b
+      - ./archive_data/ce796074-21d8-43ef-9b5e-848ce8b6e47b:/tmp/ce796074-21d8-43ef-9b5e-848ce8b6e47b
+      - ./archive_data/2c063d42-f57f-4e74-bb1c-f56af00825b7:/tmp/2c063d42-f57f-4e74-bb1c-f56af00825b7
+      - ./:/dev_utils/
     mem_limit: 256m
+    extra_hosts:
+      - ${DOCKERHOST:-dockerhost}:host-gateway
     ports:
       - "8089:8080"
     restart: always
@@ -228,6 +233,7 @@ services:
   htsget:
     image: harbor.nbis.se/gdi/htsget-rs:20240415
     command: ["htsget-actix", "--config", "/config/download-config.toml"]
+    container_name: htsget
     depends_on:
       download:
         condition: service_started