Merge pull request #1126 from neicnordic/feature/ingest-store-keyhash

[ingest] store keyhash for ingested files

.github/integration/tests/sda/20_ingest-verify_test.sh

@@ -69,4 +69,11 @@ until [ "$(curl -su guest:guest http://rabbitmq:15672/api/queues/sda/verified/ |
     sleep 2
 done
 
-echo "ingestion and verification test completed successfully"
+# check that the files have key hashes assigned
+key_hashes="$(psql -U postgres -h postgres -d sda -At -c "select count(distinct key_hash) from sda.files")"
+if [ "$key_hashes" -eq 0 ]; then
+	echo "::error::Ingested files did not have any key hashes."
+	exit 1
+fi
+
+echo "ingestion and verification test completed successfully"

sda/cmd/ingest/ingest.go

@@ -6,6 +6,7 @@ package main
 import (
 	"bytes"
 	"crypto/sha256"
+	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -14,6 +15,7 @@ import (
 	"strings"
 	"syscall"
 
+	"github.com/neicnordic/crypt4gh/keys"
 	"github.com/neicnordic/crypt4gh/model/headers"
 	"github.com/neicnordic/crypt4gh/streaming"
 	"github.com/neicnordic/sensitive-data-archive/internal/broker"
@@ -388,6 +390,19 @@ func main() {
 							continue mainWorkLoop
 						}
 
+						// Set the file's hex encoded public key
+						publicKey := keys.DerivePublicKey(*key)
+						keyhash := hex.EncodeToString(publicKey[:])
+						err = db.SetKeyHash(keyhash, fileID)
+						if err != nil {
+							log.Errorf("Key hash %s could not be set for fileID %s: (%s)", keyhash, fileID, err.Error())
+							if err = delivered.Nack(false, true); err != nil {
+								log.Errorf("Failed to Nack message, reason: (%s)", err.Error())
+							}
+
+							continue mainWorkLoop
+						}
+
 						log.Debugln("store header")
 						if err := db.StoreHeader(header, fileID); err != nil {
 							log.Errorf("StoreHeader failed, reason: (%s)", err.Error())

sda/internal/database/db_functions.go

@@ -784,6 +784,24 @@ func (dbs *SDAdb) addKeyHash(keyHash, keyDescription string) error {
 	return nil
 }
 
+func (dbs *SDAdb) SetKeyHash(keyHash, fileID string) error {
+	dbs.checkAndReconnectIfNeeded()
+	db := dbs.DB
+
+	query := "UPDATE sda.files SET key_hash = $1 WHERE id = $2;"
+	result, err := db.Exec(query, keyHash, fileID)
+	if err != nil {
+
+		return err
+	}
+	if rowsAffected, _ := result.RowsAffected(); rowsAffected == 0 {
+		return errors.New("something went wrong with the query, zero rows were changed")
+	}
+	log.Debugf("Successfully set key hash for file %v", fileID)
+
+	return nil
+}
+
 type C4ghKeyHash struct {
 	Hash         string `json:"hash"`
 	Description  string `json:"description"`

sda/internal/database/db_functions_test.go

@@ -714,3 +714,43 @@ func (suite *DatabaseTests) TestDeprecateKeyHashes_alreadyDeprecated() {
 	// we should not be able to change the deprecation date
 	assert.EqualError(suite.T(), db.DeprecateKeyHash("cbd8f5cc8d936ce437a52cd7991453839581fc69ee26e0daefde6a5d2660fc54"), "key hash not found or already deprecated", "failure when deprecating keyhash")
 }
+
+func (suite *DatabaseTests) TestSetKeyHash() {
+	// Test that using an unknown key hash produces an error
+	db, err := NewSDAdb(suite.dbConf)
+	assert.NoError(suite.T(), err, "got (%v) when creating new connection", err)
+	// Register a new key and a new file
+	keyHex := `6af1407abc74656b8913a7d323c4bfd30bf7c8ca359f74ae35357acef29dc507`
+	keyDescription := "this is a test key"
+	err = db.addKeyHash(keyHex, keyDescription)
+	assert.NoError(suite.T(), err, "failed to register key in database")
+	fileID, err := db.RegisterFile("/testuser/file1.c4gh", "testuser")
+	assert.NoError(suite.T(), err, "failed to register file in database")
+
+	// Test that the key hash can be set in the files table
+	err = db.SetKeyHash(keyHex, fileID)
+	assert.NoError(suite.T(), err, "Could not set key hash")
+
+	// Verify that the key+file was added
+	var exists bool
+	err = db.DB.QueryRow("SELECT EXISTS(SELECT 1 FROM sda.files WHERE key_hash=$1 AND id=$2)", keyHex, fileID).Scan(&exists)
+	assert.NoError(suite.T(), err, "failed to verify key hash set for file")
+	assert.True(suite.T(), exists, "key hash was not set for file in the database")
+}
+
+func (suite *DatabaseTests) TestSetKeyHash_wrongHash() {
+	// Add key hash and file
+	db, err := NewSDAdb(suite.dbConf)
+	assert.NoError(suite.T(), err, "got (%v) when creating new connection", err)
+	keyHex := "6af1407abc74656b8913a7d323c4bfd30bf7c8ca359f74ae35357acef29dc501"
+	keyDescription := "this is a test hash"
+	err = db.addKeyHash(keyHex, keyDescription)
+	assert.NoError(suite.T(), err, "failed to register key in database")
+	fileID, err := db.RegisterFile("/testuser/file2.c4gh", "testuser")
+	assert.NoError(suite.T(), err, "failed to register file in database")
+
+	// Ensure failure if a non existing hash is used
+	newKeyHex := "6af1407abc74656b8913a7d323c4bfd30bf7c8ca359f74ae35357acef29dc502"
+	err = db.SetKeyHash(newKeyHex, fileID)
+	assert.ErrorContains(suite.T(), err, "violates foreign key constraint")
+}