Add dataset event log

Since datasets now can have events similar to the files, registered, released and deprecated.

.github/integration/tests/postgres/60_mapper_queries.sh

@@ -6,14 +6,39 @@ export PGPASSWORD=mapper
 ## map file to dataset
 accession="urn:uuid:7964e232-8830-4351-8adb-e4ebb71fafed"
 dataset="urn:neic:ci-test-dataset"
-file_id=$(psql -U mapper -h postgres -d sda -At -c "SELECT file_id from local_ega.archive_files WHERE stable_id = '$accession';")
-if [ "$file_id" -ne 1 ]; then
+
+file_id=$(psql -U mapper -h postgres -d sda -At -c "SELECT id FROM sda.files WHERE stable_id = '$accession';")
+if [ "$file_id" = "" ]; then
     echo "get file_id failed"
     exit 1
 fi
 
-resp=$(psql -U mapper -h postgres -d sda -At -c "INSERT INTO local_ega_ebi.filedataset (file_id, dataset_stable_id) VALUES ('$file_id', '$dataset') ON CONFLICT DO NOTHING;")
+dataset_id=$(psql -U mapper -h postgres -d sda -At -c "INSERT INTO sda.datasets (stable_id) VALUES ('$dataset') ON CONFLICT DO NOTHING;")
+if [ "$dataset_id" != "INSERT 0 1" ]; then
+    echo "insert dataset failed"
+    exit 1
+fi
+
+resp=$(psql -U mapper -h postgres -d sda -At -c "INSERT INTO sda.file_dataset (file_id, dataset_id) SELECT '$file_id', id FROM sda.datasets WHERE stable_id = '$dataset' ON CONFLICT DO NOTHING;")
 if [ "$resp" != "INSERT 0 1" ]; then
-    echo "map to dataset failed"
+    echo "map file to dataset failed"
+    exit 1
+fi
+
+register=$(psql -U mapper -h postgres -d sda -At -c "INSERT INTO sda.dataset_event_log(dataset_id, event, message) VALUES('$dataset', 'registered', '{\"type\": \"mapping\"}');")
+if [ "$register" != "INSERT 0 1" ]; then
+    echo "update dataset event failed"
+    exit 1
+fi
+
+release=$(psql -U mapper -h postgres -d sda -At -c "INSERT INTO sda.dataset_event_log(dataset_id, event, message) VALUES('$dataset', 'released', '{\"type\": \"release\"}');")
+if [ "$release" != "INSERT 0 1" ]; then
+    echo "update dataset event failed"
+    exit 1
+fi
+
+deprecate=$(psql -U mapper -h postgres -d sda -At -c "INSERT INTO sda.dataset_event_log(dataset_id, event, message) VALUES('$dataset', 'deprecated', '{\"type\": \"deprecate\"}');")
+if [ "$deprecate" != "INSERT 0 1" ]; then
+    echo "update dataset event failed"
     exit 1
 fi

postgresql/initdb.d/01_main.sql

@@ -22,7 +22,8 @@ VALUES (0, now(), 'Created with version'),
        (5, now(), 'Add field for correlation ids'),
        (6, now(), 'Add created_at field to datasets'),
        (7, now(), 'Add permissions to mapper to files'),
-       (8, now(), 'Add ingestion functions');
+       (8, now(), 'Add ingestion functions'),
+       (9, now(), 'Add dataset event log');
 
 -- Datasets are used to group files, and permissions are set on the dataset
 -- level
@@ -140,3 +141,26 @@ CREATE TABLE file_event_log (
     success             BOOLEAN,
     error               TEXT
 );
+
+-- This table is used to define events for dataset event logging.
+CREATE TABLE dataset_events (
+    id          SERIAL PRIMARY KEY,
+    title       VARCHAR(64) UNIQUE, -- short name of the action
+    description TEXT
+);
+
+-- These are the default dataset events to log.
+INSERT INTO dataset_events(id,title,description)
+VALUES (10, 'registered', 'A dataset had been registered'),
+       (20, 'released'  , 'The dataset is released on this date'),
+       (30, 'deprecated', 'The dataset is deprecated on this date');
+
+
+-- Keeps track of all events for the datasets, with timestamps.
+CREATE TABLE dataset_event_log (
+    id         SERIAL PRIMARY KEY,
+    dataset_id TEXT REFERENCES datasets(stable_id),
+    event      TEXT REFERENCES dataset_events(title),
+    message    JSONB, -- The rabbitMQ message that initiated the dataset event
+    event_date TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT clock_timestamp()
+);

postgresql/initdb.d/04_grants.sql

@@ -101,14 +101,15 @@ GRANT USAGE, SELECT ON SEQUENCE sda.datasets_id_seq TO mapper;
 GRANT SELECT ON sda.files TO mapper;
 GRANT INSERT ON sda.file_event_log TO mapper;
 GRANT INSERT ON sda.file_dataset TO mapper;
-GRANT SELECT ON local_ega.main_to_files TO mapper;
+GRANT INSERT ON sda.dataset_event_log TO mapper;
 GRANT USAGE, SELECT ON SEQUENCE sda.file_dataset_id_seq TO mapper;
 GRANT USAGE, SELECT ON SEQUENCE sda.file_event_log_id_seq TO mapper;
+GRANT USAGE, SELECT ON SEQUENCE sda.dataset_event_log_id_seq TO mapper;
 
 -- legacy schema
 GRANT USAGE ON SCHEMA local_ega TO mapper;
 GRANT USAGE ON SCHEMA local_ega_ebi TO mapper;
-
+GRANT SELECT ON local_ega.main_to_files TO mapper;
 GRANT SELECT ON local_ega.archive_files TO mapper;
 GRANT INSERT ON local_ega_ebi.filedataset TO mapper;
 GRANT UPDATE ON local_ega.files TO mapper;

postgresql/migratedb.d/09.sql

@@ -0,0 +1,36 @@
+DO
+$$
+DECLARE
+-- The version we know how to do migration from, at the end of a successful migration
+-- we will no longer be at this version.
+  sourcever INTEGER := 8;
+  changes VARCHAR := 'Add ingestion functions ';
+BEGIN
+  IF (select max(version) from sda.dbschema_version) = sourcever then
+    RAISE NOTICE 'Doing migration from schema version % to %', sourcever, sourcever+1;
+    RAISE NOTICE 'Changes: %', changes;
+    INSERT INTO sda.dbschema_version VALUES(sourcever+1, now(), changes);
+
+    CREATE TABLE dataset_events (
+        id          SERIAL PRIMARY KEY,
+        title       VARCHAR(64) UNIQUE, -- short name of the action
+        description TEXT
+    );
+
+    INSERT INTO dataset_events(id,title,description)
+    VALUES (10, 'registered', 'A dataset had been registered'),
+           (20, 'released'  , 'The dataset is released on this date'),
+           (30, 'deprecated', 'The dataset is deprecated on this date');
+
+    CREATE TABLE dataset_event_log (
+        id         SERIAL PRIMARY KEY,
+        dataset_id TEXT REFERENCES datasets(stable_id),
+        event      TEXT REFERENCES dataset_events(title),
+        message    JSONB, -- The rabbitMQ message that initiated the dataset event
+        event_date TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT clock_timestamp()
+    );
+  ELSE
+    RAISE NOTICE 'Schema migration from % to % does not apply now, skipping', sourcever, sourcever+1;
+  END IF;
+END
+$$