#110 | Modify deleteDuplicateMedia sql to make use of PARTITION by en…

…tity_id and Rank by desc last_modified_date_time

src/main/resources/sql/etl/deleteDuplicateMedia.sql.st

@@ -2,7 +2,7 @@ DELETE
 FROM "<schemaName>"."media" t2
 WHERE t2.id IN (
     SELECT duplicates.dup_id
-    FROM (SELECT t1.id AS dup_id, ROW_NUMBER() OVER (PARTITION BY uuid, entity_id) AS occurence
+    FROM (SELECT t1.id AS dup_id, RANK() OVER (PARTITION BY entity_id ORDER BY last_modified_date_time DESC) AS rank
           FROM "<schemaName>"."media" t1) AS duplicates
-    WHERE duplicates.occurence > 1
+    WHERE duplicates.rank > 1
 );

src/test/java/org/avniproject/etl/DataSyncIntegrationTest.java

@@ -459,13 +459,22 @@ public void shouldPopulateMediaTableCorrectlyWhenTransactionalDataUpdates() thro
     @Sql(scripts = "/test-data-teardown.sql", executionPhase = Sql.ExecutionPhase.AFTER_TEST_METHOD)
     public void arrayStyleMediaObservationsCreateMultipleRowsInMediaTable() throws InterruptedException {
         String encounterImages = "[\"https://s3.amazon.com/image1.jpg\", \"https://s3.amazon.com/image2.jpg\"]";
-        String sql = "update encounter set observations = observations || '{\"44163589-f76d-447d-9b6e-f5c32aa859eb\": " + encounterImages + "}'::jsonb where id = 1900;";
+        String sql = "update encounter set last_modified_date_time = now(), observations = observations || '{\"44163589-f76d-447d-9b6e-f5c32aa859eb\": " + encounterImages + "}'::jsonb where id = 1900;";
         jdbcTemplate.execute(sql);
 
         runDataSync();
 
         List<Map<String, Object>> media = jdbcTemplate.queryForList("select * from orgc.media;");
         assertThat("Media table number of rows has not changed since last run", media.size(), is(6));
+
+        String encounterImagesSecondSet = "[\"https://s3.amazon.com/image3.jpg\", \"https://s3.amazon.com/image4.jpg\", \"https://s3.amazon.com/image5.jpg\"]";
+        String sqlSecondItr = "update encounter set last_modified_date_time = now(), observations = observations || '{\"44163589-f76d-447d-9b6e-f5c32aa859eb\": " + encounterImagesSecondSet + "}'::jsonb where id = 1900;";
+        jdbcTemplate.execute(sqlSecondItr);
+
+        runDataSync();
+
+        List<Map<String, Object>> mediaSecondItr = jdbcTemplate.queryForList("select * from orgc.media;");
+        assertThat("Media table number of rows was supposed to increase since last run", mediaSecondItr.size(), is(7));
     }
 
     @Test

src/test/resources/media-form-element.sql

@@ -3,14 +3,14 @@ insert into concept (id, data_type, high_absolute, high_normal, low_absolute, lo
 insert into concept (id, data_type, high_absolute, high_normal, low_absolute, low_normal, name, uuid, version, unit, organisation_id, is_voided, audit_id, key_values, active, created_by_id, last_modified_by_id, created_date_time, last_modified_date_time) values (108002, 'Image', null, null, null, null, 'Person''s Image','b1815016-40d6-4045-9ae0-2f659fc5db6d', 0, null, 12, false, create_audit(), null, true, 1, 1, '2022-04-13 15:49:55.019 +00:00', '2022-04-13 15:49:55.019 +00:00');
 insert into form_element (id, name, display_order, is_mandatory, key_values, concept_id, form_element_group_id, uuid, version, organisation_id, type, valid_format_regex, valid_format_description_key, audit_id, is_voided, rule, created_by_id, last_modified_by_id, created_date_time, last_modified_date_time, declarative_rule) values (50116, 'Media form element', 8, false, '[]', 108001, 5569, 'fd5ad16f-c18e-45db-841c-a2f994862957', 0, 12, 'SingleSelect', null, null, create_audit(), false, null, 1, 1, '2022-04-13 10:51:44.705 +00:00', '2022-04-13 10:51:44.705 +00:00', null);
 insert into form_element (id, name, display_order, is_mandatory, key_values, concept_id, form_element_group_id, uuid, version, organisation_id, type, valid_format_regex, valid_format_description_key, audit_id, is_voided, rule, created_by_id, last_modified_by_id, created_date_time, last_modified_date_time, declarative_rule) values (50117, 'Media form element', (select max(display_order) + 1 from form_element where form_element_group_id = 5569), false, '[]', 108002, 5569, 'ec952777-67b2-47c2-bd7c-8e277547cb6b', 0, 12, 'SingleSelect', null, null, create_audit(), false, null, 1, 1, '2022-04-13 10:51:44.705 +00:00', '2022-04-13 10:51:44.705 +00:00', null);
-update individual set observations = observations || jsonb_build_object('44163589-f76d-447d-9b6e-f5c32aa859eb', 'https://s3.amazon.com/wonderfulImage.jpg'), last_modified_date_time = '2022-04-13 10:51:44.705 +00:00' where subject_type_id = 339;
-update individual set observations = observations || jsonb_build_object('b1815016-40d6-4045-9ae0-2f659fc5db6d', 'https://s3.amazon.com/wonderfulImage.jpg'), last_modified_date_time = '2022-04-13 10:51:44.705 +00:00' where subject_type_id = 339;
+update individual set observations = observations || jsonb_build_object('44163589-f76d-447d-9b6e-f5c32aa859eb', 'https://s3.amazon.com/wonderfulImage1.jpg'), last_modified_date_time = '2022-04-13 10:51:44.705 +00:00' where subject_type_id = 339;
+update individual set observations = observations || jsonb_build_object('b1815016-40d6-4045-9ae0-2f659fc5db6d', 'https://s3.amazon.com/wonderfulImage2.jpg'), last_modified_date_time = '2022-04-13 10:51:44.705 +00:00' where subject_type_id = 339;
 
 
 -- Encounter
 insert into form_element_group (id, name, form_id, uuid, version, display_order, organisation_id, audit_id, is_voided, rule, created_by_id, last_modified_by_id, created_date_time, last_modified_date_time, declarative_rule) values (50571, 'Page', 2660, 'c28ba4eb-f159-4b18-a7c1-baf23d6f7259', 0, 10, 12, create_audit(), false, null, 1, 1, '2022-03-16 15:51:44.666 +00:00', '2022-03-16 15:51:44.666 +00:00', null);
 insert into form_element (id, name, display_order, is_mandatory, key_values, concept_id, form_element_group_id, uuid, version, organisation_id, type, valid_format_regex, valid_format_description_key, audit_id, is_voided, rule, created_by_id, last_modified_by_id, created_date_time, last_modified_date_time, declarative_rule) values (50115, 'Media form element', 1, false, '[]', 108001, 50571, '3ae42c9b-136f-40da-bd9b-2ff638e55683', 0, 12, 'SingleSelect', null, null, create_audit(), false, null, 1, 1, '2022-04-13 10:51:44.705 +00:00', '2022-04-13 10:51:44.705 +00:00', null);
 
 
-update encounter set observations = observations || jsonb_build_object('44163589-f76d-447d-9b6e-f5c32aa859eb', 'https://s3.amazon.com/encounterImage.jpg'), last_modified_date_time = '2022-04-13 10:51:44.705 +00:00' where id = 1900;
+update encounter set observations = observations || jsonb_build_object('44163589-f76d-447d-9b6e-f5c32aa859eb', 'https://s3.amazon.com/encounterImage1.jpg'), last_modified_date_time = '2022-04-13 10:51:44.705 +00:00' where id = 1900;