diff --git a/src/main/resources/sql/etl/deleteDuplicateMedia.sql.st b/src/main/resources/sql/etl/deleteDuplicateMedia.sql.st index c2216d3..70ab826 100644 --- a/src/main/resources/sql/etl/deleteDuplicateMedia.sql.st +++ b/src/main/resources/sql/etl/deleteDuplicateMedia.sql.st @@ -2,7 +2,7 @@ DELETE FROM ""."media" t2 WHERE t2.id IN ( SELECT duplicates.dup_id - FROM (SELECT t1.id AS dup_id, ROW_NUMBER() OVER (PARTITION BY uuid, entity_id) AS occurence + FROM (SELECT t1.id AS dup_id, RANK() OVER (PARTITION BY entity_id ORDER BY last_modified_date_time DESC) AS rank FROM ""."media" t1) AS duplicates - WHERE duplicates.occurence > 1 + WHERE duplicates.rank > 1 ); \ No newline at end of file diff --git a/src/test/java/org/avniproject/etl/DataSyncIntegrationTest.java b/src/test/java/org/avniproject/etl/DataSyncIntegrationTest.java index c1a17d6..c8e1bb2 100644 --- a/src/test/java/org/avniproject/etl/DataSyncIntegrationTest.java +++ b/src/test/java/org/avniproject/etl/DataSyncIntegrationTest.java @@ -459,13 +459,22 @@ public void shouldPopulateMediaTableCorrectlyWhenTransactionalDataUpdates() thro @Sql(scripts = "/test-data-teardown.sql", executionPhase = Sql.ExecutionPhase.AFTER_TEST_METHOD) public void arrayStyleMediaObservationsCreateMultipleRowsInMediaTable() throws InterruptedException { String encounterImages = "[\"https://s3.amazon.com/image1.jpg\", \"https://s3.amazon.com/image2.jpg\"]"; - String sql = "update encounter set observations = observations || '{\"44163589-f76d-447d-9b6e-f5c32aa859eb\": " + encounterImages + "}'::jsonb where id = 1900;"; + String sql = "update encounter set last_modified_date_time = now(), observations = observations || '{\"44163589-f76d-447d-9b6e-f5c32aa859eb\": " + encounterImages + "}'::jsonb where id = 1900;"; jdbcTemplate.execute(sql); runDataSync(); List> media = jdbcTemplate.queryForList("select * from orgc.media;"); assertThat("Media table number of rows has not changed since last run", media.size(), is(6)); + + String encounterImagesSecondSet = "[\"https://s3.amazon.com/image3.jpg\", \"https://s3.amazon.com/image4.jpg\", \"https://s3.amazon.com/image5.jpg\"]"; + String sqlSecondItr = "update encounter set last_modified_date_time = now(), observations = observations || '{\"44163589-f76d-447d-9b6e-f5c32aa859eb\": " + encounterImagesSecondSet + "}'::jsonb where id = 1900;"; + jdbcTemplate.execute(sqlSecondItr); + + runDataSync(); + + List> mediaSecondItr = jdbcTemplate.queryForList("select * from orgc.media;"); + assertThat("Media table number of rows was supposed to increase since last run", mediaSecondItr.size(), is(7)); } @Test diff --git a/src/test/resources/media-form-element.sql b/src/test/resources/media-form-element.sql index 28b9787..b62f260 100644 --- a/src/test/resources/media-form-element.sql +++ b/src/test/resources/media-form-element.sql @@ -3,8 +3,8 @@ insert into concept (id, data_type, high_absolute, high_normal, low_absolute, lo insert into concept (id, data_type, high_absolute, high_normal, low_absolute, low_normal, name, uuid, version, unit, organisation_id, is_voided, audit_id, key_values, active, created_by_id, last_modified_by_id, created_date_time, last_modified_date_time) values (108002, 'Image', null, null, null, null, 'Person''s Image','b1815016-40d6-4045-9ae0-2f659fc5db6d', 0, null, 12, false, create_audit(), null, true, 1, 1, '2022-04-13 15:49:55.019 +00:00', '2022-04-13 15:49:55.019 +00:00'); insert into form_element (id, name, display_order, is_mandatory, key_values, concept_id, form_element_group_id, uuid, version, organisation_id, type, valid_format_regex, valid_format_description_key, audit_id, is_voided, rule, created_by_id, last_modified_by_id, created_date_time, last_modified_date_time, declarative_rule) values (50116, 'Media form element', 8, false, '[]', 108001, 5569, 'fd5ad16f-c18e-45db-841c-a2f994862957', 0, 12, 'SingleSelect', null, null, create_audit(), false, null, 1, 1, '2022-04-13 10:51:44.705 +00:00', '2022-04-13 10:51:44.705 +00:00', null); insert into form_element (id, name, display_order, is_mandatory, key_values, concept_id, form_element_group_id, uuid, version, organisation_id, type, valid_format_regex, valid_format_description_key, audit_id, is_voided, rule, created_by_id, last_modified_by_id, created_date_time, last_modified_date_time, declarative_rule) values (50117, 'Media form element', (select max(display_order) + 1 from form_element where form_element_group_id = 5569), false, '[]', 108002, 5569, 'ec952777-67b2-47c2-bd7c-8e277547cb6b', 0, 12, 'SingleSelect', null, null, create_audit(), false, null, 1, 1, '2022-04-13 10:51:44.705 +00:00', '2022-04-13 10:51:44.705 +00:00', null); -update individual set observations = observations || jsonb_build_object('44163589-f76d-447d-9b6e-f5c32aa859eb', 'https://s3.amazon.com/wonderfulImage.jpg'), last_modified_date_time = '2022-04-13 10:51:44.705 +00:00' where subject_type_id = 339; -update individual set observations = observations || jsonb_build_object('b1815016-40d6-4045-9ae0-2f659fc5db6d', 'https://s3.amazon.com/wonderfulImage.jpg'), last_modified_date_time = '2022-04-13 10:51:44.705 +00:00' where subject_type_id = 339; +update individual set observations = observations || jsonb_build_object('44163589-f76d-447d-9b6e-f5c32aa859eb', 'https://s3.amazon.com/wonderfulImage1.jpg'), last_modified_date_time = '2022-04-13 10:51:44.705 +00:00' where subject_type_id = 339; +update individual set observations = observations || jsonb_build_object('b1815016-40d6-4045-9ae0-2f659fc5db6d', 'https://s3.amazon.com/wonderfulImage2.jpg'), last_modified_date_time = '2022-04-13 10:51:44.705 +00:00' where subject_type_id = 339; -- Encounter @@ -12,5 +12,5 @@ insert into form_element_group (id, name, form_id, uuid, version, display_order, insert into form_element (id, name, display_order, is_mandatory, key_values, concept_id, form_element_group_id, uuid, version, organisation_id, type, valid_format_regex, valid_format_description_key, audit_id, is_voided, rule, created_by_id, last_modified_by_id, created_date_time, last_modified_date_time, declarative_rule) values (50115, 'Media form element', 1, false, '[]', 108001, 50571, '3ae42c9b-136f-40da-bd9b-2ff638e55683', 0, 12, 'SingleSelect', null, null, create_audit(), false, null, 1, 1, '2022-04-13 10:51:44.705 +00:00', '2022-04-13 10:51:44.705 +00:00', null); -update encounter set observations = observations || jsonb_build_object('44163589-f76d-447d-9b6e-f5c32aa859eb', 'https://s3.amazon.com/encounterImage.jpg'), last_modified_date_time = '2022-04-13 10:51:44.705 +00:00' where id = 1900; +update encounter set observations = observations || jsonb_build_object('44163589-f76d-447d-9b6e-f5c32aa859eb', 'https://s3.amazon.com/encounterImage1.jpg'), last_modified_date_time = '2022-04-13 10:51:44.705 +00:00' where id = 1900;