From 26bfc6d00602e7b3a5055fd3fd0e1858a148ea99 Mon Sep 17 00:00:00 2001 From: Joy A Date: Fri, 15 Nov 2024 12:09:10 +0530 Subject: [PATCH 1/7] avniproject/avni-media#179 | Fix duplicate index creation on media table --- .../etl/repository/rowMappers/MediaTableMetadataBuilder.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/avniproject/etl/repository/rowMappers/MediaTableMetadataBuilder.java b/src/main/java/org/avniproject/etl/repository/rowMappers/MediaTableMetadataBuilder.java index d516bcf..e7f6388 100644 --- a/src/main/java/org/avniproject/etl/repository/rowMappers/MediaTableMetadataBuilder.java +++ b/src/main/java/org/avniproject/etl/repository/rowMappers/MediaTableMetadataBuilder.java @@ -1,5 +1,6 @@ package org.avniproject.etl.repository.rowMappers; +import org.avniproject.etl.domain.metadata.Column; import org.avniproject.etl.domain.metadata.ColumnMetadata; import org.avniproject.etl.domain.metadata.IndexMetadata; import org.avniproject.etl.domain.metadata.TableMetadata; @@ -15,7 +16,7 @@ public static TableMetadata build() { mediaTableMetadata.setName(mediaTable.name(null)); mediaTableMetadata.setType(TableMetadata.Type.Media); mediaTableMetadata.addColumnMetadata(mediaTable.columns().stream().map(column -> new ColumnMetadata(column, null, null, null)).collect(Collectors.toList())); - mediaTableMetadata.addIndexMetadata(mediaTable.columns().stream().map(column -> column.isIndexed() ? new IndexMetadata(new ColumnMetadata(column)) : null).filter(Objects::nonNull).collect(Collectors.toList())); + mediaTable.columns().stream().filter(Column::isIndexed).forEach(column -> mediaTableMetadata.addIndexMetadata(column)); return mediaTableMetadata; } } From a4d91e95ef27c66fa1fe42f12c485026836e35c5 Mon Sep 17 00:00:00 2001 From: himeshr Date: Thu, 21 Nov 2024 17:56:40 +0530 Subject: [PATCH 2/7] #112 | Obtain encounterTableName from SchemaMetadata --- .../etl/domain/metadata/SchemaMetadata.java | 5 +++++ .../sync/CleanEncounterCancelTableAction.java | 19 ++++++++++++++++--- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/avniproject/etl/domain/metadata/SchemaMetadata.java b/src/main/java/org/avniproject/etl/domain/metadata/SchemaMetadata.java index cfd6438..7a61966 100644 --- a/src/main/java/org/avniproject/etl/domain/metadata/SchemaMetadata.java +++ b/src/main/java/org/avniproject/etl/domain/metadata/SchemaMetadata.java @@ -88,6 +88,11 @@ public List getAllEncounterTables() { return tableMetadata.stream().filter(table -> table.getType() == TableMetadata.Type.Encounter).toList(); } + public List getAllIndividualAndProgramEncounterTables() { + return tableMetadata.stream().filter(table -> table.getType() == TableMetadata.Type.Encounter + || table.getType() == TableMetadata.Type.ProgramEncounter).toList(); + } + public List getAllEncounterTableNames() { List encounterTables = getAllEncounterTables(); List encounterTableNames = new ArrayList<>(); diff --git a/src/main/java/org/avniproject/etl/repository/sync/CleanEncounterCancelTableAction.java b/src/main/java/org/avniproject/etl/repository/sync/CleanEncounterCancelTableAction.java index 914d567..bff654b 100644 --- a/src/main/java/org/avniproject/etl/repository/sync/CleanEncounterCancelTableAction.java +++ b/src/main/java/org/avniproject/etl/repository/sync/CleanEncounterCancelTableAction.java @@ -12,6 +12,7 @@ import java.util.Arrays; import java.util.Date; +import java.util.Optional; import static org.avniproject.etl.repository.JdbcContextWrapper.runInOrgContext; @@ -29,13 +30,13 @@ public void perform(TableMetadata tableMetadata, Date lastSyncTime, Date dataSyn if (this.doesntSupport(tableMetadata)) { return; } - cleanUncancelledEncounters(tableMetadata); + cleanUncancelledEncounters(tableMetadata, currentSchemaMetadata); } - private void cleanUncancelledEncounters(TableMetadata tableMetadata) { + private void cleanUncancelledEncounters(TableMetadata tableMetadata, SchemaMetadata currentSchemaMetadata) { String schema = OrgIdentityContextHolder.getDbSchema(); String encounterCancelTableName = tableMetadata.getName(); - String primaryTableName = encounterCancelTableName.substring(0, encounterCancelTableName.length() - 7); + String primaryTableName = getPrimaryTableName(tableMetadata, currentSchemaMetadata); String sql = new ST(deleteUncancelledEncountersSqlTemplate) .add("schemaName", wrapInQuotes(schema)) .add("encounterCancelTableName", wrapInQuotes(encounterCancelTableName)) @@ -47,6 +48,18 @@ private void cleanUncancelledEncounters(TableMetadata tableMetadata) { }, jdbcTemplate); } + private String getPrimaryTableName(TableMetadata tableMetadata, SchemaMetadata currentSchemaMetadata) { + Optional primaryTableMetadata = currentSchemaMetadata.getAllIndividualAndProgramEncounterTables() + .stream().filter(tm -> tm.getEncounterTypeUuid() == tableMetadata.getEncounterTypeUuid() + && tm.getProgramUuid() == tableMetadata.getProgramUuid() + && tm.getSubjectTypeUuid() == tableMetadata.getSubjectTypeUuid()).findAny(); + + if (primaryTableMetadata.isPresent()) { + return primaryTableMetadata.get().getName(); + } + throw new RuntimeException(String.format("Corresponding Primary table not found for cancel table: %s", tableMetadata.getName())); + } + private String wrapInQuotes(String parameter) { return parameter == null ? "null" : "\"" + parameter + "\""; } From 7e0fcd874b48e70166fff717afb743a7cc98f2e3 Mon Sep 17 00:00:00 2001 From: himeshr Date: Thu, 21 Nov 2024 17:56:50 +0530 Subject: [PATCH 3/7] #112 | Obtain enrolmentTableName from SchemaMetadata --- .../sync/CleanEnrolmentExitTableAction.java | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/avniproject/etl/repository/sync/CleanEnrolmentExitTableAction.java b/src/main/java/org/avniproject/etl/repository/sync/CleanEnrolmentExitTableAction.java index 69c89d3..b74ca31 100644 --- a/src/main/java/org/avniproject/etl/repository/sync/CleanEnrolmentExitTableAction.java +++ b/src/main/java/org/avniproject/etl/repository/sync/CleanEnrolmentExitTableAction.java @@ -11,6 +11,7 @@ import org.stringtemplate.v4.ST; import java.util.Date; +import java.util.Optional; import static org.avniproject.etl.repository.JdbcContextWrapper.runInOrgContext; @@ -28,13 +29,13 @@ public void perform(TableMetadata tableMetadata, Date lastSyncTime, Date dataSyn if (this.doesntSupport(tableMetadata)) { return; } - cleanInvalidExits(tableMetadata); + cleanInvalidExits(tableMetadata, currentSchemaMetadata); } - private void cleanInvalidExits(TableMetadata tableMetadata) { + private void cleanInvalidExits(TableMetadata tableMetadata, SchemaMetadata currentSchemaMetadata) { String schema = OrgIdentityContextHolder.getDbSchema(); String exitTableName = tableMetadata.getName(); - String primaryTableName = exitTableName.substring(0, exitTableName.length() - 5); + String primaryTableName = getPrimaryTableName(tableMetadata, currentSchemaMetadata); String sql = new ST(deleteInvalidExitsSqlTemplate) .add("schemaName", wrapInQuotes(schema)) .add("exitTableName", wrapInQuotes(exitTableName)) @@ -46,6 +47,18 @@ private void cleanInvalidExits(TableMetadata tableMetadata) { }, jdbcTemplate); } + + private String getPrimaryTableName(TableMetadata tableMetadata, SchemaMetadata currentSchemaMetadata) { + Optional primaryTableMetadata = currentSchemaMetadata.getAllProgramEnrolmentTables() + .stream().filter(tm -> tm.getProgramUuid() == tableMetadata.getProgramUuid() + && tm.getSubjectTypeUuid() == tableMetadata.getSubjectTypeUuid()).findAny(); + + if (primaryTableMetadata.isPresent()) { + return primaryTableMetadata.get().getName(); + } + throw new RuntimeException(String.format("Corresponding Primary table not found for cancel table: %s", tableMetadata.getName())); + } + private String wrapInQuotes(String parameter) { return parameter == null ? "null" : "\"" + parameter + "\""; } From cd021dd007efa53b8922186e69da3390eebea61e Mon Sep 17 00:00:00 2001 From: Joy A Date: Mon, 18 Nov 2024 17:50:47 +0530 Subject: [PATCH 4/7] #0 | Fix missing space in history query --- .../etl/service/backgroundJob/ScheduledJobService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/avniproject/etl/service/backgroundJob/ScheduledJobService.java b/src/main/java/org/avniproject/etl/service/backgroundJob/ScheduledJobService.java index 9c4af98..0f54ce2 100644 --- a/src/main/java/org/avniproject/etl/service/backgroundJob/ScheduledJobService.java +++ b/src/main/java/org/avniproject/etl/service/backgroundJob/ScheduledJobService.java @@ -24,7 +24,7 @@ public class ScheduledJobService { private final Scheduler scheduler; private final ScheduledJobConfig scheduledJobConfig; - private static final String HISTORY_QUERY = "select sjr.started_at, sjr.ended_at, sjr.error_message, sjr.success from qrtz_job_details qjd\n" + " left outer join scheduled_job_run sjr on sjr.job_name = qjd.job_name\n" + " where qjd.job_name = ? and qjd.job_group = ?" + "order by 1 desc\n"; + private static final String HISTORY_QUERY = "select sjr.started_at, sjr.ended_at, sjr.error_message, sjr.success from qrtz_job_details qjd\n" + " left outer join scheduled_job_run sjr on sjr.job_name = qjd.job_name\n" + " where qjd.job_name = ? and qjd.job_group = ? order by 1 desc "; private static final String JOB_LIST_QUERY = "select organisationUUID, job_name from (SELECT unnest(string_to_array(?, ',')) as organisationUUID) foo\n" + " left outer join qrtz_job_details qjd on organisationUUID = qjd.job_name where qjd.job_group = ? or qjd is null"; From 1672ee2748dd9bda49c849eef32efc117a332580 Mon Sep 17 00:00:00 2001 From: himeshr Date: Thu, 21 Nov 2024 19:38:49 +0530 Subject: [PATCH 5/7] #112 | Use equals and null-safe checks --- .../repository/sync/CleanEncounterCancelTableAction.java | 7 ++++--- .../etl/repository/sync/CleanEnrolmentExitTableAction.java | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/avniproject/etl/repository/sync/CleanEncounterCancelTableAction.java b/src/main/java/org/avniproject/etl/repository/sync/CleanEncounterCancelTableAction.java index bff654b..d04fe62 100644 --- a/src/main/java/org/avniproject/etl/repository/sync/CleanEncounterCancelTableAction.java +++ b/src/main/java/org/avniproject/etl/repository/sync/CleanEncounterCancelTableAction.java @@ -12,6 +12,7 @@ import java.util.Arrays; import java.util.Date; +import java.util.Objects; import java.util.Optional; import static org.avniproject.etl.repository.JdbcContextWrapper.runInOrgContext; @@ -50,9 +51,9 @@ private void cleanUncancelledEncounters(TableMetadata tableMetadata, SchemaMetad private String getPrimaryTableName(TableMetadata tableMetadata, SchemaMetadata currentSchemaMetadata) { Optional primaryTableMetadata = currentSchemaMetadata.getAllIndividualAndProgramEncounterTables() - .stream().filter(tm -> tm.getEncounterTypeUuid() == tableMetadata.getEncounterTypeUuid() - && tm.getProgramUuid() == tableMetadata.getProgramUuid() - && tm.getSubjectTypeUuid() == tableMetadata.getSubjectTypeUuid()).findAny(); + .stream().filter(tm -> tm.getEncounterTypeUuid().equals(tableMetadata.getEncounterTypeUuid()) + && (Objects.isNull(tm.getProgramUuid()) || (Objects.nonNull(tableMetadata.getProgramUuid()) && tableMetadata.getProgramUuid().equals(tm.getProgramUuid()))) + && tm.getSubjectTypeUuid().equals(tableMetadata.getSubjectTypeUuid())).findAny(); if (primaryTableMetadata.isPresent()) { return primaryTableMetadata.get().getName(); diff --git a/src/main/java/org/avniproject/etl/repository/sync/CleanEnrolmentExitTableAction.java b/src/main/java/org/avniproject/etl/repository/sync/CleanEnrolmentExitTableAction.java index b74ca31..93f5c32 100644 --- a/src/main/java/org/avniproject/etl/repository/sync/CleanEnrolmentExitTableAction.java +++ b/src/main/java/org/avniproject/etl/repository/sync/CleanEnrolmentExitTableAction.java @@ -50,8 +50,8 @@ private void cleanInvalidExits(TableMetadata tableMetadata, SchemaMetadata curre private String getPrimaryTableName(TableMetadata tableMetadata, SchemaMetadata currentSchemaMetadata) { Optional primaryTableMetadata = currentSchemaMetadata.getAllProgramEnrolmentTables() - .stream().filter(tm -> tm.getProgramUuid() == tableMetadata.getProgramUuid() - && tm.getSubjectTypeUuid() == tableMetadata.getSubjectTypeUuid()).findAny(); + .stream().filter(tm -> tm.getProgramUuid().equals(tableMetadata.getProgramUuid()) + && tm.getSubjectTypeUuid().equals(tableMetadata.getSubjectTypeUuid())).findAny(); if (primaryTableMetadata.isPresent()) { return primaryTableMetadata.get().getName(); From 882e4b438e612e81bee32b250469fa6f46f6054c Mon Sep 17 00:00:00 2001 From: himeshr Date: Mon, 25 Nov 2024 11:45:56 +0530 Subject: [PATCH 6/7] #000 | Use app property for simplifying local run with diff db port --- src/main/resources/application.properties | 2 ++ src/main/resources/main-application.properties | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index f4ef015..14cc23d 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -1,4 +1,6 @@ avni.database=${OPENCHS_DATABASE_NAME:openchs} +avni.database.port=${OPENCHS_DATABASE_PORT:5432} +avni.database.server=${OPENCHS_DATABASE_SERVER:localhost} spring.config.import=classpath:/main-application.properties debug=${ETL_DEBUG_MODE:false} avni.current.time.offset.seconds=${AVNI_CURRENT_TIME_OFFSET_SECONDS:10} diff --git a/src/main/resources/main-application.properties b/src/main/resources/main-application.properties index aade73e..1d9c5c1 100644 --- a/src/main/resources/main-application.properties +++ b/src/main/resources/main-application.properties @@ -1,7 +1,7 @@ app.name=Avni ETL app.description=${app.name} is used to create and update the analytics schema for organisations. server.port=${ETL_SERVER_PORT:8022} -spring.datasource.url=${OPENCHS_DATABASE_URL:jdbc:postgresql://localhost:5432/${avni.database}} +spring.datasource.url=${OPENCHS_DATABASE_URL:jdbc:postgresql://${avni.database.server}:${avni.database.port}/${avni.database}} spring.datasource.username=${OPENCHS_DATABASE_USER:openchs} spring.datasource.password=${OPENCHS_DATABASE_PASSWORD:password} spring.datasource.driverClassName=org.postgresql.Driver From 7cf5560092de5882ef050e11515cb637940a30e6 Mon Sep 17 00:00:00 2001 From: himeshr Date: Mon, 25 Nov 2024 12:19:43 +0530 Subject: [PATCH 7/7] #000 | Fix test failures --- src/test/resources/application.properties | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/resources/application.properties b/src/test/resources/application.properties index c393a95..bded3ac 100644 --- a/src/test/resources/application.properties +++ b/src/test/resources/application.properties @@ -1,4 +1,6 @@ avni.database=openchs_test +avni.database.port=${OPENCHS_DATABASE_PORT:5432} +avni.database.server=${OPENCHS_DATABASE_SERVER:localhost} spring.config.import=classpath:/main-application.properties command.line.runner.enabled=false avni.current.time.offset.seconds=0