diff --git a/build.gradle b/build.gradle index 254dc91..b3799d2 100644 --- a/build.gradle +++ b/build.gradle @@ -30,6 +30,12 @@ dependencies { implementation 'org.keycloak:keycloak-admin-client:21.0.2' implementation 'org.antlr:ST4:4.3.4' implementation 'org.hibernate:hibernate-validator:7.0.5.Final' + implementation "joda-time:joda-time:2.9.4" + implementation 'com.fasterxml.jackson.core:jackson-databind:2.12.6' + implementation 'com.fasterxml.jackson.core:jackson-core:2.12.6' + implementation 'com.fasterxml.jackson.core:jackson-annotations:2.12.6' + implementation 'com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.12.6' + implementation 'com.fasterxml.jackson.datatype:jackson-datatype-joda:2.12.6' } tasks.named('test') { diff --git a/src/main/java/org/avniproject/etl/controller/ReportController.java b/src/main/java/org/avniproject/etl/controller/ReportController.java new file mode 100644 index 0000000..05c68d2 --- /dev/null +++ b/src/main/java/org/avniproject/etl/controller/ReportController.java @@ -0,0 +1,117 @@ +package org.avniproject.etl.controller; + +import org.avniproject.etl.dto.AggregateReportResult; +import org.avniproject.etl.dto.UserActivityDTO; +import org.avniproject.etl.repository.ReportRepository; +import org.avniproject.etl.util.ReportUtil; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.*; + +import java.util.List; + +@RestController +public class ReportController { + + private final ReportRepository reportRepository; + private final ReportUtil reportUtil; + + @Autowired + public ReportController(ReportRepository reportRepository, ReportUtil reportUtil) { + this.reportRepository = reportRepository; + this.reportUtil = reportUtil; + } + + @PreAuthorize("hasAnyAuthority('analytics_user')") + @RequestMapping(value = "/report/aggregate/summaryTable", method = RequestMethod.GET) + public List getSummaryTable(){ + return reportRepository.generateSummaryTable(); + } + + @PreAuthorize("hasAnyAuthority('analytics_user')") + @RequestMapping(value = "report/hr/userActivity", method = RequestMethod.GET) + public List getUserActivity(@RequestParam(value = "startDate", required = false) String startDate, + @RequestParam(value = "endDate", required = false) String endDate){ + return reportRepository.generateUserActivity( + reportUtil.getDateDynamicWhere(startDate, endDate, "registration_date"), + reportUtil.getDateDynamicWhere(startDate, endDate, "encounter_date_time"), + reportUtil.getDateDynamicWhere(startDate, endDate, "enrolment_date_time")); + } + + @PreAuthorize("hasAnyAuthority('analytics_user')") + @RequestMapping(value = "/report/hr/syncFailures",method = RequestMethod.GET) + public List getUserWiseSyncFailures(@RequestParam(value = "startDate", required = false) String startDate, + @RequestParam(value = "endDate", required = false) String endDate){ + return reportRepository.generateUserSyncFailures( + reportUtil.getDateDynamicWhere(startDate, endDate, "st.sync_start_time") + ); + } + + @PreAuthorize("hasAnyAuthority('analytics_user')") + @RequestMapping(value = "/report/hr/deviceModels", method = RequestMethod.GET) + public List getUserWiseDeviceModels() { + + return reportRepository.generateUserDeviceModels(); + } + + @PreAuthorize("hasAnyAuthority('analytics_user')") + @RequestMapping(value = "/report/hr/appVersions", method = RequestMethod.GET) + public List getUserWiseAppVersions() { + + return reportRepository.generateUserAppVersions(); + } + + @PreAuthorize("hasAnyAuthority('analytics_user')") + @RequestMapping(value = "/report/hr/userDetails", method = RequestMethod.GET) + public List getUserDetails() { + + return reportRepository.generateUserDetails(); + } + + @PreAuthorize("hasAnyAuthority('analytics_user')") + @RequestMapping(value = "/report/hr/latestSyncs", method = RequestMethod.GET) + public List getLatestSyncs(@RequestParam(value = "startDate", required = false) String startDate, + @RequestParam(value = "endDate", required = false) String endDate) { + + return reportRepository.generateLatestSyncs( + reportUtil.getDateDynamicWhere(startDate, endDate, "st.sync_end_time")); + } + + @PreAuthorize("hasAnyAuthority('analytics_user')") + @RequestMapping(value = "/report/hr/medianSync", method = RequestMethod.GET) + public List getMedianSync(@RequestParam(value = "startDate", required = false) String startDate, + @RequestParam(value = "endDate", required = false) String endDate) { + + return reportRepository.generateMedianSync( + reportUtil.getDateSeries(startDate, endDate)); + } + + @PreAuthorize("hasAnyAuthority('analytics_user')") + @RequestMapping(value = "/report/hr/championUsers", method = RequestMethod.GET) + public List getChampionUsers(@RequestParam(value = "startDate", required = false) String startDate, + @RequestParam(value = "endDate", required = false) String endDate) { + return reportRepository.generateCompletedVisitsOnTimeByProportion( + ">= 0.5", + reportUtil.getDateDynamicWhere(startDate, endDate, "encounter_date_time")); + } + + @PreAuthorize("hasAnyAuthority('analytics_user')") + @RequestMapping(value = "/report/hr/nonPerformingUsers", method = RequestMethod.GET) + public List getNonPerformingUsers(@RequestParam(value = "startDate", required = false) String startDate, + @RequestParam(value = "endDate", required = false) String endDate) { + return reportRepository.generateCompletedVisitsOnTimeByProportion( + "<= 0.5", + reportUtil.getDateDynamicWhere(startDate, endDate, "encounter_date_time")); + } + + @PreAuthorize("hasAnyAuthority('analytics_user')") + @RequestMapping(value = "/report/hr/mostCancelled", method = RequestMethod.GET) + public List getUsersCancellingMostVisits(@RequestParam(value = "startDate", required = false) String startDate, + @RequestParam(value = "endDate", required = false) String endDate) { + return reportRepository.generateUserCancellingMostVisits( + reportUtil.getDateDynamicWhere(startDate, endDate, "encounter_date_time")); + } + + +} + diff --git a/src/main/java/org/avniproject/etl/domain/metadata/SchemaMetadata.java b/src/main/java/org/avniproject/etl/domain/metadata/SchemaMetadata.java index cb80569..48c6b0e 100644 --- a/src/main/java/org/avniproject/etl/domain/metadata/SchemaMetadata.java +++ b/src/main/java/org/avniproject/etl/domain/metadata/SchemaMetadata.java @@ -42,18 +42,54 @@ public List getAllSubjectTables() { return tableMetadata.stream().filter(TableMetadata::isSubjectTable).toList(); } + public List getAllSubjectTableNames() { + List subjectTables = getAllSubjectTables(); + List subjectTableNames = new ArrayList<>(); + for(TableMetadata subject : subjectTables){ + subjectTableNames.add(subject.getName()); + } + return subjectTableNames; + } + public List getAllProgramEnrolmentTables() { return tableMetadata.stream().filter(table -> table.getType() == TableMetadata.Type.ProgramEnrolment).toList(); } + public List getAllProgramEnrolmentTableNames() { + List programEnrolmentTables = getAllProgramEnrolmentTables(); + List programEnrolmentTableNames = new ArrayList<>(); + for(TableMetadata programEnrolment : programEnrolmentTables){ + programEnrolmentTableNames.add(programEnrolment.getName()); + } + return programEnrolmentTableNames; + } + public List getAllProgramEncounterTables() { return tableMetadata.stream().filter(table -> table.getType() == TableMetadata.Type.ProgramEncounter).toList(); } + public List getAllProgramEncounterTableNames() { + List programEncounterTables = getAllProgramEncounterTables(); + List programEncounterTableNames = new ArrayList<>(); + for(TableMetadata programEncounter : programEncounterTables){ + programEncounterTableNames.add(programEncounter.getName()); + } + return programEncounterTableNames; + } + public List getAllEncounterTables() { return tableMetadata.stream().filter(table -> table.getType() == TableMetadata.Type.Encounter).toList(); } + public List getAllEncounterTableNames() { + List encounterTables = getAllEncounterTables(); + List encounterTableNames = new ArrayList<>(); + for(TableMetadata encounter : encounterTables){ + encounterTableNames.add(encounter.getName()); + } + return encounterTableNames; + } + private List findChanges(SchemaMetadata currentSchema, TableMetadata newTable) { List diffs = new ArrayList<>(); Optional optionalMatchingTable = currentSchema.findMatchingTable(newTable); diff --git a/src/main/java/org/avniproject/etl/dto/AggregateReportResult.java b/src/main/java/org/avniproject/etl/dto/AggregateReportResult.java new file mode 100644 index 0000000..e684c03 --- /dev/null +++ b/src/main/java/org/avniproject/etl/dto/AggregateReportResult.java @@ -0,0 +1,31 @@ +package org.avniproject.etl.dto; + +public class AggregateReportResult { + private String label; + private Long value; + private String id; + + public String getLabel() { + return label; + } + + public void setLabel(String label) { + this.label = label; + } + + public Long getValue() { + return value; + } + + public void setValue(Long value) { + this.value = value; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } +} diff --git a/src/main/java/org/avniproject/etl/dto/UserActivityDTO.java b/src/main/java/org/avniproject/etl/dto/UserActivityDTO.java new file mode 100644 index 0000000..5f555e3 --- /dev/null +++ b/src/main/java/org/avniproject/etl/dto/UserActivityDTO.java @@ -0,0 +1,169 @@ +package org.avniproject.etl.dto; + +import org.joda.time.DateTime; + +public class UserActivityDTO { + + private String tableName; + private String tableType; + private String userName; + private Long id; + private Long registrationCount; + private Long programEnrolmentCount; + private Long programEncounterCount; + private Long generalEncounterCount; + private Long count; + private String androidVersion; + private String appVersion; + private String deviceModel; + private String syncStatus; + private String syncSource; + private DateTime syncStart; + private DateTime syncEnd; + private DateTime lastSuccessfulSync; + private String medianSync; + + public String getTableName() { + return tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + + public String getTableType() { + return tableType; + } + + public void setTableType(String tableType) { + this.tableType = tableType; + } + + public String getUserName() { + return userName; + } + + public void setUserName(String userName) { + this.userName = userName; + } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getRegistrationCount() { + return registrationCount; + } + + public void setRegistrationCount(Long registrationCount) { + this.registrationCount = registrationCount; + } + + public Long getProgramEnrolmentCount() { + return programEnrolmentCount; + } + + public void setProgramEnrolmentCount(Long programEnrolmentCount) { + this.programEnrolmentCount = programEnrolmentCount; + } + + public Long getProgramEncounterCount() { + return programEncounterCount; + } + + public void setProgramEncounterCount(Long programEncounterCount) { + this.programEncounterCount = programEncounterCount; + } + + public Long getGeneralEncounterCount() { + return generalEncounterCount; + } + + public void setGeneralEncounterCount(Long generalEncounterCount) { + this.generalEncounterCount = generalEncounterCount; + } + + public Long getCount() { + return count; + } + + public void setCount(Long count) { + this.count = count; + } + + public String getAndroidVersion() { + return androidVersion; + } + + public void setAndroidVersion(String androidVersion) { + this.androidVersion = androidVersion; + } + + public String getAppVersion() { + return appVersion; + } + + public void setAppVersion(String appVersion) { + this.appVersion = appVersion; + } + + public String getDeviceModel() { + return deviceModel; + } + + public void setDeviceModel(String deviceModel) { + this.deviceModel = deviceModel; + } + + public DateTime getLastSuccessfulSync() { + return lastSuccessfulSync; + } + + public void setLastSuccessfulSync(DateTime lastSuccessfulSync) { + this.lastSuccessfulSync = lastSuccessfulSync; + } + + public DateTime getSyncStart() { + return syncStart; + } + + public void setSyncStart(DateTime syncStart) { + this.syncStart = syncStart; + } + + public DateTime getSyncEnd() { + return syncEnd; + } + + public void setSyncEnd(DateTime syncEnd) { + this.syncEnd = syncEnd; + } + + public String getSyncStatus() { + return syncStatus; + } + + public void setSyncStatus(String syncStatus) { + this.syncStatus = syncStatus; + } + + public String getSyncSource() { + return syncSource; + } + + public void setSyncSource(String syncSource) { + this.syncSource = syncSource; + } + + public String getMedianSync() { + return medianSync; + } + + public void setMedianSync(String medianSync) { + this.medianSync = medianSync; + } +} diff --git a/src/main/java/org/avniproject/etl/repository/ReportRepository.java b/src/main/java/org/avniproject/etl/repository/ReportRepository.java new file mode 100644 index 0000000..0b98631 --- /dev/null +++ b/src/main/java/org/avniproject/etl/repository/ReportRepository.java @@ -0,0 +1,441 @@ +package org.avniproject.etl.repository; + +import org.avniproject.etl.domain.NullObject; +import org.avniproject.etl.domain.OrgIdentityContextHolder; +import org.avniproject.etl.domain.metadata.SchemaMetadata; +import org.avniproject.etl.dto.AggregateReportResult; +import org.avniproject.etl.dto.UserActivityDTO; +import org.avniproject.etl.repository.rowMappers.reports.*; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.stereotype.Component; +import org.stringtemplate.v4.ST; + +import java.util.List; + +import static org.avniproject.etl.repository.JdbcContextWrapper.runInOrgContext; + +@Component +public class ReportRepository { + private final NamedParameterJdbcTemplate namedJdbcTemplate; + private final SchemaMetadataRepository schemaMetadataRepository; + private final JdbcTemplate jdbcTemplate; + + @Autowired + public ReportRepository(NamedParameterJdbcTemplate namedJdbcTemplate, SchemaMetadataRepository schemaMetadataRepository, JdbcTemplate jdbcTemplate) { + this.namedJdbcTemplate = namedJdbcTemplate; + this.schemaMetadataRepository = schemaMetadataRepository; + this.jdbcTemplate = jdbcTemplate; + } + + public List generateSummaryTable(){ + String baseQuery = "select name, type \n" + + "from public.table_metadata\n" + + "where schema_name = '${schemaName}'\n" + + "order by type;"; + String query= baseQuery.replace("${schemaName}", OrgIdentityContextHolder.getDbSchema()); + + runInOrgContext(() -> { + jdbcTemplate.execute(query); + return NullObject.instance(); + }, jdbcTemplate); + + return namedJdbcTemplate.query(query, new SummaryTableMapper()); + } + + public List generateUserActivity(String subjectWhere, String encounterWhere, String enrolmentWhere) { + SchemaMetadata schema = schemaMetadataRepository.getExistingSchemaMetadata(); + List subjectTableNames = schema.getAllSubjectTableNames().stream().toList(); + List encounterTableNames = schema.getAllEncounterTableNames().stream().toList(); + List programEnrolmentTableNames = schema.getAllProgramEnrolmentTableNames().stream().toList(); + List programEncounterTableNames = schema.getAllProgramEncounterTableNames().stream().toList(); + + ST baseQuery = new ST("with registrations as (\n" + + " select last_modified_by_id, count(*) as registration_count\n" + + " from $schemaName.\n" + + " where is_voided = false\n" + + " $subjectWhere\n" + + " group by last_modified_by_id\n" + + " \n" + + " where is_voided = false\n" + + " $subjectWhere \n" + + " group by last_modified_by_id \n}> " + + " ),\n " + + "encounters as (\n" + + " select last_modified_by_id, count(*) as encounter_count\n" + + " from $schemaName.\n" + + " where is_voided = false\n" + + " $encounterWhere\n" + + " group by last_modified_by_id\n" + + " \n" + + " where is_voided = false\n" + + " $encounterWhere \n" + + " group by last_modified_by_id \n}> " + + " ),\n " + + "enrolments as (\n" + + " select last_modified_by_id, count(*) as enrolment_count\n" + + " from $schemaName.\n" + + " where is_voided = false\n" + + " $enrolmentWhere\n" + + " group by last_modified_by_id\n" + + " \n" + + " where is_voided = false\n" + + " $enrolmentWhere \n" + + " group by last_modified_by_id \n}> " + + " ),\n " + + "program_encounters as (\n" + + " select last_modified_by_id, count(*) as program_encounter_count\n" + + " from $schemaName.\n" + + " where is_voided = false\n" + + " $encounterWhere\n" + + " group by last_modified_by_id\n" + + " \n" + + " where is_voided = false\n" + + " $encounterWhere \n" + + " group by last_modified_by_id \n}> " + + " ),\n " + + "activity_table as (\n" + + " select u.id as id,\n" + + " coalesce(u.name, u.username) as name,\n" + + " coalesce(registration_count, 0) as registration_count,\n" + + " coalesce(encounter_count, 0) as encounter_count,\n" + + " coalesce(enrolment_count, 0) as enrolment_count,\n" + + " coalesce(program_encounter_count, 0) as program_encounter_count\n" + + " from $schemaName.users u\n" + + " left join registrations r on r.last_modified_by_id = u.id\n" + + " left join encounters e on e.last_modified_by_id = u.id\n" + + " left join enrolments enl on enl.last_modified_by_id = u.id\n" + + " left join program_encounters enc on enc.last_modified_by_id = u.id\n" + + " where (u.is_voided = false or u.is_voided isnull) and u.organisation_id notnull\n" + + " and coalesce(coalesce(registration_count, 0) + coalesce(encounter_count, 0) + coalesce(enrolment_count, 0) +\n" + + " coalesce(program_encounter_count, 0), 0) > 0\n" + + "),\n" + + "final_table as (\n" + + " select id, name,\n"+ + " sum(distinct registration_count) as registration_count, \n" + + " sum(distinct encounter_count) as encounter_count,\n" + + " sum(distinct enrolment_count) as enrolment_count,\n" + + " sum(distinct program_encounter_count) as program_encounter_count\n" + + " from activity_table \n" + + " group by id,name\n" + + ")\n" + + "select *, coalesce(coalesce(registration_count, 0) + coalesce(encounter_count, 0) + coalesce(enrolment_count, 0) +\n" + + " coalesce(program_encounter_count, 0), 0) as total\n" + + "from final_table\n" + + "order by 7 desc\n" + + "limit 10;" + ); + baseQuery.add("subjectTableNames", subjectTableNames) + .add("encounterTableNames", encounterTableNames) + .add("programEnrolmentTableNames", programEnrolmentTableNames) + .add("programEncounterTableNames", programEncounterTableNames); + String query = baseQuery.render().replace("$schemaName", OrgIdentityContextHolder.getDbSchema()) + .replace("$subjectWhere", subjectWhere) + .replace("$encounterWhere", encounterWhere) + .replace("$enrolmentWhere", enrolmentWhere); + + runInOrgContext(() -> { + jdbcTemplate.execute(query); + return NullObject.instance(); + }, jdbcTemplate); + + return namedJdbcTemplate.query(query, new UserActivityMapper()); + } + + public List generateUserSyncFailures(String syncTelemetryWhere) { + String baseQuery = "select coalesce(u.name, u.username) as name, \n" + + " count(*) as count\n" + + "from ${schemaName}.sync_telemetry st\n" + + " join ${schemaName}.users u on st.user_id = u.id\n" + + "where sync_status = 'incomplete'\n" + + "and (u.is_voided = false or u.is_voided isnull)\n" + + "and u.organisation_id notnull\n" + + "${syncTelemetryWhere}\n"+ + "group by 1\n" + + "order by 2 desc\n" + + "limit 10;"; + String query = baseQuery + .replace("${schemaName}", OrgIdentityContextHolder.getDbSchema()) + .replace("${syncTelemetryWhere}", syncTelemetryWhere); + + runInOrgContext(() -> { + jdbcTemplate.execute(query); + return NullObject.instance(); + }, jdbcTemplate); + + return namedJdbcTemplate.query(query, new UserCountMapper()); + } + + public List generateUserAppVersions() { + String baseQuery = "select app_version as indicator,\n" + + " count(*) as count\n" + + "from ${schemaName}.users u\n" + + " join\n" + + " (select user_id,\n" + + " app_version,\n" + + " row_number() over (partition by user_id order by sync_start_time desc ) as rn\n" + + " from ${schemaName}.sync_telemetry) l on l.user_id = u.id and rn = 1\n" + + "where (u.is_voided = false or u.is_voided isnull) and u.organisation_id notnull\n" + + "group by app_version;"; + String query = baseQuery + .replace("${schemaName}", OrgIdentityContextHolder.getDbSchema()); + + runInOrgContext(() -> { + jdbcTemplate.execute(query); + return NullObject.instance(); + }, jdbcTemplate); + + return namedJdbcTemplate.query(query, new AggregateReportMapper()); + } + + public List generateUserDeviceModels() { + String baseQuery = "select device_model as indicator,\n" + + " count(*) as count\n" + + "from ${schemaName}.users u\n" + + " join\n" + + " (select user_id,\n" + + " device_name as device_model,\n" + + " row_number() over (partition by user_id order by sync_start_time desc ) as rn\n" + + " from ${schemaName}.sync_telemetry) l on l.user_id = u.id and rn = 1\n" + + "where (u.is_voided = false or u.is_voided isnull) and u.organisation_id notnull \n" + + "group by device_model;"; + String query = baseQuery + .replace("${schemaName}", OrgIdentityContextHolder.getDbSchema()); + + runInOrgContext(() -> { + jdbcTemplate.execute(query); + return NullObject.instance(); + }, jdbcTemplate); + + return namedJdbcTemplate.query(query, new AggregateReportMapper()); + } + + public List generateUserDetails() { + String baseQuery = "select coalesce(u.name, u.username) as name,\n" + + " app_version,\n" + + " device_model,\n" + + " sync_start_time\n" + + "from ${schemaName}.users u\n" + + " join\n" + + " (select user_id,\n" + + " app_version,\n" + + " device_name as device_model,\n" + + " sync_start_time,\n" + + " row_number() over (partition by user_id order by sync_start_time desc ) as rn\n" + + " from ${schemaName}.sync_telemetry\n" + + " where sync_status = 'complete') l on l.user_id = u.id and rn = 1\n" + + "where (u.is_voided = false or u.is_voided isnull)\n" + + " and u.organisation_id notnull\n" + + "order by 1 desc;"; + String query = baseQuery + .replace("${schemaName}", OrgIdentityContextHolder.getDbSchema()); + + runInOrgContext(() -> { + jdbcTemplate.execute(query); + return NullObject.instance(); + }, jdbcTemplate); + + return namedJdbcTemplate.query(query, new UserDetailsMapper()); + } + + public List generateLatestSyncs(String syncTelemetryWhere) { + String baseQuery = "SELECT coalesce(u.name,u.username) as name, \n" + + " android_version, app_version, device_name, sync_start_time, sync_end_time, sync_status, sync_source\n" + + "FROM public.sync_telemetry st\n" + + "join ${schemaName}.users u on st.last_modified_by_id = u.id\n" + + "where (u.is_voided = false or u.is_voided isnull) and u.organisation_id notnull\n" + + "${syncTelemetryWhere}\n"+ + "order by 6 desc\n" + + "limit 10;\n"; + String query = baseQuery + .replace("${schemaName}", OrgIdentityContextHolder.getDbSchema()) + .replace("${syncTelemetryWhere}", syncTelemetryWhere); + + runInOrgContext(() -> { + jdbcTemplate.execute(query); + return NullObject.instance(); + }, jdbcTemplate); + + return namedJdbcTemplate.query(query, new LatestSyncMapper()); + } + + public List generateMedianSync(String syncTelemetryWhere) { + String baseQuery = "with weeks as (\n" + + " select day::date start_date, day::date+6 end_date\n" + + " ${syncTelemetryWhere}\n" + + ")\n" + + "select w.start_date, w.end_date, \n" + + " coalesce(percentile_cont(0.5) within group (order by (st.sync_end_time-st.sync_start_time)), '00:00:00') as median_sync_time\n" + + "from weeks w\t\n" + + "left join ${schemaName}.sync_telemetry st\n" + + "on st.sync_start_time::date >= w.start_date and st.sync_end_time::date <= w.end_date\n" + + "and st.sync_source = 'manual'\n" + + "group by 1,2;"; + String query = baseQuery + .replace("${schemaName}", OrgIdentityContextHolder.getDbSchema()) + .replace("${syncTelemetryWhere}", syncTelemetryWhere); + + runInOrgContext(() -> { + jdbcTemplate.execute(query); + return NullObject.instance(); + }, jdbcTemplate); + + return namedJdbcTemplate.query(query, new MedianSyncMapper()); + } + + public List generateCompletedVisitsOnTimeByProportion(String proportionCondition, String encounterWhere) { + SchemaMetadata schema = schemaMetadataRepository.getExistingSchemaMetadata(); + List encounterTableNames = schema.getAllEncounterTableNames().stream().toList(); + List programEncounterTableNames = schema.getAllProgramEncounterTableNames().stream().toList(); + + ST baseQuery = new ST("with program_enc_data as (\n " + + " select last_modified_by_id, \n" + + " count(*) filter ( where encounter_date_time \\<= max_visit_date_time ) visits_done_on_time,\n" + + " count(*) filter ( where encounter_date_time notnull and earliest_visit_date_time notnull ) total_scheduled\n" + + " from $schemaName.\n" + + " where is_voided = false\n" + + " $encounterWhere\n" + + " group by last_modified_by_id\n" + + " \n" + + " where is_voided = false\n" + + " $encounterWhere\n" + + " group by last_modified_by_id\n }> " + + "),\n" + + "general_enc_data as (\n" + + " select last_modified_by_id,\n" + + " count(*) filter ( where encounter_date_time \\<= max_visit_date_time ) visits_done_on_time,\n" + + " count(*) filter ( where encounter_date_time notnull and earliest_visit_date_time notnull ) total_scheduled\n" + + " from $schemaName.\n" + + " where is_voided = false\n" + + " $encounterWhere\n" + + " group by last_modified_by_id\n" + + "\n" + + " where is_voided = false\n" + + " $encounterWhere\n" + + " group by last_modified_by_id \n }>" + + "),\n" + + "total_visits_table as (\n" + + " select coalesce(u.name, u.username) as indicator,\n" + + " coalesce(sum(distinct ged.visits_done_on_time), 0) as ged_visits_on_time,\n" + + " coalesce(sum(distinct ped.visits_done_on_time), 0) as ped_visits_on_time,\n" + + " coalesce(sum(distinct ged.total_scheduled), 0) as ged_total_scheduled,\n" + + " coalesce(sum(distinct ped.total_scheduled), 0) as ped_total_scheduled\n" + + " from $schemaName.users u \n" + + " left join general_enc_data ged on ged.last_modified_by_id = u.id\n" + + " left join program_enc_data ped on ped.last_modified_by_id = u.id\n" + + " where u.organisation_id notnull\n" + + " and (is_voided = false or is_voided isnull)\n" + + " group by u.name, u.username \n" + + ")\n" + + "select indicator,\n" + + " ged_visits_on_time + ped_visits_on_time as count\n" + + "from total_visits_table\n" + + "where ged_visits_on_time + ped_visits_on_time > 0\n" + + " and ((ged_visits_on_time + ped_visits_on_time) /\n" + + "nullif(ged_total_scheduled + ped_total_scheduled, 0)) $proportion_condition;" + ); + baseQuery.add("programEncounterTableNames", programEncounterTableNames) + .add("encounterTableNames", encounterTableNames); + String query = baseQuery.render().replace("$proportion_condition", proportionCondition) + .replace("$schemaName", OrgIdentityContextHolder.getDbSchema()) + .replace("$encounterWhere", encounterWhere); + + runInOrgContext(() -> { + jdbcTemplate.execute(query); + return NullObject.instance(); + }, jdbcTemplate); + + return namedJdbcTemplate.query(query, new AggregateReportMapper()); + } + + public List generateUserCancellingMostVisits(String encounterWhere) { + SchemaMetadata schema = schemaMetadataRepository.getExistingSchemaMetadata(); + List encounterTableNames = schema.getAllEncounterTableNames().stream().toList(); + List programEncounterTableNames = schema.getAllProgramEncounterTableNames().stream().toList(); + + ST baseQuery = new ST("with program_enc_data as (\n " + + " select last_modified_by_id,\n" + + " count(*) filter ( where cancel_date_time notnull ) cancelled_visits\n" + + " from $schemaName.\n" + + " where is_voided = false\n" + + " $encounterWhere\n" + + " group by last_modified_by_id\n" + + "\n" + + " where is_voided = false\n" + + " $encounterWhere\n" + + " group by last_modified_by_id \n }>" + + "),\n" + + "general_enc_data as (\n" + + " select last_modified_by_id,\n" + + " count(*) filter ( where cancel_date_time notnull ) cancelled_visits\n" + + " from $schemaName.\n" + + " where is_voided = false\n" + + " $encounterWhere\n" + + " group by last_modified_by_id\n" + + "\n" + + " where is_voided = false\n" + + " $encounterWhere\n" + + " group by last_modified_by_id \n }>" + + "),\n" + + "cancelled_visits_table as (\n" + + "select coalesce(u.name, u.username) as indicator,\n" + + " coalesce(sum(distinct ged.cancelled_visits), 0) as ged_cancelled_visits,\n" + + " coalesce(sum(distinct ped.cancelled_visits), 0) as ped_cancelled_visits\n" + + "from $schemaName.users u\n" + + " left join general_enc_data ged on ged.last_modified_by_id = u.id\n" + + " left join program_enc_data ped on ped.last_modified_by_id = u.id\n" + + "where u.organisation_id notnull\n" + + " and (is_voided = false or is_voided isnull)\n" + + " and coalesce(ged.cancelled_visits, 0) + coalesce(ped.cancelled_visits, 0) > 0 \n" + + " group by u.name,u.username \n" + + ")\n" + + "select indicator,\n" + + " ged_cancelled_visits + ped_cancelled_visits as count\n" + + "from cancelled_visits_table\n" + + "order by ged_cancelled_visits + ped_cancelled_visits desc\n" + + "limit 5;" + ); + baseQuery.add("programEncounterTableNames", programEncounterTableNames) + .add("encounterTableNames", encounterTableNames); + String query = baseQuery.render().replace("$schemaName", OrgIdentityContextHolder.getDbSchema()) + .replace("$encounterWhere", encounterWhere); + + runInOrgContext(() -> { + jdbcTemplate.execute(query); + return NullObject.instance(); + }, jdbcTemplate); + + return namedJdbcTemplate.query(query, new AggregateReportMapper()); + } +} diff --git a/src/main/java/org/avniproject/etl/repository/rowMappers/reports/AggregateReportMapper.java b/src/main/java/org/avniproject/etl/repository/rowMappers/reports/AggregateReportMapper.java new file mode 100644 index 0000000..d214b57 --- /dev/null +++ b/src/main/java/org/avniproject/etl/repository/rowMappers/reports/AggregateReportMapper.java @@ -0,0 +1,19 @@ +package org.avniproject.etl.repository.rowMappers.reports; + +import org.avniproject.etl.dto.AggregateReportResult; +import org.springframework.jdbc.core.RowMapper; + +import java.sql.ResultSet; +import java.sql.SQLException; + +public class AggregateReportMapper implements RowMapper { + @Override + public AggregateReportResult mapRow(ResultSet rs, int rowNum) throws SQLException { + AggregateReportResult aggregateReportResult = new AggregateReportResult(); + aggregateReportResult.setLabel(rs.getString("indicator")); + aggregateReportResult.setValue(rs.getLong("count")); + aggregateReportResult.setId(rs.getString("indicator")); + + return aggregateReportResult; + } +} diff --git a/src/main/java/org/avniproject/etl/repository/rowMappers/reports/LatestSyncMapper.java b/src/main/java/org/avniproject/etl/repository/rowMappers/reports/LatestSyncMapper.java new file mode 100644 index 0000000..640d437 --- /dev/null +++ b/src/main/java/org/avniproject/etl/repository/rowMappers/reports/LatestSyncMapper.java @@ -0,0 +1,25 @@ +package org.avniproject.etl.repository.rowMappers.reports; + +import org.avniproject.etl.dto.UserActivityDTO; +import org.joda.time.DateTime; +import org.springframework.jdbc.core.RowMapper; + +import java.sql.ResultSet; +import java.sql.SQLException; + +public class LatestSyncMapper implements RowMapper { + + @Override + public UserActivityDTO mapRow(ResultSet rs, int rowNum) throws SQLException { + UserActivityDTO userActivityDTO = new UserActivityDTO(); + userActivityDTO.setUserName(rs.getString("name")); + userActivityDTO.setAndroidVersion(rs.getString("android_version")); + userActivityDTO.setAppVersion(rs.getString("app_version")); + userActivityDTO.setDeviceModel(rs.getString("device_name")); + userActivityDTO.setSyncStart(new DateTime(rs.getDate("sync_start_time"))); + userActivityDTO.setSyncEnd(new DateTime(rs.getDate("sync_end_time"))); + userActivityDTO.setSyncStatus(rs.getString("sync_status")); + userActivityDTO.setSyncSource(rs.getString("sync_source")); + return userActivityDTO; + } +} diff --git a/src/main/java/org/avniproject/etl/repository/rowMappers/reports/MedianSyncMapper.java b/src/main/java/org/avniproject/etl/repository/rowMappers/reports/MedianSyncMapper.java new file mode 100644 index 0000000..4b905e9 --- /dev/null +++ b/src/main/java/org/avniproject/etl/repository/rowMappers/reports/MedianSyncMapper.java @@ -0,0 +1,19 @@ +package org.avniproject.etl.repository.rowMappers.reports; + +import org.avniproject.etl.dto.UserActivityDTO; +import org.joda.time.DateTime; +import org.springframework.jdbc.core.RowMapper; + +import java.sql.ResultSet; +import java.sql.SQLException; + +public class MedianSyncMapper implements RowMapper { + @Override + public UserActivityDTO mapRow(ResultSet rs, int rowNum) throws SQLException { + UserActivityDTO userActivityDTO = new UserActivityDTO(); + userActivityDTO.setSyncStart(new DateTime(rs.getDate("start_date"))); + userActivityDTO.setSyncEnd(new DateTime(rs.getDate("end_date"))); + userActivityDTO.setMedianSync(rs.getString("median_sync_time")); + return userActivityDTO; + } +} diff --git a/src/main/java/org/avniproject/etl/repository/rowMappers/reports/SummaryTableMapper.java b/src/main/java/org/avniproject/etl/repository/rowMappers/reports/SummaryTableMapper.java new file mode 100644 index 0000000..d882cde --- /dev/null +++ b/src/main/java/org/avniproject/etl/repository/rowMappers/reports/SummaryTableMapper.java @@ -0,0 +1,17 @@ +package org.avniproject.etl.repository.rowMappers.reports; + +import org.avniproject.etl.dto.UserActivityDTO; +import org.springframework.jdbc.core.RowMapper; + +import java.sql.ResultSet; +import java.sql.SQLException; + +public class SummaryTableMapper implements RowMapper { + @Override + public UserActivityDTO mapRow(ResultSet rs, int rowNum) throws SQLException { + UserActivityDTO userActivityDTO = new UserActivityDTO(); + userActivityDTO.setTableName(rs.getString("name")); + userActivityDTO.setTableType(rs.getString("type")); + return userActivityDTO; + } +} diff --git a/src/main/java/org/avniproject/etl/repository/rowMappers/reports/UserActivityMapper.java b/src/main/java/org/avniproject/etl/repository/rowMappers/reports/UserActivityMapper.java new file mode 100644 index 0000000..c2ab736 --- /dev/null +++ b/src/main/java/org/avniproject/etl/repository/rowMappers/reports/UserActivityMapper.java @@ -0,0 +1,21 @@ +package org.avniproject.etl.repository.rowMappers.reports; + +import org.avniproject.etl.dto.UserActivityDTO; +import org.springframework.jdbc.core.RowMapper; + +import java.sql.ResultSet; +import java.sql.SQLException; + +public class UserActivityMapper implements RowMapper { + @Override + public UserActivityDTO mapRow(ResultSet rs, int rowNum) throws SQLException { + UserActivityDTO userActivityDTO = new UserActivityDTO(); + userActivityDTO.setId(rs.getLong("id")); + userActivityDTO.setUserName(rs.getString("name")); + userActivityDTO.setRegistrationCount(rs.getLong("registration_count")); + userActivityDTO.setGeneralEncounterCount(rs.getLong("encounter_count")); + userActivityDTO.setProgramEnrolmentCount(rs.getLong("enrolment_count")); + userActivityDTO.setProgramEncounterCount(rs.getLong("program_encounter_count")); + return userActivityDTO; + } +} diff --git a/src/main/java/org/avniproject/etl/repository/rowMappers/reports/UserCountMapper.java b/src/main/java/org/avniproject/etl/repository/rowMappers/reports/UserCountMapper.java new file mode 100644 index 0000000..3ce01a9 --- /dev/null +++ b/src/main/java/org/avniproject/etl/repository/rowMappers/reports/UserCountMapper.java @@ -0,0 +1,17 @@ +package org.avniproject.etl.repository.rowMappers.reports; + +import org.avniproject.etl.dto.UserActivityDTO; +import org.springframework.jdbc.core.RowMapper; + +import java.sql.ResultSet; +import java.sql.SQLException; + +public class UserCountMapper implements RowMapper { + @Override + public UserActivityDTO mapRow(ResultSet rs, int rowNum) throws SQLException { + UserActivityDTO userActivityDTO = new UserActivityDTO(); + userActivityDTO.setUserName(rs.getString("name")); + userActivityDTO.setCount(rs.getLong("count")); + return userActivityDTO; + } +} diff --git a/src/main/java/org/avniproject/etl/repository/rowMappers/reports/UserDetailsMapper.java b/src/main/java/org/avniproject/etl/repository/rowMappers/reports/UserDetailsMapper.java new file mode 100644 index 0000000..5f95123 --- /dev/null +++ b/src/main/java/org/avniproject/etl/repository/rowMappers/reports/UserDetailsMapper.java @@ -0,0 +1,20 @@ +package org.avniproject.etl.repository.rowMappers.reports; + +import org.avniproject.etl.dto.UserActivityDTO; +import org.joda.time.DateTime; +import org.springframework.jdbc.core.RowMapper; + +import java.sql.ResultSet; +import java.sql.SQLException; + +public class UserDetailsMapper implements RowMapper { + @Override + public UserActivityDTO mapRow(ResultSet rs, int rowNum) throws SQLException { + UserActivityDTO userActivityDTO = new UserActivityDTO(); + userActivityDTO.setUserName(rs.getString("name")); + userActivityDTO.setAppVersion(rs.getString("app_version")); + userActivityDTO.setDeviceModel(rs.getString("device_model")); + userActivityDTO.setLastSuccessfulSync(new DateTime(rs.getDate("sync_start_time"))); + return userActivityDTO; + } +} diff --git a/src/main/java/org/avniproject/etl/util/ObjectMapperSingleton.java b/src/main/java/org/avniproject/etl/util/ObjectMapperSingleton.java new file mode 100644 index 0000000..ede7d43 --- /dev/null +++ b/src/main/java/org/avniproject/etl/util/ObjectMapperSingleton.java @@ -0,0 +1,17 @@ +package org.avniproject.etl.util; + +import com.fasterxml.jackson.datatype.joda.JodaModule; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.PropertySource; + +@Configuration +@PropertySource("classpath:application.properties") +public class ObjectMapperSingleton { + + @Bean + public JodaModule jodaModule() { + JodaModule module = new JodaModule(); + return module; + } +} \ No newline at end of file diff --git a/src/main/java/org/avniproject/etl/util/ReportUtil.java b/src/main/java/org/avniproject/etl/util/ReportUtil.java new file mode 100644 index 0000000..98436b1 --- /dev/null +++ b/src/main/java/org/avniproject/etl/util/ReportUtil.java @@ -0,0 +1,32 @@ +package org.avniproject.etl.util; + +import org.springframework.stereotype.Service; + +import java.util.List; + +import static java.lang.String.format; + +@Service +public class ReportUtil { + public String getDateDynamicWhere(String startDate, String endDate, String columnName) { + if (startDate != null) { + return format("and %s::date between '%s'::date and '%s'::date", columnName, startDate, endDate); + } + return ""; + } + + public String getDateSeries(String startDate, String endDate) { + if (startDate != null) { + return format("from generate_series('%s'::date - interval '3 months', '%s'::date, '7d'::interval) day", startDate, endDate); + } + return "from generate_series(current_date at time zone 'UTC'+'5:30'- interval '3 months' , current_date at time zone 'UTC'+'5:30' , '7d'::interval) day"; + } + + public String getDynamicUserWhere(List userIds, String columnName) { + if (!userIds.isEmpty()) { + return format("and %s in (%s)", columnName, StringUtil.joinLongToList(userIds)); + } + return ""; + } + +} diff --git a/src/main/java/org/avniproject/etl/util/StringUtil.java b/src/main/java/org/avniproject/etl/util/StringUtil.java new file mode 100644 index 0000000..8f0ccf4 --- /dev/null +++ b/src/main/java/org/avniproject/etl/util/StringUtil.java @@ -0,0 +1,13 @@ +package org.avniproject.etl.util; + +import java.util.List; +import java.util.stream.Collectors; + +public class StringUtil { + + public static String joinLongToList(List lists) { + return lists.isEmpty() ? "" : lists.stream().map(String::valueOf) + .collect(Collectors.joining(",")); + } + +} diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 1446e28..f4ef015 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -2,3 +2,5 @@ avni.database=${OPENCHS_DATABASE_NAME:openchs} spring.config.import=classpath:/main-application.properties debug=${ETL_DEBUG_MODE:false} avni.current.time.offset.seconds=${AVNI_CURRENT_TIME_OFFSET_SECONDS:10} +spring.jackson.serialization.write-dates-as-timestamps=false +spring.jackson.default-property-inclusion=non_null